Triforce v0.2.0

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
ruckus7382sfgsgdf
2026-11-10 19:32:48 -06:00
commit 34b3683441
1730 changed files with 554853 additions and 0 deletions

85
nx2/adr_entries.py Normal file
View File

@@ -0,0 +1,85 @@
"""Code addresses the code itself takes (ADR, or ADRP followed by ADD on the same register): extra function entries.
Discovery starts functions at .eh_frame starts, exports and call targets. A function the game only reaches through
a pointer it builds with ADR/ADRP+ADD (a callback handed to a formatter, say) and that has no unwind entry is not
found, and the lifted code then has nothing to call: "indirect call to an address with no function" (OoT: the
character-output callback at 0x824c5560 of a printf-style formatter, on the thread Porter). MKW had none.
Most such addresses are labels inside functions (jump-table bases, landing pads). As entries those are costly: each
one lifts the rest of its function again, and in OoT the 1,512 new ones added 1.68 M instructions (+9.5%), mostly
inside a few 30k-instruction functions near 0x81b8c000, which made one chunk 122 MB and took clang hours. So a
target strictly inside a function the unwind info covers (its FDE's [pc_begin, pc_begin + pc_range)) is dropped:
such functions are found from .eh_frame anyway. lift.py adds the rest unless --no-adr-entries.
"""
import struct
import numpy as np
def fde_spans(modules):
"""(start, end) of every function .eh_frame_hdr's table points to an FDE for, absolute addresses, sorted.
The table is (initial_loc, fde) pairs, datarel sdata4 from the header. The FDE: u32 length, u32 CIE pointer,
pc_begin (pcrel sdata4, as the SDK's toolchain emits), pc_range (u32). An FDE whose pc_begin disagrees with
the table is skipped rather than guessed."""
spans = []
for m in modules:
s, e = m.eh_frame_hdr
if e <= s:
continue
img = m.image
ver, ptr_enc, cnt_enc, tbl_enc = img[s:s + 4]
if ver != 1 or tbl_enc != 0x3B or cnt_enc != 0x03:
continue
count = struct.unpack_from("<I", img, s + 8)[0]
for i in range(count):
loc, fde = struct.unpack_from("<ii", img, s + 12 + 8 * i)
loc, fde = s + loc, s + fde
if fde + 16 > len(img):
continue
length, cie, begin, rng = struct.unpack_from("<IIiI", img, fde)
if length < 12 or cie == 0 or fde + 8 + begin != loc:
continue
spans.append((m.base + loc, m.base + loc + rng))
spans.sort()
return spans
def code_address_refs(image_bin, meta, max_ahead=4, spans=None):
"""The 4-aligned code addresses formed by ADR / ADRP+ADD in the image's .text segments (with spans: only the
ones not strictly inside one of those functions)"""
base = meta["base"]
texts = [(s["addr"], s["addr"] + s["size"]) for m in meta["modules"] for s in m["segments"] if s["name"] == ".text"]
found = []
for lo, hi in texts:
w = np.frombuffer(image_bin[lo - base:hi - base], dtype="<u4").astype(np.int64)
pcs = lo + 4 * np.arange(len(w), dtype=np.int64)
imm = (((w >> 5) & 0x7FFFF) << 2) | ((w >> 29) & 3)
imm = np.where(imm & (1 << 20), imm - (1 << 21), imm)
adr = (w & 0x9F000000) == 0x10000000
found.append(pcs[adr] + imm[adr])
idx = np.nonzero((w & 0x9F000000) == 0x90000000)[0]
page = (pcs & ~0xFFF) + (imm << 12)
for d in range(1, max_ahead + 1):
j = idx + d
j = j[j < len(w)]
i = j - d
y = w[j]
add = ((y & 0xFFC00000) == 0x91000000) & (((y >> 5) & 31) == (w[i] & 31)) # ADD Xd, Xn, #imm12
found.append(page[i][add] + ((y[add] >> 10) & 0xFFF))
t = np.concatenate(found) if found else np.zeros(0, np.int64)
inside = np.zeros(len(t), bool)
for lo, hi in texts:
inside |= (t >= lo) & (t < hi)
t = np.unique(t[inside & (t % 4 == 0)])
if spans:
lo = np.array([a for a, _ in spans], np.int64)
hi = np.array([b for _, b in spans], np.int64)
# the latest-ending span among those starting at or before t (spans may nest or repeat)
best = np.maximum.accumulate(hi)
k = np.searchsorted(lo, t, side="right") - 1
ok = k >= 0
within = np.zeros(len(t), bool)
within[ok] = (t[ok] < best[k[ok]]) & ~np.isin(t[ok], lo)
t = t[~within]
return {int(a) for a in t}

1150
nx2/build.py Normal file

File diff suppressed because it is too large Load Diff

64
nx2/census.py Normal file
View File

@@ -0,0 +1,64 @@
"""Instruction census: every distinct word in each module's .text, through the lifter's classify() and suyu's
translator (a64emit), counted by capstone mnemonic. Counts occurrences, so data in .text and padding are included;
the per-function numbers come from lift.py (discovered code only).
Usage: python -I nx2/census.py [--exefs <dir>] [--modules main,sdk] [--top 40] [--nx2] (with the Switch 2 forms)
"""
import argparse
import collections
import os
import struct
import sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # (python -I leaves the script folder off)
import kit
import nso2 # noqa: F401 (DT_RELR)
import nso
import a64emit
import a64_lift
import capstone
def main():
p = argparse.ArgumentParser()
p.add_argument("--exefs", default=kit.EXEFS)
p.add_argument("--modules", default="")
p.add_argument("--top", type=int, default=40)
p.add_argument("--nx2", action="store_true", help="apply the Switch 2 instruction extension first")
a = p.parse_args()
a.exefs = a.exefs or kit.need_exefs()
if a.nx2:
import insn
insn.install()
md = capstone.Cs(capstone.CS_ARCH_ARM64, capstone.CS_MODE_ARM)
want = set(a.modules.split(",")) if a.modules else None
total = collections.Counter()
for m in nso.load_exefs(a.exefs):
if want and m.name not in want:
continue
lo, hi = m.code_range()
words = struct.unpack_from("<%dI" % ((hi - lo) // 4), m.image, lo)
count = collections.Counter(words)
ops = [w for w in count if a64_lift.classify(w, 0x10000)[0] == "op"]
_, flags = a64emit.translate([(0x10000, w) for w in ops])
bad = collections.Counter()
bad_words = collections.Counter()
for w, f in zip(ops, flags):
if f & a64emit.UNHANDLED:
ins = next(md.disasm(struct.pack("<I", w), 0x10000), None)
key = ins.mnemonic if ins else "(not an instruction)"
bad[key] += count[w]
bad_words[key] += 1
nbad = sum(bad.values())
print("== %s: %d words, %d distinct, %d distinct translatable-kind; untranslated %d words (%.3f%%) in %d distinct"
% (m.name, len(words), len(count), len(ops), nbad, 100.0 * nbad / len(words), sum(bad_words.values())))
for k, v in bad.most_common(a.top):
print(" %9d %-22s (%d distinct)" % (v, k, bad_words[k]))
total.update(bad)
print("== total untranslated by mnemonic")
for k, v in total.most_common(a.top):
print(" %9d %s" % (v, k))
if __name__ == "__main__":
main()

304
nx2/chunks.py Normal file
View File

@@ -0,0 +1,304 @@
"""Stable chunk boundaries for the lifted C (lift.py --write).
a64-kit's write() closes a chunk once it reaches CHUNK_BYTES of text, so a function that grows (a jump table found,
a fix in one form) moves every boundary after it, and build.py recompiles every chunk from there on: a fix to two
functions cost a full 45-minute rebuild. Here the boundaries are guest addresses, kept in work/image/chunks.txt:
a chunk ends after the last function below the next boundary, so only the chunks holding changed functions differ.
Without the file (or with --rechunk) the boundaries come from the sizes, as before, and are saved.
write()'s test is `size >= chunk_bytes`. chunk_bytes is a Boundaries instance: an int whose reflected comparison
(__le__, which Python tries first for a subclass on the right) decides from the function just emitted.
The functions' C is made by several processes (fork, so Linux and macOS; one process elsewhere): each emits a run of
consecutive functions from the lifter as it stands, and write() takes the texts in its own order, as if it had made
them. The lifter's counters (stats, unhandled, ijump, and any passed in `counters`, such as poll.stats) get each
process's additions. The output is the one-process output, byte for byte (tests/test_parallel_emit.py).
"""
import bisect
import collections
import glob
import multiprocessing
import os
import re
import whole_write # (rewrites written beside and moved over: issue 1197)
import a64_lift
import lifted_c
FILE = "chunks.txt"
class Boundaries(int):
def __new__(cls, size, starts, funcs):
self = super().__new__(cls, size)
self.starts = sorted(starts)
self.recorded = [] # (without saved boundaries) where the size rule started a chunk
self.current = None
self.after = None # the function emitted after `current`
order = sorted(funcs)
self.ends = set()
for b in self.starts:
i = bisect.bisect_left(order, b)
if 0 < i < len(order):
self.ends.add(order[i - 1])
self.next_of = {a: b for a, b in zip(order, order[1:])}
return self
def __le__(self, size): # `size >= self`
e = self.current
if self.starts:
return e in self.ends
if size >= int(self):
nxt = self.next_of.get(e)
if nxt is not None:
self.recorded.append(nxt)
return True
return False
def load(img_dir):
path = os.path.join(img_dir, FILE)
if not os.path.exists(path):
return []
return [int(l, 16) for l in open(path) if l.strip() and not l.startswith("#")]
def save(img_dir, starts):
with open(os.path.join(img_dir, FILE), "w") as f:
f.write("# first guest address of each lifted C chunk after the first (nx2/chunks.py)\n")
f.write("".join("%08x\n" % s for s in sorted(starts)))
def seed_from_gen(gen_dir):
"""The boundaries of an existing size-chunked lift: the first function defined in each code_NNN.c after the
first"""
starts = []
pat = re.compile(r"^void f_([0-9a-f]+)\(Ctx \*__restrict A64_NOESC c\) \{")
n = 1
while os.path.exists(os.path.join(gen_dir, "code_%03d.c" % n)):
for line in open(os.path.join(gen_dir, "code_%03d.c" % n)):
m = pat.match(line)
if m:
starts.append(int(m.group(1), 16))
break
n += 1
return starts
_job = None # (the forked workers' view: the emitter, the instruction texts and the counters, inherited)
def _counts(counters):
return [collections.Counter(c) for c in counters]
def _emit_run(es):
emit, texts, counters = _job
before = _counts(counters)
out = [(e,) + tuple(emit(e, texts)) for e in es]
return out, [collections.Counter({k: v - b.get(k, 0) for k, v in c.items() if v != b.get(k, 0)})
for b, c in zip(before, counters)]
def _available_gb():
"""MemAvailable from /proc/meminfo in GB (None where there is none)"""
try:
with open("/proc/meminfo", encoding="ascii") as f:
for line in f:
if line.startswith("MemAvailable:"):
return int(line.split()[1]) / (1024 * 1024)
except (OSError, ValueError, IndexError):
pass
return None
def emit_jobs():
"""Processes for the emit: HOOKSHOT_LIFT_JOBS, else the cores this process may use (at most 16; each holds its run's
texts and the pages of the lift it touches), and at most one per 2 GB of available memory: a Steam Deck (16 GB) ran
out with 8, and the system's out-of-memory handling stopped the lift ("lift.py stopped (signal 15): BrokenPipeError")
or one worker (the update then sat at "jump tables" for good). 1 where there is no fork"""
if "fork" not in multiprocessing.get_all_start_methods():
return 1
if os.environ.get("HOOKSHOT_LIFT_JOBS"):
return max(1, int(os.environ["HOOKSHOT_LIFT_JOBS"]))
try:
n = len(os.sched_getaffinity(0))
except AttributeError:
n = os.cpu_count() or 1
n = max(1, min(16, n))
gb = _available_gb()
if gb is not None:
n = max(1, min(n, int(gb // 2)))
return n
def parallel_emit(emit, order, texts, counters, jobs, run=2000):
"""(e, text, refs) for every function of order, in order, emitted by `jobs` forked processes. A worker that dies
(stopped for memory) ends the pool with an error instead of a wait forever (multiprocessing.Pool's imap never
returns then); the runs not done yet are emitted here, one process."""
global _job
if jobs <= 1 or len(order) < 2 * run:
for e in order:
yield (e,) + tuple(emit(e, texts))
return
import gc
from concurrent.futures import ProcessPoolExecutor
from concurrent.futures.process import BrokenProcessPool
runs = [order[i:i + run] for i in range(0, len(order), run)]
done = 0
_job = (emit, texts, counters)
gc.collect()
gc.freeze() # (the lift's objects out of the collector's sight: its passes would write to every page the children share)
try:
ctx = multiprocessing.get_context("fork")
with ProcessPoolExecutor(max_workers=jobs, mp_context=ctx) as pool:
for out, deltas in pool.map(_emit_run, runs):
for c, d in zip(counters, deltas):
for k, v in d.items():
c[k] = c.get(k, 0) + v
done += 1
yield from out
except BrokenProcessPool:
print("emit: a worker process stopped (out of memory?): the last %d of %d runs are emitted in this process"
% (len(runs) - done, len(runs)), flush=True)
finally:
_job = None
gc.unfreeze()
for es in runs[done:]:
for e in es:
yield (e,) + tuple(emit(e, texts))
def write(L, gen_dir, img_dir, rechunk=False, counters=(), jobs=None):
"""L.write(gen_dir) with stable boundaries; returns its files. counters: dicts the emit adds to besides the
lifter's own (poll.stats); jobs: emit processes (emit_jobs())"""
starts = [] if rechunk else load(img_dir)
if not starts and not rechunk:
starts = seed_from_gen(gen_dir)
b = Boundaries(a64_lift.CHUNK_BYTES, starts, L.funcs)
base_emit = L.emit_function
jobs = emit_jobs() if jobs is None else jobs
made = None
def emit_function(e, texts):
nonlocal made
b.current = e
if made is None: # (the first call: write() has its texts now, and asks for the functions in sorted order)
made = parallel_emit(base_emit, sorted(L.funcs), texts,
[L.stats, L.unhandled, L.ijump] + list(counters), jobs)
got = next(made, None)
if got is None or got[0] != e:
raise SystemExit("chunks: write() asked for f_%08x, the emit made %s" % (e, "f_%08x" % got[0] if got else "nothing"))
return got[1], got[2]
L.emit_function = emit_function
try:
files = L.write(gen_dir, chunk_bytes=b)
finally:
del L.emit_function
if made is not None:
made.close() # (the processes go now, not when the generator is collected)
# (the numbered chunks only: files has code_hot.c too, and counting it skipped the first stale chunk, linked twice)
n = sum(1 for f in files if re.fullmatch(r"code_\d{3}\.c", os.path.basename(f)))
while os.path.exists(os.path.join(gen_dir, "code_%03d.c" % n)): # (fewer chunks than the last lift)
os.remove(os.path.join(gen_dir, "code_%03d.c" % n))
n += 1
# (an earlier lift's hot pieces go now, with or without a new code_hot.c, before the patches and the page-mark groups read
# code_*.c: split_hot.py makes this lift's after it. A lift with no code_hot.c (no hot list, A64_HOT_CHUNK unset, or none of the
# listed functions in it): split_hot.py had renamed the old one code_hot.c.whole and left code_hot_*.c, which build.py compiled
# again, the earlier lift's hot functions, each defined in a chunk too, linked twice. A lift with one: the old pieces, patched
# already, and the new whole file each held the output-size places, and nx2/oot_patches.py's count of them failed the lift)
for f in os.listdir(gen_dir):
if f == "code_hot.c.whole" or HOT_PIECE.fullmatch(f):
os.remove(os.path.join(gen_dir, f))
files += split_big(gen_dir, files)
save(img_dir, starts or b.recorded)
print("chunks: %d, boundaries %s" % (len(files), "kept" if starts else "from sizes (saved)"))
return files
HEAD = lifted_c.HEAD_NOTED
PART = re.compile(r"code_(\d{3})_(\d+)\.c")
HOT_PIECE = re.compile(r"code_hot_\d+\.c") # (nx2/split_hot.py's pieces of code_hot.c)
# (a function's compile time grows with the square of its size: LLVM's passes over one function of tens of thousands of blocks. Fitted to
# a cold build of the 48 chunks at -O1 on a 24-thread desktop: seconds ~ 1.5 x MB + 4.4 x MB^2 per function, so a function of QUAD bytes
# costs twice its size. A 24 MB chunk of small functions took ~45 s; one of 20 MB holding a few 2 MB functions, 350 s)
QUAD = 350000
COSTS = "chunk_costs.txt" # (in the gen folder: each file's estimated compile cost, for build.py's order)
def fn_cost(size):
"""A function's estimated compile cost, in bytes of plain code"""
return size + size * size // QUAD
def text_cost(text):
"""The estimated compile cost of a C file of lifted functions"""
marks = [m.start() for m in HEAD.finditer(text)] + [len(text)]
return marks[0] + sum(fn_cost(b - a) for a, b in zip(marks, marks[1:]))
def write_costs(gen_dir, costs):
"""Adds or replaces files' costs in chunk_costs.txt; costs: {file name: cost}"""
path = os.path.join(gen_dir, COSTS)
known = {}
if os.path.exists(path):
for line in open(path):
name, _, n = line.partition(" ")
if n.strip().isdigit() and os.path.exists(os.path.join(gen_dir, name)):
known[name] = int(n)
known.update(costs)
with open(path, "w", newline="\n") as f:
f.write("".join("%s %d\n" % kv for kv in sorted(known.items())))
def split_big(gen_dir, files, budget=None):
"""Chunks whose estimated compile time (fn_cost) is well past a plain chunk's, cut at function starts into parts of about equal
cost: code_NNN.c, code_NNN_1.c ... A chunk of the giant functions took 350 s, alone at the end of the build, where a chunk of
ordinary functions takes 45; the boundaries kept in chunks.txt also let chunks grow (44 MB beside 25). The first part keeps the
name, so seed_from_gen and the boundaries are as before. Writes chunk_costs.txt for every chunk; returns the new parts' paths"""
budget = budget or a64_lift.CHUNK_BYTES
made, costs = [], {}
for path in files:
m = re.fullmatch(r"code_(\d{3})\.c", os.path.basename(path))
if not m:
continue
with open(path, encoding="utf-8", newline="") as f:
text = f.read()
marks = [h.start() for h in HEAD.finditer(text)]
ends = marks[1:] + [len(text)]
cost = [fn_cost(b - a) for a, b in zip(marks, ends)]
total = sum(cost)
if total <= budget * 3 // 2 or len(marks) < 2:
costs[os.path.basename(path)] = marks[0] + total if marks else len(text)
continue
parts = -(-total // budget)
cuts, run, k = [0], 0, 1 # (indexes into marks where a part starts)
for i, c in enumerate(cost):
if i and run >= total * k // parts:
cuts.append(i)
k = run * parts // total + 1 # (past a giant function: the next share, not a sliver after it)
run += c
bounds = [marks[i] for i in cuts] + [len(text)]
preamble = text[:marks[0]]
for j in range(len(cuts)):
out = path if j == 0 else os.path.join(gen_dir, "code_%s_%d.c" % (m.group(1), j))
piece = text[:bounds[1]] if j == 0 else preamble + text[bounds[j]:bounds[j + 1]]
whole_write.write_whole(out, piece)
costs[os.path.basename(out)] = len(preamble) + sum(cost[cuts[j]:cuts[j + 1] if j + 1 < len(cuts) else len(cost)])
if j:
made.append(out)
print("chunks: %s (%d MB, cost %d MB) in %d parts" % (os.path.basename(path), len(text) >> 20, total >> 20, len(cuts)))
for f in os.listdir(gen_dir): # (parts of an earlier lift that this one did not make)
if PART.fullmatch(f) and os.path.join(gen_dir, f) not in made:
os.remove(os.path.join(gen_dir, f))
write_costs(gen_dir, costs)
return made
def edit_c(gen, stats, supported, whole=True):
"""(nx2/lift.py, after the lift writes its C; here, beside the writing, so it imports without the lift's tools) The edits of the written C, in this order: the game's patches (nx2/oot_patches.py, which find the lift's exact text; the
supported build's whole standalone lift fails when an output-size edit finds no place), then, last, the page marks of each
group of stores shared (a64_lift.group_marks), which changes that text. stats: the lift's, for its report"""
import oot_patches
oot_patches.apply(gen, protect_hearts=supported, required=supported and whole)
a64_lift.group_marks(sorted(glob.glob(os.path.join(glob.escape(gen), "code_*.c"))), stats)

31
nx2/disasm.py Normal file
View File

@@ -0,0 +1,31 @@
"""Disassemble the guest image (Eden layout) around an address, for reading crash reports.
python -I nx2/disasm.py 0x800bae78 # 16 instructions before and after
python -I nx2/disasm.py 0x800b9320 0x800b9500 # a range
"""
import sys, os
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import kit
import capstone
import image as eden_image
def main():
a = [int(x, 0) for x in sys.argv[1:]]
img = eden_image.EdenImage(kit.need_exefs())
lo, hi = (a[0] - 64, a[0] + 64) if len(a) == 1 else (a[0], a[1])
mod = next((m for m in img.modules if m.base <= lo < m.base + len(m.image)), None)
names = getattr(img, "names", {}) or {}
md = capstone.Cs(capstone.CS_ARCH_ARM64, capstone.CS_MODE_ARM)
for pc in range(lo & ~3, hi, 4):
w = img.word(pc)
if pc in names:
print(f"{names[pc]}:")
ins = next(md.disasm(w.to_bytes(4, "little"), pc), None)
text = f"{ins.mnemonic} {ins.op_str}" if ins else f".word {w:#010x}"
mark = " <--" if len(a) == 1 and pc == a[0] else ""
print(f" {pc:#x} {mod.name if mod else '?'}+{pc - (mod.base if mod else 0):#x}: {w:08x} {text}{mark}")
if __name__ == "__main__":
main()

105
nx2/disk.py Normal file
View File

@@ -0,0 +1,105 @@
"""Is a folder on a hard drive (a spinning disk) or on an SSD? A build reads and writes tens of gigabytes for the best part of an
hour, and an SSD makes that much faster. The answer is only ever a warning: a player may have no other drive, so the build goes
ahead on a hard drive.
spinning(path) -> True for a spinning disk, False for an SSD or flash, None when it cannot be told (a network share, an unknown
device, another operating system). Windows asks the volume for its seek penalty (IOCTL_STORAGE_QUERY_PROPERTY,
StorageDeviceSeekPenaltyProperty: a disk that seeks says so); Linux reads the kernel's queue/rotational for the device behind the
folder, through its partitions and device-mapper/RAID slaves.
"""
import os
import sys
def existing_parent(path):
"""The nearest folder above path that exists (the path's own folder may be made later, as the game folder often is)"""
p = os.path.abspath(os.path.expanduser(path or "."))
while not os.path.exists(p):
up = os.path.dirname(p)
if up == p:
break
p = up
return p
def spinning(path):
"""True on a hard drive, False on an SSD, None when the drive cannot be told"""
folder = existing_parent(path)
try:
if os.name == "nt":
return _windows_spinning(folder)
if sys.platform.startswith("linux"):
return _linux_spinning(folder)
except (OSError, ValueError, AttributeError, ImportError):
return None
return None
def _windows_spinning(path):
"""The volume's DEVICE_SEEK_PENALTY_DESCRIPTOR (kernel32's DeviceIoControl): a spinning disk incurs a seek penalty"""
import ctypes
from ctypes import wintypes
root = os.path.splitdrive(path)[0]
if not root:
return None
class Query(ctypes.Structure):
_fields_ = [("PropertyId", wintypes.DWORD), ("QueryType", wintypes.DWORD), ("Additional", wintypes.DWORD)]
class SeekPenalty(ctypes.Structure):
_fields_ = [("Version", wintypes.DWORD), ("Size", wintypes.DWORD), ("IncursSeekPenalty", ctypes.c_ubyte)]
kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
kernel32.CreateFileW.argtypes = [wintypes.LPCWSTR, wintypes.DWORD, wintypes.DWORD, ctypes.c_void_p,
wintypes.DWORD, wintypes.DWORD, wintypes.HANDLE]
kernel32.CreateFileW.restype = wintypes.HANDLE
kernel32.DeviceIoControl.argtypes = [wintypes.HANDLE, wintypes.DWORD, ctypes.c_void_p, wintypes.DWORD,
ctypes.c_void_p, wintypes.DWORD, ctypes.POINTER(wintypes.DWORD), ctypes.c_void_p]
kernel32.DeviceIoControl.restype = wintypes.BOOL
kernel32.CloseHandle.argtypes = [wintypes.HANDLE]
invalid = wintypes.HANDLE(-1).value
handle = kernel32.CreateFileW("\\\\.\\" + root, 0, 0x1 | 0x2, None, 3, 0, None) # (read + write share, OPEN_EXISTING)
if not handle or handle == invalid:
return None
try:
query, answer, returned = Query(7, 0, 0), SeekPenalty(), wintypes.DWORD() # (StorageDeviceSeekPenaltyProperty, standard query)
ok = kernel32.DeviceIoControl(handle, 0x002D1400, ctypes.byref(query), ctypes.sizeof(query), ctypes.byref(answer),
ctypes.sizeof(answer), ctypes.byref(returned), None)
if not ok or returned.value < ctypes.sizeof(answer):
return None
return bool(answer.IncursSeekPenalty)
finally:
kernel32.CloseHandle(handle)
def _linux_spinning(path):
"""queue/rotational of the block device behind path's file system"""
try:
st = os.stat(path)
except OSError:
return None
return _device_rotational("/sys/dev/block/%d:%d" % (os.major(st.st_dev), os.minor(st.st_dev)))
def _device_rotational(node, depth=0):
"""The device behind sysfs's node: its slaves decide for device-mapper and RAID (any spinning one makes it slow), and a
partition's own folder has no queue/rotational, so the disk above it is read"""
path = os.path.realpath(node)
if depth < 4:
slaves = os.path.join(path, "slaves")
try:
names = sorted(os.listdir(slaves))
except OSError:
names = []
answers = [a for a in (_device_rotational(os.path.join(slaves, n), depth + 1) for n in names) if a is not None]
if answers:
return any(answers)
while True:
try:
with open(os.path.join(path, "queue", "rotational"), encoding="ascii") as f:
return f.read().strip() == "1"
except OSError:
up = os.path.dirname(path)
if up == path:
return None
path = up

36
nx2/dllmap.py Normal file
View File

@@ -0,0 +1,36 @@
"""hookshot_game.dll offset -> lifted function (f_<guest address>) + offset, from the linker map (build.py --dll).
Usage: python -I nx2/dllmap.py <hex offset> [...]
"""
import bisect
import os
import re
import sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import kit
BASE = 0x180000000
def load():
rows = []
for line in open(os.path.join(kit.WORK, "build", "hookshot_game.map"), errors="replace"):
m = re.match(r"\s*[0-9a-f]{4}:[0-9a-f]{8}\s+(\S+)\s+([0-9a-f]{16})", line)
if m:
rows.append((int(m.group(2), 16), m.group(1)))
rows.sort()
return rows
def main():
rows = load()
keys = [r[0] for r in rows]
for a in sys.argv[1:]:
t = BASE + int(a, 16)
i = bisect.bisect_right(keys, t) - 1
print("%s: %s+%#x" % (a, rows[i][1], t - rows[i][0]) if i >= 0 else "%s: ?" % a)
if __name__ == "__main__":
main()

205
nx2/dump.py Normal file
View File

@@ -0,0 +1,205 @@
"""What a dump is: program id, Switch 1 or Switch 2, modules, build ids, SDK, graphics API, romfs size.
Reads the user's decrypted dump in place (exefs/ + romfs/); writes nothing into the repository. With --set it
records the dump's folder in work/dump.txt (ignored by git), where nx2/kit.py finds it.
Switch 1 or Switch 2 is decided from several independent signals, all of which must agree:
- the program id's prefix (Switch 1 titles are 01xx..., Switch 2 ones 04xx...);
- main.npdm's ACID layout (the Switch 1 ACID has its magic at +0x200; the Switch 2 one does not);
- the services the npdm allows (Switch 1 games open nvdrv and vi:u for NVN; Switch 2 ones go through nx2 gs/gpu);
- DT_RELR packed relocations (the Switch 2 toolchain emits them; the Switch 1 one emits RELA only);
- ARMv8.1+ instructions in main (Cortex-A78C: PAC returns, LSE atomics, LDAPR; the A57 has none of them);
- the NVN names in main (NVN2 adds DLSS and mesh-task procs) and the shader container names in romfs
(Nin_NX_NVN for Maxwell, Nin_Ounce_NVN2 for Ampere).
Usage: python -I nx2/dump.py [--dump <folder with exefs/ and romfs/>] [--set] [--no-romfs]
"""
import argparse
import collections
import os
import re
import struct
import sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # (python -I leaves the script folder off)
import kit
import nso2 # noqa: F401 (DT_RELR)
import nso
import numpy as np
def npdm(path):
b = open(path, "rb").read()
out = {"size": len(b)}
if b[:4] != b"META":
out["error"] = "no META magic"
return out
flags = b[0xC]
out["is64"] = bool(flags & 1)
out["address_space"] = {0: "32-bit", 1: "36-bit", 2: "32-bit (no reserved)", 3: "39-bit"}.get((flags >> 1) & 7,
"type %d" % ((flags >> 1) & 7))
out["main_thread_priority"] = b[0xE]
out["main_thread_core"] = b[0xF]
out["system_resource_size"] = struct.unpack_from("<I", b, 0x14)[0]
out["version"] = struct.unpack_from("<I", b, 0x18)[0]
out["main_thread_stack"] = struct.unpack_from("<I", b, 0x1C)[0]
out["name"] = b[0x20:0x30].split(b"\0")[0].decode("ascii", "replace")
out["product_code"] = b[0x30:0x40].split(b"\0")[0].decode("ascii", "replace")
aoff, asz, doff, dsz = struct.unpack_from("<IIII", b, 0x70)
aci = b[aoff:aoff + asz]
if aci[:4] == b"ACI0":
out["program_id"] = struct.unpack_from("<Q", aci, 0x10)[0]
sacoff, sacsz = struct.unpack_from("<II", aci, 0x28)
sac, i, names = aci[sacoff:sacoff + sacsz], 0, []
while i < len(sac):
c = sac[i]
n = (c & 7) + 1
names.append(sac[i + 1:i + 1 + n].decode("ascii", "replace") + (" (host)" if c & 0x80 else ""))
i += 1 + n
out["services"] = names
kacoff, kacsz = struct.unpack_from("<II", aci, 0x30)
kac = struct.unpack_from("<%dI" % (kacsz // 4), aci, kacoff)
svcs = set()
for d in kac:
low = (~d & (d + 1)) & 0xFFFFFFFF # the descriptor type is the count of trailing ones
if low == 1 << 4: # 0b01111: enabled SVCs, 24 per descriptor
idx = (d >> 29) & 7
mask = (d >> 5) & 0xFFFFFF
svcs |= {idx * 24 + j for j in range(24) if mask >> j & 1}
out["svcs"] = sorted(svcs)
acid = b[doff:doff + dsz]
out["acid_switch1_layout"] = acid[0x200:0x204] == b"ACID"
return out
def arm_v81_census(m):
"""occurrences in .text of instruction classes the Cortex-A57 (Switch 1) does not have"""
lo, hi = m.code_range()
w = np.frombuffer(bytes(m.image[lo:hi]), dtype=np.uint32)
c = collections.OrderedDict()
c["RETAA/RETAB"] = int(np.count_nonzero((w == 0xD65F0BFF) | (w == 0xD65F0FFF)))
c["PACIBSP/PACIASP"] = int(np.count_nonzero((w == 0xD503237F) | (w == 0xD503233F)))
c["LSE (LDADD..SWP)"] = int(np.count_nonzero((w & 0x3F200C00) == 0x38200000))
c["CAS/CASP"] = int(np.count_nonzero(((w & 0x3FA07C00) == 0x08A07C00) | ((w & 0xBFA07C00) == 0x08207C00)))
c["LDAPR"] = int(np.count_nonzero((w & 0x3FFFFC00) == 0x38BFC000))
c["words"] = int(len(w))
return c
def strings_with(m, pat, limit=12):
data = bytes(m.image)
out = []
for hit in re.finditer(pat, data):
s = data[hit.start():data.find(b"\0", hit.start())]
if s not in out:
out.append(s)
if len(out) >= limit:
break
return [s.decode("ascii", "replace") for s in out]
def romfs_summary(root, cap=400000):
files = size = 0
shaders = collections.Counter()
top = collections.Counter()
for d, dirs, fs in os.walk(root):
rel = os.path.relpath(d, root)
head = rel.split(os.sep)[0] if rel != "." else "."
for f in fs:
files += 1
try:
s = os.path.getsize(os.path.join(d, f))
except OSError:
s = 0
size += s
top[head] += s
for tag in ("Nin_NX_NVN", "Nin_Ounce_NVN2", "NVN2", ".bfsha", ".bnsh", ".bntx", ".sarc", ".zs", ".szs"):
if tag in f:
shaders[tag] += 1
if files >= cap:
break
return files, size, top, shaders
def main():
p = argparse.ArgumentParser()
p.add_argument("--dump", default=kit.DUMP, help="the folder with exefs/ and romfs/")
p.add_argument("--set", action="store_true", help="record --dump in work/dump.txt")
p.add_argument("--no-romfs", action="store_true")
a = p.parse_args()
if not a.dump:
raise SystemExit("give --dump <folder with exefs/ and romfs/>")
exefs = os.path.join(a.dump, "exefs")
romfs = os.path.join(a.dump, "romfs")
if not os.path.isdir(exefs):
raise SystemExit("%s: no exefs/" % a.dump)
if a.set:
os.makedirs(kit.WORK, exist_ok=True)
with open(os.path.join(kit.WORK, "dump.txt"), "w", encoding="utf-8") as f:
f.write("# the user's decrypted dump (exefs/ + romfs/), read in place; never copied into the repo\n%s\n"
% os.path.abspath(a.dump))
print("recorded in %s" % os.path.join(kit.WORK, "dump.txt"))
print("== dump: %s" % os.path.abspath(a.dump))
votes = {}
meta = None
if os.path.exists(os.path.join(exefs, "main.npdm")):
meta = npdm(os.path.join(exefs, "main.npdm"))
pid = meta.get("program_id")
print("npdm: name %r, product %r, %s, %s address space, main thread prio %d core %d stack %#x, "
"system resource %#x, version %d" % (
meta.get("name"), meta.get("product_code"), "64-bit" if meta.get("is64") else "32-bit",
meta.get("address_space"), meta.get("main_thread_priority", 0), meta.get("main_thread_core", 0),
meta.get("main_thread_stack", 0), meta.get("system_resource_size", 0), meta.get("version", 0)))
if pid is not None:
print("program id: %016X" % pid)
votes["program id prefix"] = {0x01: 1, 0x04: 2}.get(pid >> 56)
votes["ACID layout"] = 1 if meta.get("acid_switch1_layout") else 2
svc = meta.get("services", [])
print("services (%d): %s" % (len(svc), " ".join(svc)))
print("SVCs allowed (%d): %s" % (len(meta.get("svcs", [])), " ".join("%#x" % s for s in meta.get("svcs", []))))
votes["nvdrv in services"] = 1 if "nvdrv" in svc else 2
mods = nso.load_exefs(exefs)
print("modules (rtld's load order):")
print(" %-8s %-40s %12s %12s %12s %10s %9s %8s" % ("name", "build id", ".text", ".rodata", ".data", ".bss",
"eh starts", "RELR"))
relr = 0
for m in mods:
segs = {n: s for _, s, n in m.segments}
print(" %-8s %-40s %12s %12s %12s %10s %9s %8s" % (
m.name, m.build_id[:40], "{:,}".format(segs[".text"]), "{:,}".format(segs[".rodata"]),
"{:,}".format(segs[".data"]), "{:,}".format(m.bss_size), "{:,}".format(len(m.eh_funcs)),
"{:,}".format(getattr(m, "relr_count", 0))))
relr += getattr(m, "relr_count", 0)
votes["DT_RELR"] = 2 if relr else 1
main_m = next((m for m in mods if m.name == "main"), None)
sdk_m = next((m for m in mods if m.name == "sdk"), None)
if main_m:
c = arm_v81_census(main_m)
print("main .text ARMv8.1+ (Switch 2 CPU) forms: " + ", ".join("%s %s" % (k, "{:,}".format(v)) for k, v in c.items()))
v81 = c["RETAA/RETAB"] + c["PACIBSP/PACIASP"] + c["LSE (LDADD..SWP)"] + c["CAS/CASP"] + c["LDAPR"]
votes["ARMv8.1+ in main"] = 2 if v81 > c["words"] // 2000 else 1
nvn = sorted(set(strings_with(main_m, rb"nvn[A-Z][A-Za-z0-9]{3,80}\0", limit=100000)))
dlss = [s for s in nvn if "Dlss" in s or "MeshTask" in s]
print("main names %d nvn* procs (%d DLSS / mesh-task)" % (len(nvn), len(dlss)))
if nvn:
votes["NVN names"] = 2 if dlss else 1
for m in (sdk_m, main_m):
if m:
sdkv = strings_with(m, rb"SDK MW\+[^\0]{4,120}\0", limit=20)
if sdkv:
print("%s middleware strings: %s" % (m.name, "; ".join(sdkv[:12])))
if not a.no_romfs and os.path.isdir(romfs):
files, size, top, shaders = romfs_summary(romfs)
print("romfs: %s files, %.2f GiB; biggest folders: %s" % ("{:,}".format(files), size / 2 ** 30, ", ".join(
"%s %.2f GiB" % (k, v / 2 ** 30) for k, v in top.most_common(8))))
print("romfs names: " + ", ".join("%s %d" % kv for kv in shaders.most_common()))
if shaders["Nin_NX_NVN"] or shaders["Nin_Ounce_NVN2"] or shaders["NVN2"]:
votes["romfs shader names"] = 2 if (shaders["Nin_Ounce_NVN2"] or shaders["NVN2"]) else 1
print("platform signals: " + ", ".join("%s: Switch %s" % (k, v if v else "?") for k, v in votes.items()))
known = {v for v in votes.values() if v}
print("PLATFORM: %s" % ("Switch %d" % known.pop() if len(known) == 1 else "signals disagree: %s" % votes))
if __name__ == "__main__":
main()

63
nx2/fault.py Normal file
View File

@@ -0,0 +1,63 @@
"""Read an ArmRecomp crash report (EDEN_NX2_RECOMP_REPORT, run-mkw.ps1's recomp_fault.txt): the lifted function
the host rip is in, and for each guest return address of the backtrace, its function and the call before it.
Usage: python -I nx2/fault.py work/runs/<run>/recomp_fault.txt
"""
import bisect
import json
import os
import re
import sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import kit
import capstone
import dllmap
import image as eden_image
def main():
text = open(sys.argv[1]).read()
rows = dllmap.load()
keys = [r[0] for r in rows]
entries = sorted(int(e, 16) for e in json.load(open(os.path.join(kit.WORK, "image", "functions.json"))))
img = eden_image.EdenImage(kit.need_exefs())
md = capstone.Cs(capstone.CS_ARCH_ARM64, capstone.CS_MODE_ARM)
def func_of(pc):
i = bisect.bisect_right(entries, pc) - 1
if i < 0:
return "?"
m = next((m for m in img.modules if m.base <= pc < m.base + len(m.image)), None)
return "f_%x+%#x (%s+%#x)" % (entries[i], pc - entries[i], m.name if m else "?", pc - (m.base if m else 0))
def insn(pc):
try:
w = img.word(pc)
except Exception:
return "?"
i = next(md.disasm(w.to_bytes(4, "little"), pc), None)
return "%s %s" % (i.mnemonic, i.op_str) if i else ".word %#x" % w
for block in text.split("[recomp] host access violation")[1:]:
block = "[recomp] host access violation" + block
print(block.splitlines()[0])
for line in block.splitlines()[1:]:
if "backtrace" in line or line.startswith("[recomp] x"):
continue
print(line)
m = re.search(r"rip in hookshot_game\.dll\+(0x[0-9a-f]+)", block)
if m:
t = dllmap.BASE + int(m.group(1), 16)
i = bisect.bisect_right(keys, t) - 1
print(" crashed in %s+%#x" % (rows[i][1], t - rows[i][0]))
m = re.search(r"guest backtrace: (.*)", block)
if m:
print(" guest backtrace (return address: the call before it, in its function):")
for a in m.group(1).replace("lr ", "").split():
lr = int(a, 16)
print(" %#x %-40s %s" % (lr, func_of(lr), insn(lr - 4)))
if __name__ == "__main__":
main()

23
nx2/host_cpu.py Normal file
View File

@@ -0,0 +1,23 @@
"""The x86-64 level this PC's CPU runs (nx2/build.py's -march; issue 363). The game is built on the PC it runs on, so a
CPU without x86-64-v3's features (AVX2, FMA, BMI2, MOVBE) gets x86-64-v2, or the x86-64 baseline. RX_MARCH, read by
build.py, still chooses."""
import os
def host_march():
want = ("avx2", "fma", "bmi2", "movbe")
try:
if os.name == "nt":
import ctypes
pf = ctypes.windll.kernel32.IsProcessorFeaturePresent
# (PF_AVX2_INSTRUCTIONS_AVAILABLE 40; PF_SSE4_2_INSTRUCTIONS_AVAILABLE 38. Windows reports no FMA/BMI2 bit, but
# every AVX2 CPU has them, as x86-64-v3 counts them together)
if pf(40):
return "x86-64-v3"
return "x86-64-v2" if pf(38) else "x86-64"
flags = open("/proc/cpuinfo").read().split("flags", 1)[1].split(chr(10), 1)[0].split()
if all(f in flags for f in want):
return "x86-64-v3"
return "x86-64-v2" if all(f in flags for f in ("sse4_2", "popcnt", "ssse3")) else "x86-64"
except Exception:
return "x86-64-v3"

66
nx2/image.py Normal file
View File

@@ -0,0 +1,66 @@
"""The guest image at the addresses Eglantine (Eden) loads it to, so lifted code can run inside the emulator.
a64-kit's image builder (hos/nx_image.py) places main, subsdk*, sdk from 0x08000000, 1 MB aligned, and leaves rtld
out: its own loader is rtld. Eden instead loads rtld, main, subsdk0..9, sdk back to back (loader/nso.cpp,
LoadModule): each module takes PageAlign(.data end + header bss size), plus 0x9000 for rtld's argument block when
program arguments are set, from the process code address. For a 39-bit process that is 0x80000000 + the ASLR
offset, (rng_seed << 12) & 0xfff000 (deconstructed_rom_directory.cpp), so recomp runs set rng_seed and lift with
the same offset.
rtld is part of this image: in the emulator it runs first and binds the GOTs itself, to the same values this
builder binds them to (same module order, same lookup).
"""
import kit # noqa: F401 (sys.path)
import nso2 # noqa: F401 (DT_RELR)
import nso
import nx_image
PAGE = 0x1000
CODE_39BIT = 0x80000000
ARGS_SIZE = 0x9000
def page_align(v):
return (v + PAGE - 1) & ~(PAGE - 1)
def eden_size(m):
"""LoadModule's image size: the .data segment's end plus the header's bss size, page aligned"""
data_off, data_size, _ = m.segments[2]
return page_align(data_off + data_size + m.bss_size)
class EdenImage(nx_image.NxImage):
def __init__(self, exefs, aslr_offset=0, program_args=False, hle=()):
base = CODE_39BIT + aslr_offset
nx_image.MAIN_BASE = base # (NxImage's word/read/write index the image from this module global)
self.modules = nso.load_exefs(exefs)
self.skipped = []
for m in self.modules:
m.base = base
size = eden_size(m) + (ARGS_SIZE if m.name == "rtld" and program_args else 0)
if len(m.image) < size:
m.image.extend(bytes(size - len(m.image)))
base += size
self.end = base
self.mem = bytearray(self.end - nx_image.MAIN_BASE)
for m in self.modules:
self.mem[m.base - nx_image.MAIN_BASE:m.base - nx_image.MAIN_BASE + len(m.image)] = m.image
self.hle = set(hle)
self.exports = {}
for m in self.modules:
for s in m.symbols:
if s["defined"] and s["name"] and s["name"] not in self.exports:
self.exports[s["name"]] = (m, s)
self.hooks, self.hook_syms, self.tramp, self.unresolved = {}, {}, {}, {}
for sym in sorted(self.hle):
hit = self.exports.get(sym)
if hit:
m, s = hit
name = nx_image.c_ident(sym)
self.hooks[m.base + s["value"]] = name
self.hook_syms[name] = sym
self._relocate()
self.code = [(m.base + lo, m.base + hi, m.name) for m in self.modules for lo, hi in [m.code_range()]]
self._entries()
self.names = self.symbol_names()

66
nx2/imports.py Normal file
View File

@@ -0,0 +1,66 @@
"""The SDK surface the game's modules use: every symbol main and the subsdk* modules import that the sdk defines,
grouped by namespace, and (with --hle <file of mangled names>) which of them an existing HLE layer already covers
(a64-kit's hos/, TOTKPort).
Usage: python -I nx2/imports.py [--hle names.txt] [--list]
"""
import argparse
import collections
import os
import re
import sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import kit
import image as eden_image
def namespace(name):
"""nn::os, nn::fs, ... from an Itanium mangled name (the first two nested names), or the C name"""
m = re.match(r"_Z(?:N|L)?(?:\d+)?", name)
parts, s = [], name[2:] if name.startswith("_Z") else None
if s is None:
return "(C) " + name.split("_")[0][:12]
if s.startswith("N"):
s = s[1:]
while s and len(parts) < 2:
m = re.match(r"(\d+)", s)
if not m:
break
n = int(m.group(1))
s = s[len(m.group(1)):]
parts.append(s[:n])
s = s[n:]
return "::".join(parts) or name[:20]
def main():
p = argparse.ArgumentParser()
p.add_argument("--hle")
p.add_argument("--list", action="store_true")
a = p.parse_args()
img = eden_image.EdenImage(kit.need_exefs())
sdk = next(m for m in img.modules if m.name == "sdk")
sdk_defs = {s["name"] for s in sdk.symbols if s["defined"] and s["name"]}
used = set()
for m in img.modules:
if m.name in ("sdk", "rtld"):
continue
for s in m.symbols:
if not s["defined"] and s["name"] in sdk_defs:
used.add(s["name"])
hle = set(open(a.hle).read().split()) if a.hle else set()
by_ns = collections.defaultdict(lambda: [0, 0])
for n in used:
by_ns[namespace(n)][0] += 1
by_ns[namespace(n)][1] += n in hle
print("sdk symbols the game modules import: %d; covered by the HLE list: %d" % (len(used), len(used & hle)))
for ns, (n, h) in sorted(by_ns.items(), key=lambda kv: -kv[1][0]):
print(" %-28s %5d covered %4d" % (ns, n, h))
if a.list:
for n in sorted(used):
print((" * " if n in hle else " ") + n)
if __name__ == "__main__":
main()

291
nx2/insn.py Normal file
View File

@@ -0,0 +1,291 @@
"""Switch 2 instruction forms for a64-kit: the ARMv8.1-8.5 encodings suyu's translator (lift/suyu/arm64_to_c.h)
does not know, which Switch 2 code uses everywhere (MKW: ~121k RETAB, ~39k LDAPR, ~32k LSE, ~7k CAS).
install() wraps the two entry points a64_lift.py and tests/insn_test.py use, without editing the vendored kit:
- a64_lift.classify: RETAA/RETAB are returns; BRAA*/BRAB* and BLRAA*/BLRAB* are BR/BLR through Xn (the
modifier is ignored);
- a64emit.translate: the forms below get C from emit() here; everything else goes to suyu as before.
The C is written in the same dialect as suyu's (c->x[n], A64_PTR, A64_MARK), so the lifter's later passes
(dead flags, register localisation, dirty-register sets) see ordinary register reads and writes.
Model:
- LSE atomics, SWP, CAS: one host atomic read-modify-write on the guest address (__atomic builtins, seq_cst; on
x86-64 every locked RMW is a full barrier anyway, so the A/L ordering bits cost nothing and are all honoured).
MIN/MAX are a compare-exchange loop. Rt = XZR (the ST* aliases) discards the old value.
- CASP: 64-bit or 128-bit compare-exchange (the 128-bit one needs cmpxchg16b: build with -mcx16 / x86-64-v2+).
- LDAPR*: acquire load (a plain load on x86-64, ordered for the compiler).
- Pointer authentication is the identity, as in Eglantine's dynarmic: PAC*/AUT* (register forms) and XPAC* leave
Xd unchanged, PACGA writes 0. Nothing in the game's own code can tell, since no pointer is ever signed.
- MTE: tags are always 0. STG/ST2G only do their base-register writeback.
"""
import a64emit
import a64_lift
LSE_OPS = ["add", "clr", "eor", "set", "smax", "smin", "umax", "umin"]
UTYPE = {0: "uint8_t", 1: "uint16_t", 2: "uint32_t", 3: "uint64_t"}
STYPE = {0: "int8_t", 1: "int16_t", 2: "int32_t", 3: "int64_t"}
MASK = {0: "0xFFULL", 1: "0xFFFFULL", 2: "0xFFFFFFFFULL", 3: "0xFFFFFFFFFFFFFFFFULL"}
def xr(n):
"""C for reading Xn where 31 is XZR"""
return "0ULL" if n == 31 else "c->x[%d]" % n
def xw(n, value):
"""C for writing Xn where 31 is XZR (the value is dropped)"""
return "" if n == 31 else "c->x[%d] = %s;" % (n, value)
def sx(v, bits):
return v - (1 << bits) if v & (1 << (bits - 1)) else v
# ---- control flow ----------------------------------------------------------------------------------------
def classify_ext(w, pc):
if w in (0xD65F0BFF, 0xD65F0FFF): # RETAA / RETAB
return ("ret", 30)
if (w & 0xFEFFF800) == 0xD61F0800: # BRAA, BRAB, BRAAZ, BRABZ
return ("br", (w >> 5) & 31)
if (w & 0xFEFFF800) == 0xD63F0800: # BLRAA, BLRAB, BLRAAZ, BLRABZ
return ("blr", (w >> 5) & 31)
return None
# ---- data ----------------------------------------------------------------------------------------------
def lse(w):
size, a, r = w >> 30, (w >> 23) & 1, (w >> 22) & 1
rs, o3, opc, rn, rt = (w >> 16) & 31, (w >> 15) & 1, (w >> 12) & 7, (w >> 5) & 31, w & 31
T, S = UTYPE[size], STYPE[size]
n = 1 << size
head = "{ uint64_t _a = c->x[%d]; %s *_p = (%s *)A64_PTR(_a); " % (rn, T, T)
if o3 == 1 and opc == 4 and a == 1 and r == 0 and rs == 31: # LDAPR(B/H)
return head + "%s }" % xw(rt, "(uint64_t)__atomic_load_n(_p, __ATOMIC_ACQUIRE)")
if o3 == 1 and opc == 0: # SWP
return head + "%s _o = __atomic_exchange_n(_p, (%s)%s, __ATOMIC_SEQ_CST); A64_MARK(_a, %d); %s }" % (
T, T, xr(rs), n, xw(rt, "(uint64_t)_o"))
if o3 != 0:
return None
op = LSE_OPS[opc]
s = "(%s)%s" % (T, xr(rs))
if op in ("add", "clr", "eor", "set"):
fn, arg = {"add": ("__atomic_fetch_add", s), "clr": ("__atomic_fetch_and", "(%s)~%s" % (T, s)),
"eor": ("__atomic_fetch_xor", s), "set": ("__atomic_fetch_or", s)}[op]
body = "%s _o = %s(_p, %s, __ATOMIC_SEQ_CST);" % (T, fn, arg)
else:
cmp = {"smax": "(%s)_o > (%s)_s" % (S, S), "smin": "(%s)_o < (%s)_s" % (S, S),
"umax": "_o > _s", "umin": "_o < _s"}[op]
body = ("%s _s = %s, _o = __atomic_load_n(_p, __ATOMIC_RELAXED), _n; "
"do { _n = (%s) ? _o : _s; } while (!__atomic_compare_exchange_n(_p, &_o, _n, 1, __ATOMIC_SEQ_CST, "
"__ATOMIC_RELAXED));" % (T, s, cmp))
return head + body + " A64_MARK(_a, %d); %s }" % (n, xw(rt, "(uint64_t)_o"))
def cas(w):
size, rs, rn, rt = w >> 30, (w >> 16) & 31, (w >> 5) & 31, w & 31
T = UTYPE[size]
return ("{ uint64_t _a = c->x[%d]; %s _e = (%s)%s; __atomic_compare_exchange_n((%s *)A64_PTR(_a), &_e, (%s)%s, 0, "
"__ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); A64_MARK(_a, %d); %s }"
% (rn, T, T, xr(rs), T, T, xr(rt), 1 << size, xw(rs, "(uint64_t)_e")))
def casp(w):
sz, rs, rn, rt = (w >> 30) & 1, (w >> 16) & 31, (w >> 5) & 31, w & 31
if rs & 1 or rt & 1:
return None
if not sz: # two 32-bit registers as one 64-bit word, the first register at the lower address
return ("{ uint64_t _a = c->x[%d]; uint64_t _e = (uint32_t)%s | (uint64_t)(uint32_t)%s << 32; "
"uint64_t _n = (uint32_t)%s | (uint64_t)(uint32_t)%s << 32; "
"__atomic_compare_exchange_n((uint64_t *)A64_PTR(_a), &_e, _n, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); "
"A64_MARK(_a, 8); %s %s }" % (rn, xr(rs), xr(rs + 1), xr(rt), xr(rt + 1),
xw(rs, "(uint32_t)_e"), xw(rs + 1, "_e >> 32")))
return ("{ uint64_t _a = c->x[%d]; unsigned __int128 _e = (unsigned __int128)%s | (unsigned __int128)%s << 64; "
"unsigned __int128 _n = (unsigned __int128)%s | (unsigned __int128)%s << 64; "
"__atomic_compare_exchange_n((unsigned __int128 *)A64_PTR(_a), &_e, _n, 0, __ATOMIC_SEQ_CST, "
"__ATOMIC_SEQ_CST); A64_MARK(_a, 16); %s %s }" % (rn, xr(rs), xr(rs + 1), xr(rt), xr(rt + 1),
xw(rs, "(uint64_t)_e"), xw(rs + 1, "(uint64_t)(_e >> 64)")))
def stg(w):
"""STG / ST2G with writeback (tags are not modelled: the only effect is the base register update)"""
opc, imm, op2, rn = (w >> 22) & 3, sx((w >> 12) & 0x1FF, 9) << 4, (w >> 10) & 3, (w >> 5) & 31
if opc not in (0, 2) or op2 == 0:
return None
if op2 == 2: # signed offset, no writeback
return "/* stg: memory tags are not modelled */"
return "c->x[%d] += (uint64_t)(int64_t)%d; /* st%sg, writeback: tags are not modelled */" % (
rn, imm, "2" if opc == 2 else "")
# ---- the few SIMD forms suyu lacks that MKW's sdk uses (62 sites) -------------------------------------
# Saturations set FPSR.QC (bit 27), as the hardware does.
QC = "c->fpsr |= 1u << 27;"
def sqdmlal(w):
"""SQDMLAL{2} (vector and by element): Vd.(2e) = sat(Vd + sat(2 * Vn.e * Vm.e))"""
q, size, rn, rd = (w >> 30) & 1, (w >> 22) & 3, (w >> 5) & 31, w & 31
if size not in (1, 2):
return None
if (w & 0xBF20FC00) == 0x0E209000: # vector
rm, idx = (w >> 16) & 31, None
else: # by element
L, M, H = (w >> 21) & 1, (w >> 20) & 1, (w >> 11) & 1
if size == 1:
rm, idx = (w >> 16) & 15, H << 2 | L << 1 | M
else:
rm, idx = (w >> 16) & 31, H << 1 | L
e, n = (16, 4) if size == 1 else (32, 2)
T, W = ("int16_t", "int32_t") if size == 1 else ("int32_t", "int64_t")
lo = n * q # the "2" form reads the upper half of Vn (and Vm, vector form)
b = "_m[%d]" % idx if idx is not None else "_m[%d + _i]" % lo
return ("{ %s _n[%d], _m[%d]; %s _d[%d]; memcpy(_n, c->vreg[%d], 16); memcpy(_m, c->vreg[%d], 16); "
"memcpy(_d, c->vreg[%d], 16); for (int _i = 0; _i < %d; _i++) { %s _p; "
"if (_n[%d + _i] == (%s)(-%dLL - 1) && %s == (%s)(-%dLL - 1)) { _p = (%s)((1ULL << %d) - 1); %s } "
"else _p = (%s)2 * _n[%d + _i] * %s; "
"if (__builtin_add_overflow(_d[_i], _p, &_d[_i])) { _d[_i] = _p < 0 ? (%s)(-%dLL - 1) - 0 : (%s)((1ULL << %d) - 1); %s } } "
"memcpy(c->vreg[%d], _d, 16); }"
% (T, 2 * n, 2 * n, W, n, rn, rm, rd, n, W, lo, T, (1 << (e - 1)) - 1, b, T, (1 << (e - 1)) - 1,
W, 2 * e - 1, QC, W, lo, b, W, (1 << (2 * e - 1)) - 1, W, 2 * e - 1, QC, rd))
def sqshl_imm(w):
"""SQSHL / SQSHLU (vector, immediate)"""
q, u, immh, immb, rn, rd = (w >> 30) & 1, (w >> 29) & 1, (w >> 19) & 15, (w >> 16) & 7, (w >> 5) & 31, w & 31
if not immh or (immh & 8 and not q):
return None
e = 8 << (immh.bit_length() - 1)
shift = (immh << 3 | immb) - e
lanes = (16 if q else 8) * 8 // e
S = {8: "int8_t", 16: "int16_t", 32: "int32_t", 64: "int64_t"}[e]
U = {8: "uint8_t", 16: "uint16_t", 32: "uint32_t", 64: "uint64_t"}[e]
smax, smin, umax = (1 << (e - 1)) - 1, -(1 << (e - 1)), (1 << e) - 1
if u:
body = ("if (_x < 0) { _r[_i] = 0; if (_x) %s } else { __int128 _v = (__int128)_x * ((__int128)1 << %d); "
"if (_v > (__int128)%dULL) { _r[_i] = (%s)%dULL; %s } else _r[_i] = (%s)_v; }"
% (QC, shift, umax, U, umax, QC, U))
out = U
else:
body = ("__int128 _v = (__int128)_x * ((__int128)1 << %d); if (_v > (__int128)%dLL) { _r[_i] = (%s)%dLL; %s } "
"else if (_v < (__int128)(%dLL - 1)) { _r[_i] = (%s)(%dLL - 1); %s } else _r[_i] = (%s)_v;"
% (shift, smax, S, smax, QC, smin + 1, S, smin + 1, QC, S))
out = S
return ("{ %s _a[%d]; %s _r[%d]; memcpy(_a, c->vreg[%d], 16); for (int _i = 0; _i < %d; _i++) { %s _x = _a[_i]; %s } "
"c->vreg[%d][0] = 0; c->vreg[%d][1] = 0; memcpy(c->vreg[%d], _r, %d); }"
% (S, 128 // e, out, 128 // e, rn, lanes, S, body, rd, rd, rd, lanes * e // 8))
def fneg_h(w):
"""FNEG (vector, half precision): flip each lane's sign bit (no exceptions, NaNs keep their payload)"""
q, rn, rd = (w >> 30) & 1, (w >> 5) & 31, w & 31
m = "0x8000800080008000ULL"
return ("{ uint64_t _lo = c->vreg[%d][0] ^ %s, _hi = %s; c->vreg[%d][0] = _lo; c->vreg[%d][1] = _hi; }"
% (rn, m, ("c->vreg[%d][1] ^ %s" % (rn, m)) if q else "0", rd, rd))
def dot(w):
"""SDOT / UDOT (vector, and by element; FEAT_DotProd): each 32-bit lane of Vd += the sum of the four products of
the matching bytes of Vn and Vm (Vm's lane `index` for the element form), wrapping. OoT: 4 sites in main."""
q, u, rm, rn, rd = (w >> 30) & 1, (w >> 29) & 1, (w >> 16) & 31, (w >> 5) & 31, w & 31
elem = (w >> 24) & 1
T = "uint8_t" if u else "int8_t"
if elem:
index = ((w >> 11) & 1) << 1 | ((w >> 21) & 1) # (H:L; Rm is M:Rm, all 5 bits, for size 10)
mb = "_m[4 * %d + _j]" % index
else:
mb = "_m[4 * _i + _j]"
lanes = 4 if q else 2
return ("{ %s _n[16], _m[16]; uint32_t _d[4]; memcpy(_n, c->vreg[%d], 16); memcpy(_m, c->vreg[%d], 16); "
"memcpy(_d, c->vreg[%d], 16); for (int _i = 0; _i < %d; _i++) { int32_t _s = 0; "
"for (int _j = 0; _j < 4; _j++) _s += (int32_t)_n[4 * _i + _j] * (int32_t)%s; _d[_i] += (uint32_t)_s; } "
"c->vreg[%d][1] = 0; memcpy(c->vreg[%d], _d, %d); }"
% (T, rn, rm, rd, lanes, mb, rd, rd, lanes * 4))
def ldar_stlr(w):
"""LDAR / STLR (and the LO forms LDLAR / STLLR) as host atomics.
suyu translates them as plain loads and stores: right for x86's own ordering, but the C compiler may then move
other guest accesses across them, or hoist an LDAR out of a spin loop. And an STLR followed by an LDAR must not be
reordered (AArch64's release/acquire are RCsc), which x86 does to a store and a later load unless the store is
locked. So: LDAR = seq_cst load (a plain mov), STLR = seq_cst store (xchg). MKW: 1,900 LDAR, 1,321 STLR."""
size, load, rn, rt = w >> 30, (w >> 22) & 1, (w >> 5) & 31, w & 31
T = UTYPE[size]
if load:
v = "(uint64_t)__atomic_load_n((%s *)A64_PTR(_a), __ATOMIC_SEQ_CST)" % T
return "{ uint64_t _a = c->x[%d]; %s }" % (rn, xw(rt, v) if rt != 31 else "(void)" + v + ";")
return ("{ uint64_t _a = c->x[%d]; __atomic_store_n((%s *)A64_PTR(_a), (%s)%s, __ATOMIC_SEQ_CST); A64_MARK(_a, %d); }"
% (rn, T, T, xr(rt), 1 << size))
def emit(w):
"""C for one of the forms above, or None (then suyu translates it)"""
if (w & 0x3FBF7C00) == 0x089F7C00: # LDAR, STLR, LDLAR, STLLR (o0 = bit 15)
return ldar_stlr(w)
if (w & 0xBF20FC00) == 0x0E209000 or (w & 0xBF00F400) == 0x0F003000:
return sqdmlal(w)
if (w & 0xBF80FC00) in (0x0F007400, 0x2F006400):
return sqshl_imm(w)
if (w & 0xBFFFFC00) == 0x2EF8F800:
return fneg_h(w)
if (w & 0x9FE0FC00) == 0x0E809400 or (w & 0x9FC0F400) == 0x0F80E000: # SDOT / UDOT (vector, by element)
return dot(w)
if (w & 0x3F200C00) == 0x38200000:
return lse(w)
if (w & 0x3FA07C00) == 0x08A07C00:
return cas(w)
if (w & 0xBFA07C00) == 0x08207C00:
return casp(w)
if (w & 0xFFE0FC00) == 0x9AC03000: # PACGA Xd, Xn, Xm
return xw(w & 31, "0") or ";"
if (w & 0xFFFFFBE0) == 0xDAC143E0: # XPACI / XPACD
return "/* xpac: pointers carry no PAC */"
if (w & 0xFFFFC000) == 0xDAC10000: # PACIA/IB/DA/DB, AUTIA/IB/DA/DB (+ Z forms)
return "/* pac/aut: pointer authentication is the identity */"
if (w & 0xFF200000) == 0xD9200000:
return stg(w)
return None
def is_identity_model(w):
"""Forms whose result here is by design not the hardware's (PAC, MTE): instruction tests skip them."""
return ((w & 0xFFE0FC00) == 0x9AC03000 or (w & 0xFFFFFBE0) == 0xDAC143E0 or (w & 0xFFFFC000) == 0xDAC10000
or (w & 0xFF200000) == 0xD9200000)
# ---- install -------------------------------------------------------------------------------------------
_installed = False
def install():
global _installed
if _installed:
return
_installed = True
base_classify = a64_lift.classify
base_translate = a64emit.translate
def classify(w, pc):
return classify_ext(w, pc) or base_classify(w, pc)
def translate(items, batch=65536):
mine = {}
rest = []
for k, (pc, w) in enumerate(items):
t = emit(w)
if t is None:
rest.append(k)
else:
mine[k] = t
texts, flags = [None] * len(items), [0] * len(items)
if rest:
tx, fl = base_translate([items[k] for k in rest], batch)
for k, t, f in zip(rest, tx, fl):
texts[k], flags[k] = t, f
for k, t in mine.items():
texts[k] = t + "\n"
return texts, flags
a64_lift.classify = classify
a64emit.translate = translate

68
nx2/kit.py Normal file
View File

@@ -0,0 +1,68 @@
"""Paths: the vendored a64-kit (third_party/a64-kit) on sys.path, the dump, the work folder.
The dump (the user's own decrypted copy: a folder with exefs/ and romfs/) is never copied into the repository. Its
location comes from, in order: OOT_EXEFS (the exefs folder itself), OOT_DUMP (the folder holding exefs/ and romfs/),
or the first line of work/dump.txt (the same, written once by hand or by nx2/dump.py).
"""
import os
import sys
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
KIT = os.path.join(ROOT, "third_party", "a64-kit")
for sub in ("lift", "tools", "hos", "rt"):
p = os.path.join(KIT, sub)
if p not in sys.path:
sys.path.insert(0, p)
HERE = os.path.dirname(os.path.abspath(__file__))
if HERE not in sys.path:
sys.path.insert(0, HERE)
WORK = os.environ.get("OOT_WORK", os.path.join(ROOT, "work"))
def _dump_root():
if os.environ.get("OOT_DUMP"):
return os.environ["OOT_DUMP"]
cfg = os.path.join(WORK, "dump.txt")
if os.path.exists(cfg):
for line in open(cfg, encoding="utf-8"):
line = line.strip()
if line and not line.startswith("#"):
return line
return None
DUMP = _dump_root()
EXEFS = os.environ.get("OOT_EXEFS") or (os.path.join(DUMP, "exefs") if DUMP else None)
ROMFS = os.environ.get("OOT_ROMFS") or (os.path.join(DUMP, "romfs") if DUMP else None)
def need_exefs():
if not EXEFS or not os.path.isdir(EXEFS):
raise SystemExit("no exefs: set OOT_DUMP (the folder with exefs/ and romfs/) or OOT_EXEFS, or write the dump "
"folder into %s" % os.path.join(WORK, "dump.txt"))
return EXEFS
def game_modules(exefs=None):
"""The exefs's code modules in rtld's load order (rtld, main, subsdk0..9, sdk) that exist, without rtld"""
exefs = exefs or need_exefs()
order = ["main"] + ["subsdk%d" % i for i in range(10)] + ["sdk"]
return [m for m in order if os.path.exists(os.path.join(exefs, m))]
# Other sessions' benchmarks need a quiet machine: they hold this file for the few minutes a timing run takes.
# Heavy jobs (builds, test runs, the full emit) wait while it exists; a job already running may finish.
BENCH_LOCK = os.path.join(os.path.dirname(ROOT), ".bench-lock")
def wait_for_quiet(what="this job"):
import time
waited = 0
while os.path.exists(BENCH_LOCK):
if not waited:
print("waiting for %s to go (a benchmark is running) before %s" % (BENCH_LOCK, what), flush=True)
time.sleep(60)
waited += 1
if waited:
print("benchmark lock gone after ~%d min" % waited, flush=True)

204
nx2/lift.py Normal file
View File

@@ -0,0 +1,204 @@
"""The lift driver: image (we are rtld) -> function discovery -> C, on the vendored a64-kit plus the Switch 2
additions (nso2: DT_RELR, insn: ARMv8.1-8.5 forms, lift_fixes: jump tables).
Usage: python -I nx2/lift.py [--exefs <dir>] [--work <dir>] [--write] [--max-func N] [--layout eden|a64] [--aslr 0]
--layout eden (default): Eglantine's addresses, rtld included, from 0x80000000 + --aslr (image.py; an emulator
run must use the same offset: rng_seed_enabled=true, rng_seed=<aslr >> 12>);
--layout a64: a64-kit's own layout from 0x08000000 (its native runtime, below 4 GB)
Always: builds <work>/image (image.bin, image.json), discovers every function of rtld-less main + subsdk* + sdk,
and prints coverage: functions, instructions, jump tables, indirect branches, and the functions whose every
instruction translates (coverage.txt has the untranslated words by site count).
--write: also writes the lifted C to <work>/gen (about a gigabyte; game code: stays local, never committed).
--standalone: the standalone build's lift (<work>/sa/image, <work>/sa/gen): the SDK's OS-facing functions hooked
to the native runtime in host/ or to generated stubs, plus hos_image.c (nx2/standalone.py). It shares the
chunk boundaries of <work>/image, so its unhooked chunks are the emulator build's, byte for byte.
"""
import argparse
import collections
import glob
import json
import os
import struct
import sys
import time
import capstone
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # (python -I leaves the script folder off)
import kit
import nso2 # noqa: F401 (DT_RELR)
import insn
import lift_fixes
import chunks
import standalone
import adr_entries # noqa: E402
import nso
import poll
import nx_image
import image as eden_image
import a64emit
import a64_lift
import a64_rtgen
insn.install()
lift_fixes.install()
def module_of(img, pc):
for lo, hi, name in img.code:
if lo <= pc < hi:
return name
return "?"
def coverage(L, img, out_dir):
"""Per-module: functions, instructions, untranslated words; and the share of functions that translate fully."""
keys = {}
for body, _ in L.funcs.values():
for pc, (k, w) in body.items():
if k[0] == "op":
keys.setdefault(w, pc)
words = list(keys)
_, flags = a64emit.translate([(keys[w], w) for w in words])
bad = {w for w, f in zip(words, flags) if f & a64emit.UNHANDLED}
# UDF and unallocated encodings trap on the hardware too (code falls into them after noreturn calls): the
# lifter's trap is their exact translation, so they do not count against coverage
md = capstone.Cs(capstone.CS_ARCH_ARM64, capstone.CS_MODE_ARM)
undefined = set()
for w in bad:
ins = next(md.disasm(struct.pack("<I", w), 0), None)
if ins is None or ins.mnemonic == "udf":
undefined.add(w)
bad -= undefined
per = collections.defaultdict(collections.Counter)
sites = collections.Counter()
for e, (body, _) in L.funcs.items():
m = module_of(img, e)
n_bad = 0
for pc, (k, w) in body.items():
if k[0] == "op" and w in bad:
n_bad += 1
sites[w] += 1
elif k[0] == "op" and w in undefined:
per[m]["undefined"] += 1
kinds = collections.Counter(k[0] for k, _ in body.values())
c = per[m]
c["functions"] += 1
c["instructions"] += len(body)
c["untranslated"] += n_bad
c["complete"] += n_bad == 0
c["leaf"] += not (kinds["bl"] or kinds["blr"] or kinds["svc"])
c["indirect_br"] += sum(1 for pc, (k, _) in body.items() if k[0] == "br" and pc not in L.jt)
c["blr"] += kinds["blr"]
total = collections.Counter()
lines = ["module functions complete leaf instructions untranslated BLR BR(not table)"]
for m in sorted(per, key=lambda m: -per[m]["instructions"]):
c = per[m]
total.update(c)
lines.append("%-8s %9d %6.2f%% %8d %12d %12d %8d %8d" % (
m, c["functions"], 100.0 * c["complete"] / c["functions"], c["leaf"], c["instructions"], c["untranslated"],
c["blr"], c["indirect_br"]))
lines.append("%-8s %9d %6.2f%% %8d %12d %12d %8d %8d" % (
"all", total["functions"], 100.0 * total["complete"] / total["functions"], total["leaf"],
total["instructions"], total["untranslated"], total["blr"], total["indirect_br"]))
lines.append("undefined encodings reached (UDF, unallocated: trap, as on the hardware): %d sites" % total["undefined"])
lines.append("translated instructions: %.4f%% of %d (shared tails counted once per function)" % (
100.0 - 100.0 * total["untranslated"] / max(1, total["instructions"]), total["instructions"]))
print("\n".join(lines))
with open(os.path.join(out_dir, "coverage.txt"), "w") as f:
f.write("\n".join(lines) + "\n\nuntranslated words by site count\n")
f.write("".join("%08x %d\n" % (w, n) for w, n in sites.most_common()))
return total
def main():
kit.wait_for_quiet('the lift')
p = argparse.ArgumentParser()
p.add_argument("--exefs", default=kit.EXEFS)
p.add_argument("--work", default=kit.WORK)
p.add_argument("--write", action="store_true")
p.add_argument("--max-func", type=int, default=0)
p.add_argument("--layout", choices=("eden", "a64"), default="eden")
p.add_argument("--aslr", type=lambda v: int(v, 0), default=0)
p.add_argument("--no-polls", action="store_true", help="no loop-header interrupt polls (nx2/poll.py)")
p.add_argument("--rechunk", action="store_true", help="chunk boundaries from the sizes again (nx2/chunks.py)")
p.add_argument("--standalone", action="store_true", help="the standalone build's lift (nx2/standalone.py)")
p.add_argument("--entries", help="extra entry addresses, one hex address a line (e.g. the misses ArmRecomp logged)")
p.add_argument("--no-adr-entries", action="store_true",
help="no extra entries at the code addresses ADR / ADRP+ADD take (nx2/adr_entries.py)")
a = p.parse_args()
a.exefs = a.exefs or kit.need_exefs()
t0 = time.time()
hle = ()
if a.standalone:
hle, stubbed = standalone.hle_set(nso.load_exefs(a.exefs))
print("standalone: %d HLE functions in host/, %d OS-facing imports stubbed" % (len(hle), len(stubbed)))
hle = hle | stubbed
img = eden_image.EdenImage(a.exefs, a.aslr, hle=hle) if a.layout == "eden" else nx_image.NxImage(a.exefs)
out_dir = os.path.join(a.work, "sa") if a.standalone else a.work
img_dir = os.path.join(out_dir, "image")
meta = img.write(img_dir)
print("image: %s; %d entries, %d trampolines (%d unresolved non-weak imports) (%.0fs)" % (
", ".join("%s@%#x+%#x" % (m["name"], m["base"], m["size"]) for m in meta["modules"]),
len(img.entries), len(img.tramp), len(img.unresolved), time.time() - t0))
if img.unresolved:
print(" unresolved: " + ", ".join(sorted(img.unresolved)[:16]))
if a.entries:
extra = {int(l.split()[0], 16) for l in open(a.entries) if l.strip() and not l.startswith("#")}
new = {e for e in extra if img.in_code(e)} - img.entries
img.entries |= new
print("extra entries: %d of %d listed are new code addresses" % (len(new), len(extra)))
if not a.no_adr_entries: # (hookshot) callbacks reached only through a pointer the code builds
spans = adr_entries.fde_spans(img.modules)
refs = adr_entries.code_address_refs(open(os.path.join(img_dir, "image.bin"), "rb").read(), meta, spans=spans)
new = {e for e in refs if img.in_code(e)} - img.entries
img.entries |= new
print("ADR / ADRP+ADD code addresses outside the %d unwind-covered functions: %d, %d of them new entries" % (
len(spans), len(refs), len(new)))
L = a64_lift.Lifter(img)
if a.standalone: # (hookshot) widescreen and ultrawide: C after a few of the game's instructions
import oot_wide
oot_wide.install(L, img)
import oot_jitter # (the temporal upscalers' camera jitter, DLSS's and FSR 3.1's: none unless one runs, host/rx_backend.c)
oot_jitter.install(L, img)
import oot_occlusion # (no occlusion queries when HOOKSHOT_OCCLUSION_OFF=1: Fritchii's Occlusion Off)
oot_occlusion.install(L, img)
L.discover(max_insns=a.max_func)
print("discovered %d functions (%.0fs)" % (len(L.funcs), time.time() - t0))
total = coverage(L, img, img_dir)
jt = L.stats
print("jump tables %d (bounded %d, bounded by function %d, ADR table %d), BR without a table %d, PLT stubs %d" % (
len(L.jt), jt["jt_bounded"], jt["jt_unbounded"], jt["jt_adr_table"], len(L.jt_missing), len(L.plt)))
if jt["jt_table_reg_reused"]:
print("jump tables whose table register is reused before the br: %d" % jt["jt_table_reg_reused"])
for pc, reg in sorted(L.jt_missing.items()):
print(" BR without a table: %#x (x%d)" % (pc, reg))
with open(os.path.join(img_dir, "functions.json"), "w") as f:
json.dump({"%x" % e: [len(b), module_of(img, e)] for e, (b, _) in L.funcs.items()}, f)
if a.write:
if not a.no_polls:
poll.install()
gen = os.path.join(out_dir, "gen")
files = chunks.write(L, gen, os.path.join(a.work, "image"), a.rechunk and not a.standalone,
counters=[] if a.no_polls else [poll.stats])
import oot_patches
main_build = next((m["build_id"] for m in meta["modules"] if m["name"] == "main"), "")
chunks.edit_c(gen, L.stats, a.standalone and main_build == oot_patches.LIFE_BUILD_ID, whole=not a.max_func)
a64_rtgen.write(gen)
if a.standalone:
start, exit_ = standalone.write_image_c(gen, img, meta, a.exefs)
print("standalone: hos_image.c (thread hooks %#x, %#x), %d import stubs" % (
start, exit_, standalone.gen_stubs(gen, meta)))
nvn = standalone.nvn_exports(nso.load_exefs(a.exefs))
standalone.write_nvn_exports(gen, nvn)
print("standalone: nvn2_exports.c (%d sdk NVN functions routed to host/nvn2.c)" % len(nvn))
L.report()
print("interrupt polls: %d loop headers in %d functions" % (poll.stats["polls"], poll.stats["functions"]))
print("wrote %d code files to %s (%.0fs)" % (len(files), gen, time.time() - t0))
print("done (%.0fs)" % (time.time() - t0))
if __name__ == "__main__":
main()

114
nx2/lift_fixes.py Normal file
View File

@@ -0,0 +1,114 @@
"""Fixes to a64-kit's control-flow analysis (lift/a64_lift.py) found on MKW, applied by wrapping its methods.
1. Jump tables whose table address is set by a plain ADR outside the scan window. a64-kit follows a hoisted table
register back to ADRP (+ADD) only. rtld's dynamic-section loop (rtld+0x8a0) sets it with `adr x25, table` before
the loop and dispatches with `adr x9, base; ldrb w10, [x25, x8]; add x9, x9, x10, lsl #2; br x9`. The window
scan forgets x25 at the loop's `b`, and the reaching search does not know ADR. That left the BR an indirect
jump, and the first recompiled boot stopped there (a jump to rtld+0xaf4, a case label). function_constant (the
last fallback for a hoisted table register) now also counts ADR assignments.
2. Jump tables whose hoisted table register is reused between the table load and the BR. main+0xa6b520:
`adr x9, base; ldrh w10, [x11, x8, lsl #1]; add x9, x9, x10, lsl #2; ...; adrp x11, str; add x11, ...; br x9`,
x11 set to the table before the window. a64-kit resolves a hoisted table register at the BR, where x11 already
holds a string address, so the table was never found: the BR stayed an indirect jump, and the boot ran that
thread in the JIT from there on. The table register is now resolved at the load when the window writes it
after the load.
3. The symbolic window before a BR ends at ret/b/br, but not at the pointer-authenticated returns and branches
(RETAA/RETAB, BRAA/BRAB and their Z forms), so register values from the block before a `retab` leaked into
the dispatch block. zlib's inflate (main+0x1105e70) loses its table register x0 that way, to the
`mov w0, #-2` before the `retab` above the dispatch. The window now starts after the last such instruction.
"""
import a64_lift
_installed = False
def install():
global _installed
if _installed:
return
_installed = True
base_function_constant = a64_lift.Lifter.function_constant
def function_constant(self, br_pc, reg):
v = base_function_constant(self, br_pc, reg)
if v is not None:
return v
img = self.img
lo, hi = self.function_extent(br_pc)
if hi - lo > 0x40000:
return None
page, found = {}, set()
for pc in range(lo, hi, 4):
if not img.in_code(pc):
continue
w = img.word(pc)
d = w & 31
if (w & 0x9F000000) == 0x90000000: # ADRP
imm = ((w >> 3) & 0x1FFFFC) | ((w >> 29) & 3)
page[d] = (pc & ~0xFFF) + (a64_lift.sx(imm, 21) << 12)
elif (w & 0xFF800000) == 0x91000000 and ((w >> 5) & 31) in page: # ADD Xd, Xn, #imm
if d == reg:
found.add(page[(w >> 5) & 31] + (((w >> 10) & 0xFFF) << (12 if w & (1 << 22) else 0)))
elif (w & 0x9F000000) == 0x10000000 and d == reg: # ADR
imm = ((w >> 3) & 0x1FFFFC) | ((w >> 29) & 3)
found.add(pc + a64_lift.sx(imm, 21))
if len(found) == 1:
self.stats["jt_adr_table"] += 1
return found.pop()
return None
a64_lift.Lifter.function_constant = function_constant
base_jump_table = a64_lift.Lifter.jump_table
def table_load(self, br_pc, window):
"""(pc, base register) of the last register-offset integer load in the BR's window, if the window writes
its base register again after it"""
img = self.img
for pc in range(br_pc - 4, br_pc - 4 * window, -4):
if not img.in_code(pc):
return None
w = img.word(pc)
if (w & 0xFC000000) == 0x14000000 or (w & 0xFFFFFC1F) in (0xD65F0000, 0xD61F0000):
return None
if (w & 0x3B200C00) == 0x38200800 and not (w >> 26) & 1 and (w >> 22) & 3: # LDR* (register offset)
n = (w >> 5) & 31
if any(self.writes(img.word(q), n) for q in range(pc + 4, br_pc, 4)):
return pc, n
return None
return None
def auth_end(w):
"""RETAA, RETAB, BRAAZ/BRABZ, BRAA/BRAB: the end of a block, as ret and br are"""
return w in (0xD65F0BFF, 0xD65F0FFF) or (w & 0xFFFFF81F) == 0xD61F081F or (w & 0xFFFFF800) == 0xD71F0800
def jump_table(self, br_pc, reg, window=32):
img = self.img
for k in range(1, window + 1):
pc = br_pc - 4 * k
if img.in_code(pc) and auth_end(img.word(pc)):
window = k - 1
self.stats["jt_window_cut_at_auth_ret"] += 1
break
hit = table_load(self, br_pc, window)
if not hit:
return base_jump_table(self, br_pc, reg, window)
load_pc, treg = hit
redirected = {}
for name in ("reaching_constants", "hoisted_constant", "function_constant"):
bound = getattr(self, name)
def at_load(at, r, *rest, _f=bound):
return _f(load_pc if (at, r) == (br_pc, treg) else at, r, *rest)
redirected[name] = at_load
self.__dict__.update(redirected)
try:
out = base_jump_table(self, br_pc, reg, window)
finally:
for name in redirected:
del self.__dict__[name]
if out:
self.stats["jt_table_reg_reused"] += 1
return out
a64_lift.Lifter.jump_table = jump_table

16
nx2/lifted_c.py Normal file
View File

@@ -0,0 +1,16 @@
"""Where the lifted functions begin in the lift's C (code_*.c): each definition is one line, `void f_<address>(Ctx *__restrict
A64_NOESC c) {`, its name in group 1. Small on purpose: nx2/build.py reads it for every compile, without the lifter.
HEAD the definitions alone (split_hot: a piece takes each function from its own line)
HEAD_NOTED with the /* ... */ line the lift may put just above one (chunks: a part keeps a function's note with it)
HEAD_BYTES HEAD, for bytes (build.py's memory estimate reads the files as they lie)
The lift writes its C with \n line ends; a file with \r\n ones (written in text mode on Windows) is read the same: none of the
patterns needs the end of the definition's line, and a note's line may end in \r\n.
"""
import re
_DEF = r"void (f_[0-9a-f]+)\(Ctx \*__restrict A64_NOESC c\) \{"
HEAD = re.compile(r"^" + _DEF, re.M)
HEAD_NOTED = re.compile(r"^(?:/\* .* \*/\r?\n)?" + _DEF, re.M)
HEAD_BYTES = re.compile((r"^" + _DEF).encode(), re.M)

98
nx2/logo_rgba.py Normal file
View File

@@ -0,0 +1,98 @@
"""The start-up screen's logo (gpu/vk/vk_imgui.cpp, drawn by vk_menu.cpp) from assets/hookshot.png: a C header with its pixels as straight
RGBA bytes, written by nx2/build.py into the build's own folder (no second copy of the picture in the repository).
python nx2/logo_rgba.py [picture.png] [out.h]
Standard library only: the PNG is decoded here (8-bit RGBA or RGB, not interlaced: what the asset is).
"""
import os
import struct
import sys
import zlib
ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
SOURCE = os.path.join(ROOT, "assets", "triforce.png") # (Triforce: its crest; upstream: assets/hookshot.png)
def decode(data):
"""(width, height, RGBA bytes) of an 8-bit, non-interlaced RGBA or RGB PNG"""
if data[:8] != b"\x89PNG\r\n\x1a\n":
raise ValueError("not a PNG")
at, idat, head = 8, b"", None
while at < len(data):
n, kind = struct.unpack(">I4s", data[at:at + 8])
body = data[at + 8:at + 8 + n]
if kind == b"IHDR":
head = struct.unpack(">IIBBBBB", body)
elif kind == b"IDAT":
idat += body
elif kind == b"IEND":
break
at += 12 + n
w, h, depth, colour, _, _, interlace = head
if depth != 8 or colour not in (2, 6) or interlace:
raise ValueError("only 8-bit RGB/RGBA, not interlaced: %r" % (head,))
bpp = 4 if colour == 6 else 3
raw, stride = zlib.decompress(idat), w * bpp
out, prev = bytearray(), bytearray(stride)
for y in range(h):
f = raw[y * (stride + 1)]
line = bytearray(raw[y * (stride + 1) + 1:(y + 1) * (stride + 1)])
for x in range(stride):
a = line[x - bpp] if x >= bpp else 0
b = prev[x]
c = prev[x - bpp] if x >= bpp else 0
if f == 1:
line[x] = (line[x] + a) & 255
elif f == 2:
line[x] = (line[x] + b) & 255
elif f == 3:
line[x] = (line[x] + ((a + b) >> 1)) & 255
elif f == 4:
p = a + b - c
pa, pb, pc = abs(p - a), abs(p - b), abs(p - c)
line[x] = (line[x] + (a if pa <= pb and pa <= pc else b if pb <= pc else c)) & 255
prev = line
if bpp == 4:
out += line
else:
for x in range(w):
out += line[x * 3:x * 3 + 3] + b"\xff"
return w, h, bytes(out)
def header(png_path=SOURCE):
if not os.path.isfile(png_path): # (a source tree without the picture: the screen shows the wordmark alone)
return "// generated by nx2/logo_rgba.py: no assets/hookshot.png\n#pragma once\n"
w, h, px = decode(open(png_path, "rb").read())
lines = ["// generated by nx2/logo_rgba.py from assets/hookshot.png: do not edit",
"#pragma once",
"#define HOOKSHOT_LOGO 1",
"static const unsigned hookshot_logo_w = %d, hookshot_logo_h = %d;" % (w, h),
"static const unsigned char hookshot_logo_rgba[%d] = {" % len(px)]
for i in range(0, len(px), 32):
lines.append(",".join(str(b) for b in px[i:i + 32]) + ",")
lines.append("};")
return "\n".join(lines) + "\n"
def write(out_path, png_path=SOURCE):
"""Writes the header when it differs (an unchanged one keeps its timestamp); returns its text"""
text = header(png_path)
try:
with open(out_path, encoding="utf-8") as f:
if f.read() == text:
return text
except OSError:
pass
os.makedirs(os.path.dirname(out_path) or ".", exist_ok=True)
with open(out_path, "w", encoding="utf-8", newline="\n") as f:
f.write(text)
return text
if __name__ == "__main__":
src = sys.argv[1] if len(sys.argv) > 1 else SOURCE
out = sys.argv[2] if len(sys.argv) > 2 else "hookshot_logo.h"
write(out, src)
print("logo_rgba: %s" % out)

29
nx2/lower_test.py Normal file
View File

@@ -0,0 +1,29 @@
"""(hookshot) Builds gpu/vk/tests/lower_test.exe and runs it over a folder of dumped native programs
(RX_VK_DUMP_FAILED=<folder> in a run): what the Ampere -> Maxwell lowering still rejects, by reason and opcode.
python -I nx2/lower_test.py <folder> [samples]
"""
import os, subprocess, sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import kit # noqa: E402,F401 (puts the a64-kit tools on sys.path)
import toolchain # noqa: E402
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
SRC = os.path.join(ROOT, "gpu", "vk", "tests", "lower_test.cpp")
HDR = os.path.join(ROOT, "gpu", "vk", "eden", "shader_recompiler", "frontend", "ampere", "lower_to_maxwell.h")
EXE = os.path.join(ROOT, "work", "lower_test.exe")
def main():
if len(sys.argv) < 2:
print(__doc__)
return 2
if not os.path.exists(EXE) or max(os.path.getmtime(SRC), os.path.getmtime(HDR)) > os.path.getmtime(EXE):
cmd = [toolchain.find("clang++")] + toolchain.native_target() + [
"-std=c++20", "-O1", "-Wno-everything", "-I" + os.path.join(ROOT, "gpu", "vk", "eden"),
"-I" + os.path.join(ROOT, "gpu", "vk", "compat"), SRC, "-o", EXE]
subprocess.run(cmd, check=True)
return subprocess.run([EXE] + sys.argv[1:]).returncode
if __name__ == "__main__":
sys.exit(main())

1752
nx2/make_game.py Normal file

File diff suppressed because it is too large Load Diff

67
nx2/make_icon.py Normal file
View File

@@ -0,0 +1,67 @@
"""The app icon from a picture: an .ico for the builder and a .res the game's exe links in (nx2/build.py).
python nx2/make_icon.py <picture.png> [out folder, default work]
The picture is the user's and stays out of the repository, so these go to work/. Needs Pillow.
"""
import io
import os
import struct
import sys
from PIL import Image
SIZES = (16, 24, 32, 48, 64, 128, 256)
def square(im, size):
"""fitted in a transparent square, not stretched"""
im = im.convert("RGBA")
k = size / max(im.size)
fit = im.resize((max(1, round(im.width * k)), max(1, round(im.height * k))), Image.LANCZOS)
out = Image.new("RGBA", (size, size), (0, 0, 0, 0))
out.paste(fit, ((size - fit.width) // 2, (size - fit.height) // 2), fit)
return out
def png_of(im):
b = io.BytesIO()
im.save(b, "PNG", optimize=True)
return b.getvalue()
def res_entry(kind, name, data, flags=0x1010):
header = struct.pack("<II", len(data), 32) + struct.pack("<HHHH", 0xFFFF, kind, 0xFFFF, name)
header += struct.pack("<IHHII", 0, flags, 0x0409, 0, 0)
return header + data + b"\0" * (-len(data) & 3)
def main():
src = sys.argv[1]
out = sys.argv[2] if len(sys.argv) > 2 else os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "work")
im = Image.open(src)
images = [(s, png_of(square(im, s))) for s in SIZES]
head = struct.pack("<HHH", 0, 1, len(images))
offset = len(head) + 16 * len(images)
entries, body = b"", b""
group = head
for i, (s, data) in enumerate(images):
b = s & 255
entries += struct.pack("<BBBBHHII", b, b, 0, 0, 1, 32, len(data), offset + len(body))
group += struct.pack("<BBBBHHIH", b, b, 0, 0, 1, 32, len(data), i + 1)
body += data
with open(os.path.join(out, "hookshot.ico"), "wb") as f:
f.write(head + entries + body)
res = res_entry(0, 0, b"", 0) # (a .res file starts with an empty entry)
for i, (s, data) in enumerate(images):
res += res_entry(3, i + 1, data) # RT_ICON
res += res_entry(14, 1, group, 0x1030) # RT_GROUP_ICON: resource 1, what the window class loads
with open(os.path.join(out, "hookshot.res"), "wb") as f:
f.write(res)
print("make_icon: %d sizes, %d KB" % (len(images), len(res) >> 10))
if __name__ == "__main__":
main()

55
nx2/nso2.py Normal file
View File

@@ -0,0 +1,55 @@
"""Switch 2 additions to a64-kit's NSO loader (tools/nso.py), applied by importing this module.
DT_RELR (tags 35/36/37): every Switch 2 module (main, sdk, subsdk*, rtld of MKW) keeps its relative relocations
in the packed RELR format instead of RELA. Each RELR entry is either an address (even: relocate that word, then
continue after it) or a bitmap (odd: bit i of the 63 bits above bit 0 relocates the word i words further on, then
the cursor moves 63 words). The addend is implicit: the module offset stored at the location. They are added to
Nso.relocs as ordinary RELATIVE entries, so the image builder (hos/nx_image.py) applies them unchanged.
Without them every vtable and function pointer in .data stays a module offset.
"""
import struct
import kit # noqa: F401 (sys.path)
import nso
DT_RELRSZ, DT_RELR, DT_RELRENT = 35, 36, 37
nso.DT.update({DT_RELRSZ: "RELRSZ", DT_RELR: "RELR", DT_RELRENT: "RELRENT", 21: "DEBUG", 0x6FFFFFFB: "FLAGS_1"})
def relr_offsets(image, start, size):
out = []
where = 0
for off in range(start, start + size, 8):
e = struct.unpack_from("<Q", image, off)[0]
if not e & 1:
out.append(e)
where = e + 8
else:
bits = e >> 1
i = 0
while bits:
if bits & 1:
out.append(where + 8 * i)
bits >>= 1
i += 1
where += 63 * 8
return out
_orig_mod0 = nso.Nso._mod0
def _mod0(self):
_orig_mod0(self)
d = lambda k: self.dynamic.get(k, [0])[0]
start, size = d("RELR"), d("RELRSZ")
self.relr_count = 0
if start and size:
if d("RELRENT") not in (0, 8):
raise ValueError(f"{self.name}: RELRENT {d('RELRENT')}")
for off in relr_offsets(self.image, start, size):
self.relocs.append({"offset": off, "type": "RELATIVE", "sym": None, "addend": self.u64(off)})
self.relr_count += 1
nso.Nso._mod0 = _mod0

663
nx2/nup.py Normal file
View File

@@ -0,0 +1,663 @@
"""Read a .nup package (a decrypted Switch 2 dump: titles, each with an exefs/ and a romfs/ tree) and extract it.
python -I nx2/nup.py <file.nup> list the titles and their trees
python -I nx2/nup.py <file.nup> --verify read-only hashes/extents and expected game .zs headers
python -I nx2/nup.py <file.nup> --extract <dir> [--title <n|version>] [--only exefs|romfs]
The layout (worked out from the file; little-endian):
0x000 "NUP\\x01" header, u64 file size at +0x40
0x080 "TOC\\x01", u32 titles, u64 dirs, u64 files, u64 strings offset (from 0x80), u64 strings size
0x0b0 titles, 0x40 each: u64 title id, u32 root dir, u32 exefs dir, u32 romfs dir, u32 version, ...,
u64 offset + u64 size of the title's metadata blob at +0x20
then dirs, 24 bytes each: u32 name offset, u32 name length, u32 first child, u32 children, u64 0
then files, 80 bytes each: u32 name offset, u32 name length, u64 size, u64 data offset, u64 stored size,
32-byte hash, u8 codec (0 raw, 1 zstd), u8 flags, u16 reserved, 12 bytes reserved
then the child list, u32 each: a node index; below the dir count a dir, else (index - dirs) a file
then the name strings (not 0-terminated)
The data of a file is stored as is at its data offset, or, in .nup files from newer dumpers, as a zstd frame whose content is the
file: its size field is then the size of the file, not of the frame (seen on rtld: 10524 bytes in the listing, frame content 10524).
A file that is itself zstd in the game (the romfs .zs files) is not one of those: its frame's content is larger than the file.
"""
import argparse
from contextlib import suppress
import errno
import hashlib
import os
import re
import shutil
import stat
import struct
import sys
from time import monotonic
ZSTD = b"\x28\xb5\x2f\xfd"
WINDOWS_DEVICES = {"CON", "CONIN$", "CONOUT$", "PRN", "AUX", "NUL"} | {p + n for p in ("COM", "LPT") for n in "123456789¹²³"}
def frame_content_size(head):
"""The content size a zstd frame header states, None when it states none"""
if head[:4] != ZSTD or len(head) < 18:
return None
fhd = head[4]
single = (fhd >> 5) & 1
at = 5 + (0 if single else 1) + (0, 1, 2, 4)[fhd & 3]
n = (1 if single else 0, 2, 4, 8)[fhd >> 6]
if not n:
return None
size = int.from_bytes(head[at:at + n], "little")
return size + 256 if n == 2 else size
class Nup:
def __init__(self, path, *, strict=False):
self.path = path
self.f = open(path, "rb")
h = self.f.read(0x80)
if strict and len(h) != 0x80:
self.f.close()
raise ValueError("truncated NUP header")
if h[:4] != b"NUP\x01":
raise SystemExit("%s: not a NUP package (magic %r)" % (path, h[:4]))
self.file_size = struct.unpack_from("<Q", h, 0x40)[0]
self.toc_offset, self.toc_size = struct.unpack_from("<QQ", h, 8)
self.toc_sha256 = h[24:56]
self.blob_offset = struct.unpack_from("<Q", h, 0x38)[0]
# (a package that ends early (a download or copy that did not finish) failed 30 minutes in as "short read in the
# package": said now, with the sizes)
have = os.fstat(self.f.fileno()).st_size
if strict:
try:
self._verify_toc() # check metadata hash/bounds before parsing records
except (ValueError, OSError, struct.error):
self.f.close()
raise
self.f.seek(0x80)
if self.file_size and have < self.file_size:
raise SystemExit("the .nup file is incomplete: it is %d MB, its header says %d MB. Download or copy it again "
"(wait for the download to finish) and check that the drive had room" % (have >> 20, self.file_size >> 20))
t = self.f.read(0x30)
if t[:4] != b"TOC\x01":
raise SystemExit("%s: no TOC at 0x80" % path)
nt, nd, nf, soff, ssz = struct.unpack_from("<IQQQQ", t, 4)
self.nd, self.nf = nd, nf
dirs_at = 0x80 + 0x30 + 0x40 * nt
files_at = dirs_at + 24 * nd
kids_at = files_at + 80 * nf
strings_at = 0x80 + soff
self.f.seek(0)
meta = self.f.read(strings_at + ssz)
self.strings = meta[strings_at:strings_at + ssz]
self.titles = []
for i in range(nt):
o = 0xb0 + 0x40 * i
tid, root, exefs, romfs, ver = struct.unpack_from("<QIIII", meta, o)
self.titles.append(dict(index=i, id=tid, root=root, exefs=exefs, romfs=romfs, version=ver))
self.dirs = [struct.unpack_from("<4I", meta, dirs_at + 24 * i) for i in range(nd)]
self.files = []
self.file_details = [] # hash, codec flags and reserved bytes; keep files tuples compatible
for i in range(nf):
no, nl, size, off, size2 = struct.unpack_from("<IIQQQ", meta, files_at + 80 * i)
self.files.append((no, nl, size, off, size2, meta[files_at + 80 * i + 64]))
at = files_at + 80 * i
self.file_details.append((meta[at + 32:at + 64], meta[at + 65], meta[at + 66:at + 80]))
nk = (strings_at - kids_at) // 4
self.kids = struct.unpack_from("<%dI" % nk, meta, kids_at)
if strict:
try:
self._verify_tree()
except ValueError:
self.f.close()
raise
def _verify_toc(self):
"""Check parsed metadata bounds and the complete TOC hash."""
self.f.seek(0)
h = self.f.read(0x80)
physical = os.fstat(self.f.fileno()).st_size
if (len(h) != 0x80 or self.file_size != physical or self.toc_offset != 0x80 or
self.toc_size < 0x30 or self.toc_offset + self.toc_size > self.blob_offset or
self.blob_offset > physical or self.blob_offset % 16):
raise ValueError("invalid declared file/TOC/blob bounds")
self.f.seek(self.toc_offset)
t = self.f.read(0x30)
if len(t) != 0x30 or t[:4] != b"TOC\x01":
raise ValueError("invalid TOC header")
nt, nd, nf, soff, ssz = struct.unpack_from("<IQQQQ", t, 4)
tables_end = 0x30 + 64 * nt + 24 * nd + 80 * nf
if (not nt or tables_end > soff or (soff - tables_end) % 4 or
soff + ssz > self.toc_size):
raise ValueError("parsed title/directory/file/child/string tables exceed declared TOC")
self.f.seek(self.toc_offset)
left, digest = self.toc_size, hashlib.sha256()
while left:
block = self.f.read(min(left, 8 << 20))
if not block:
raise ValueError("truncated TOC")
left -= len(block)
digest.update(block)
if digest.digest() != self.toc_sha256:
raise ValueError("TOC SHA-256 mismatch")
def _verify_tree(self):
"""Validate ranges/reachability and reject cycles without recursive traversal."""
for no, nl, first, count in self.dirs:
if no + nl > len(self.strings) or first + count > len(self.kids):
raise ValueError("directory name/child range outside table")
self.name(no, nl)
for record in self.files:
if record[0] + record[1] > len(self.strings):
raise ValueError("file name outside string table")
self.name(record[0], record[1])
if any(child >= self.nd + self.nf for child in self.kids):
raise ValueError("child index outside directory/file tables")
reached_dirs, reached_files = set(), set()
for title in self.titles:
if title["root"] >= self.nd or any(title[part] != 0xffffffff and title[part] >= self.nd
for part in ("exefs", "romfs")):
raise ValueError("title %d has an invalid directory index" % title["index"])
seen, active, pending = set(), set(), [(title["root"], False)]
while pending:
directory, leaving = pending.pop()
if leaving:
active.remove(directory)
seen.add(directory)
continue
if directory in active:
raise ValueError("directory cycle in title %d" % title["index"])
if directory in seen:
continue
active.add(directory)
pending.append((directory, True))
_, _, first, count = self.dirs[directory]
for child in self.kids[first:first + count]:
if child < self.nd:
pending.append((child, False))
else:
reached_files.add(child - self.nd)
if any(title[part] != 0xffffffff and title[part] not in seen for part in ("exefs", "romfs")):
raise ValueError("title subtree is not reachable from its root")
reached_dirs.update(seen)
if len(reached_dirs) != self.nd or len(reached_files) != self.nf:
raise ValueError("orphan directory/file records are not reachable from title roots")
def verify_walk(self, directory):
"""Already validated nodes, iteratively (including data/ and deep trees)."""
pending = [(directory, "")]
while pending:
node, prefix = pending.pop()
if node < self.nd:
no, nl, first, count = self.dirs[node]
for child in reversed(self.kids[first:first + count]):
path = prefix + self.name(*self.dirs[child][:2]) + "/" if child < self.nd else prefix
pending.append((child, path))
else:
record = self.files[node - self.nd]
yield prefix + self.name(record[0], record[1]), record
def name(self, off, n):
return self.strings[off:off + n].decode("utf-8")
def walk(self, d, prefix=""):
"""(relative path, file record) for every file under dir d."""
no, nl, first, n = self.dirs[d]
for k in self.kids[first:first + n]:
if k < self.nd:
cno, cnl = self.dirs[k][:2]
yield from self.walk(k, prefix + self.name(cno, cnl) + "/")
else:
r = self.files[k - self.nd]
yield prefix + self.name(r[0], r[1]), r
def inner_is_zstd(self, r):
"""Whether what this frame holds is itself a zstd frame"""
import zstandard
self.f.seek(r[3])
try:
return zstandard.ZstdDecompressor().decompressobj().decompress(self.f.read(min(r[2], 1 << 20)))[:4] == ZSTD
except zstandard.ZstdError: # (made with one of the game's dictionaries: the game's own file)
return False
def is_packed(self, r, name=""):
"""A file the package keeps compressed. A frame that states no size is tried: it is the file when it unpacks to the listed size.
A .zs file is zstd in the game too, and three of them happen to hold exactly as much as they take: that one is packed only if
what it holds is zstd again"""
if r[5] == 1: # The TOC explicitly marks outer compression, even for a non-Zstd .zs payload.
return True
if not name:
name = self.name(r[0], r[1])
self.f.seek(r[3])
head = self.f.read(18)
if head[:4] != ZSTD:
return False
size = frame_content_size(head)
if name.endswith(".zs") and (size is None or size == r[2]):
return size == r[2] and self.inner_is_zstd(r)
if size is not None:
return size == r[2]
import zstandard
self.f.seek(r[3])
d = zstandard.ZstdDecompressor().decompressobj()
got = 0
left = r[2]
try:
while left and got <= r[2] and not d.eof:
b = self.f.read(min(left, 1 << 20))
if not b:
break
left -= len(b)
got += len(d.decompress(b))
except zstandard.ZstdError:
return False
return got == r[2]
def copy_to(self, r, g, name="", *, packed=None):
"""The file, as itself, into the open file g"""
exact = packed is not None
if packed is None:
packed = self.is_packed(r, name)
self.f.seek(r[3])
left = r[2]
if packed:
left = r[4] if r[5] == 1 else left + (1 << 16) # explicit extents must not read the next member
import zstandard
d = zstandard.ZstdDecompressor().decompressobj()
got = 0
while left and not getattr(d, "eof", False):
b = self.f.read(min(left, 8 << 20))
if not b:
break
left -= len(b)
try:
out = d.decompress(b)
except zstandard.ZstdError: # (the end of the frame, and the next file's bytes after it)
break
got += len(out)
if got > r[2]:
raise SystemExit("a compressed file expanded past its declared size of %d bytes" % r[2])
g.write(out)
if exact and (left or d.unused_data):
raise SystemExit("a compressed extent has trailing bytes")
if r[5] == 1 and not d.eof:
raise SystemExit("a compressed file ends before its frame completes")
if got != r[2]:
raise SystemExit("a compressed file came out as %d bytes, %d expected" % (got, r[2]))
return
while left:
b = self.f.read(min(left, 8 << 20))
if not b:
raise SystemExit("the .nup file ends early (short read at %d of %d MB): it is incomplete. Download or copy it again"
% (self.f.tell() >> 20, os.fstat(self.f.fileno()).st_size >> 20))
g.write(b)
left -= len(b)
def read(self, r, n=None):
import io
b = io.BytesIO()
self.copy_to(r, b)
return b.getvalue() if n is None else b.getvalue()[:n]
class VerifySink:
"""A streamed digest and just enough prefix for the inner game-file header."""
def __init__(self):
self.size, self.prefix, self.hash = 0, b"", hashlib.sha256()
def write(self, data):
self.size += len(data)
self.hash.update(data)
self.prefix += data[:max(0, 18 - len(self.prefix))]
def verify(nup, titles=None, only=None):
"""Read-only container checks plus expected .zs headers, not full inner decoding/authenticity."""
import zstandard
bad = checked = 0
cache = {}
physical = os.fstat(nup.f.fileno()).st_size
try:
nup._verify_toc()
nup._verify_tree()
except (ValueError, OSError, struct.error) as error:
print("BAD package: %s" % error)
return dict(checked_records=0, decoded_extents=0, bad=1)
indices = {id(record): i for i, record in enumerate(nup.files)}
for title in nup.titles if titles is None else titles:
parts = [(only, title[only])] if only else [("", title["root"])]
for part, directory in parts:
if directory == 0xffffffff:
continue
for name, record in nup.verify_walk(directory):
checked += 1
expected, flags, reserved = nup.file_details[indices[id(record)]]
size, off, stored, codec = record[2:]
reasons = []
if codec not in (0, 1) or flags or any(reserved):
reasons.append("unsupported codec/flags/reserved bytes")
if (size == 0 and (off or stored or codec)) or (size and (
not stored or off < nup.blob_offset or off % 16 or off > physical or stored > physical - off)):
reasons.append("invalid extent bounds")
if codec == 0 and stored != size:
reasons.append("raw extent length differs from member length")
key = (off, stored, codec, size)
if not reasons:
if key not in cache:
sink, error = VerifySink(), None
try:
# Honor the explicit codec; leave legacy extraction heuristics unchanged.
nup.copy_to(record, sink, name, packed=codec == 1)
except (OSError, SystemExit, zstandard.ZstdError) as failure:
error = str(failure)
cache[key] = (sink.size, sink.hash.digest(), sink.prefix, error)
count, digest, prefix, error = cache[key]
if error:
reasons.append("member decode failed: " + error)
if count != size:
reasons.append("decoded length differs from member length")
if digest != expected:
reasons.append("member SHA-256 mismatch")
if name.endswith(".zs"):
try:
if prefix[:4] != ZSTD:
raise ValueError("missing magic")
zstandard.get_frame_parameters(prefix)
except (ValueError, zstandard.ZstdError):
reasons.append(".zs member lacks the expected game Zstd header")
if reasons:
bad += 1
print("BAD title %d %016x v%d %s/%s: %s" % (
title["index"], title["id"], title["version"], part, name, "; ".join(reasons)))
print("verified %d records, %d unique extents: %d bad; game .zs headers only (not inner decode/authenticity)" % (
checked, len(cache), bad))
return dict(checked_records=checked, decoded_extents=len(cache), bad=bad)
def pick_title(nup, want):
if want is None:
return max(nup.titles, key=lambda t: t["version"]) # (the newest: the update over the base)
for t in nup.titles:
if str(t["index"]) == want or str(t["version"]) == want or "%016x" % t["id"] == want.lower():
return t
raise SystemExit("no title %s" % want)
def long_path(p):
"""(Windows) p as an extended-length path (\\\\?\\...), so a game folder whose deepest file passes 260 characters (MAX_PATH)
still unpacks, links and is read (#1468: a 262-character path in Event\\EventSetting stopped the build with
FileNotFoundError). Python takes these everywhere, without Windows' long-paths setting. Elsewhere, or one already: as is."""
if os.name != "nt" or not p:
return p
p = os.path.abspath(p)
if p.startswith("\\\\?\\"):
return p
if p.startswith("\\\\"): # (a share: \\server\share -> \\?\UNC\server\share)
return "\\\\?\\UNC\\" + p[2:]
return "\\\\?\\" + p
def extraction_path(out, part, rel, temporary_suffix=".part"):
"""A package path stays under its part, without following existing output links."""
parts = rel.split("/")
if (not rel or "\\" in rel or any(not p or p in (".", "..") or p[-1] in " ." or
any(c in '<>:"|?*' or ord(c) < 32 for c in p) or p.split(".")[0].rstrip(" ").upper() in WINDOWS_DEVICES
for p in parts)):
raise SystemExit("unsafe path in the package: %r" % rel)
root = long_path(os.path.realpath(os.path.abspath(out)))
base = os.path.join(root, part)
dst = os.path.join(base, *parts)
if os.path.commonpath((os.path.normcase(base), os.path.normcase(os.path.abspath(dst)))) != os.path.normcase(base):
raise SystemExit("path leaves the extraction folder: %r" % rel)
at = root
paths = [at]
for p in (part, *parts):
at = os.path.join(at, p)
paths.append(at)
temporary = dst + temporary_suffix
paths.append(temporary)
for path in paths:
try:
info = os.lstat(path)
except FileNotFoundError:
continue
if stat.S_ISLNK(info.st_mode) or getattr(info, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0):
raise SystemExit("extraction path is a link or reparse point: %s" % path)
if path == temporary and info.st_nlink > 1:
raise SystemExit("extraction temporary file has other hard links: %s" % path)
return dst
class ProgressSink:
"""Report successful writes without changing the package reader's copy interface."""
def __init__(self, output, report):
self.output = output
self.report = report
def write(self, data):
count = self.output.write(data)
self.report(count)
return count
def file_system(path):
"""The file system of the volume path is on (NTFS, exFAT, FAT32, ReFS): Windows only, None elsewhere"""
if os.name != "nt":
return None
import ctypes
k = ctypes.WinDLL("kernel32")
root, name = ctypes.create_unicode_buffer(261), ctypes.create_unicode_buffer(261)
if not k.GetVolumePathNameW(path, root, 261) or not k.GetVolumeInformationW(root, None, 0, None, None, None, name, 261):
return None
return name.value
def out_of_space(error, dst, part, rel, size):
"""A full drive said with the member being written and what its drive reports, to tell a full disk from a write the drive
refused with room to spare (issue 1388: 196 GB free on NTFS, stopped at 12 GB). None for any other error"""
if not isinstance(error, OSError) or error.errno != errno.ENOSPC:
return None
folder = os.path.dirname(dst)
try:
free = "%.1f GB free there" % (shutil.disk_usage(folder).free / 1e9)
except OSError:
free = "free space unknown"
kind = file_system(folder)
return OSError(errno.ENOSPC, "No space left on device writing %s/%s (%d bytes): %s%s" % (
part, rel, size, free, ", " + kind if kind else ""))
def extract(nup, t, out, only=None, quiet=False):
total = n = written = 0
interval = 2
next_report = 0 if quiet else monotonic() + interval
def progress(size):
nonlocal written, next_report
written += size
now = monotonic()
if now >= next_report:
print(" %.1f MiB written (%d files completed)" % (written / (1 << 20), n), flush=True)
next_report = now + interval
for part in ("exefs", "romfs"):
if only and part != only:
continue
for rel, r in nup.walk(t[part]):
dst = extraction_path(out, part, rel)
if os.path.exists(dst) and os.path.getsize(dst) == r[2]:
continue # (re-runs pick up where they stopped)
os.makedirs(os.path.dirname(dst), exist_ok=True)
temporary = dst + ".part"
g = open(temporary, "wb")
try:
with g:
nup.copy_to(r, g if quiet else ProgressSink(g, progress), rel)
os.replace(temporary, dst)
except BaseException as error:
full = out_of_space(error, dst, part, rel, r[2]) # (measured before the member's space is freed)
# Free the failed member's space without hiding the extraction error.
with suppress(OSError):
os.unlink(temporary)
if full:
raise full from error
raise
total += r[2]
n += 1
return n, total
def heal(nup, t, out, names):
"""An update's exefs need not hold every module whole: what is not an NSO there is taken from the package's other titles"""
for m in names:
p = extraction_path(out, "exefs", m)
have = b""
if os.path.exists(p):
with open(p, "rb") as f:
have = f.read(16)
if have[:4] == b"NSO0":
continue
for other in reversed(nup.titles):
if other["index"] == t["index"]:
continue
for rel, r in nup.walk(other["exefs"]):
if rel == m and nup.read(r, 4) == b"NSO0":
with open(p + ".part", "wb") as g:
g.write(nup.read(r))
os.replace(p + ".part", p)
print(" %s: taken from title %d (v%d)" % (m, other["index"], other["version"]))
break
else:
continue
break
else:
if have:
print(" %s is not a game module in any title: it starts %s" % (m, have.hex()))
def zstd_start(head):
"""Whether a file starts as zstd data: a frame, or a skippable frame (magic 0x184D2A50 to 0x184D2A5F)"""
return head == ZSTD or (len(head) == 4 and head[1:] == b"\x2a\x4d\x18" and head[0] & 0xf0 == 0x50)
def damaged_game_files(romfs, progress=None):
"""The romfs .zs files that do not start as zstd: in the 23.3 GB dumps three of them begin with zeros (the complete
dump is about 23.8 GB), and the game then leaves out every model that uses one: the Water Temple's walls and floors,
the Great Fairy Fountain's floor (#1002, #718, #1259). progress(done, total), when given, after each file: the files
are listed first, then opened one by one (about 100,000, minutes where a virus scanner looks at each one)"""
root = long_path(romfs) # (walked and read as long paths; the paths returned are under romfs as given)
paths = [] # (the long path read, the path under romfs as given)
for folder, dirs, files in os.walk(root):
dirs.sort()
rel = os.path.relpath(folder, root)
paths += [(os.path.join(folder, name), os.path.normpath(os.path.join(romfs, rel, name)))
for name in sorted(files) if name.endswith(".zs")]
found = []
for i, (path, given) in enumerate(paths):
with open(path, "rb") as f:
if not zstd_start(f.read(4)):
found.append(given)
if progress:
progress(i + 1, len(paths))
return found
def stand_in_for(path, damaged=()):
"""A whole file of the same kind beside a damaged one, to load in its place: the same last part of the name
(_Nrm.bntx.zs: a normal map for a normal map), the longest common start of the name, then the nearest size. Never
one of the damaged files, even one already given a stand-in"""
folder, name = os.path.split(path)
suffix = re.search(r"(_[A-Za-z]+\d*)?(\.[^_]*)$", name).group(0)
size = os.path.getsize(long_path(path))
best = None
for other in os.listdir(long_path(folder)):
if other == name or not other.endswith(suffix):
continue
candidate = os.path.join(folder, other)
if candidate in damaged:
continue
with open(long_path(candidate), "rb") as f:
if not zstd_start(f.read(4)):
continue
score = (len(os.path.commonprefix((name, other))), -abs(os.path.getsize(long_path(candidate)) - size))
if best is None or score > best[0]:
best = (score, candidate)
return best and best[1]
def stand_in_damaged(romfs, progress=None):
"""Each damaged .zs file in an unpacked romfs replaced by a copy of its stand-in: (damaged, stand-in or None) each.
Only for a romfs this tool unpacked: the player's own folder is never written to. progress: damaged_game_files'"""
done = []
damaged = damaged_game_files(romfs, progress)
for path in damaged:
other = stand_in_for(path, set(damaged))
if other:
shutil.copyfile(long_path(other), long_path(path + ".part"))
os.replace(long_path(path + ".part"), long_path(path))
done.append((path, other))
return done
def main():
ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
ap.add_argument("nup")
ap.add_argument("--extract", metavar="DIR")
ap.add_argument("--verify", action="store_true", help="read-only hashes/extents and expected game .zs headers; all titles unless --title")
ap.add_argument("--title", help="index, version or title id (extract default: newest; verify default: all)")
ap.add_argument("--only", choices=("exefs", "romfs"))
a = ap.parse_args()
if a.verify and a.extract:
ap.error("--verify does not extract; run extraction separately")
try:
nup = Nup(a.nup, strict=a.verify)
except (OSError, ValueError, struct.error, IndexError, SystemExit) as error:
if not a.verify:
raise
print("BAD package: %s" % error)
return 1
if a.verify:
try:
titles = None if a.title is None else [pick_title(nup, a.title)]
return 1 if verify(nup, titles, a.only)["bad"] else 0
finally:
nup.f.close()
print("%s: %.2f GB, %d titles, %d dirs, %d files" % (os.path.basename(a.nup), nup.file_size / 1e9,
len(nup.titles), nup.nd, nup.nf))
for t in nup.titles:
ex = list(nup.walk(t["exefs"]))
ro = list(nup.walk(t["romfs"]))
print("title %d: %016x v%d exefs %d files (%s) romfs %d files, %.2f GB" % (
t["index"], t["id"], t["version"], len(ex), ", ".join("%s %d" % (p, r[2]) for p, r in ex),
len(ro), sum(r[2] for _, r in ro) / 1e9))
same = sum(1 for _, r in ro if r[2] != r[4])
if same:
print(" (%d romfs files with two different sizes)" % same)
if a.extract:
t = pick_title(nup, a.title)
# Measure the destination volume, including an existing mount or junction.
folder = os.path.realpath(a.extract)
while not os.path.isdir(folder):
parent = os.path.dirname(folder)
if parent == folder:
break
folder = parent
free = shutil.disk_usage(folder).free
print("extracting title %d (v%d) to %s (%.0f GB free)" % (t["index"], t["version"], a.extract, free / 1e9))
n, total = extract(nup, t, a.extract, a.only)
print("done: %d files written, %.2f GB" % (n, total / 1e9))
if a.only != "romfs": # (main is ZBIC-compressed on firmware 22+: made a plain NSO for the tools)
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import unpack
heal(nup, t, a.extract, unpack.NAMES)
for m in unpack.NAMES:
p = extraction_path(a.extract, "exefs", m, temporary_suffix=".plain")
if os.path.exists(p) and unpack.unpack_nso(p):
print(" %s: ZBIC -> plain NSO" % m)
if __name__ == "__main__":
for stream in (sys.stdout, sys.stderr): # (-I ignores PYTHONIOENCODING: a .nup's or a file's name in other letters crashed a print)
try:
stream.reconfigure(encoding="utf-8", errors="replace")
except (AttributeError, ValueError):
pass
sys.exit(main())

66
nx2/oot_jitter.py Normal file
View File

@@ -0,0 +1,66 @@
"""The temporal upscalers' camera jitter for Ocarina of Time 1.1.1: C added after a few instructions of the game (a64_lift.py
Lifter.after). One jitter for all of them: NVIDIA DLSS and AMD FSR 4 (gpu/vk/vk_temporal.cpp, the frame chain they share) and
AMD FSR 3.1 (gpu/vk/vk_fsr3_inputs.cpp).
A temporal upscaler resolves detail finer than a pixel from frames whose camera moved by a different fraction of a
pixel each time. The game's cameras do not move so: these sites give the scene's projection a sub-pixel offset each frame,
in the GPU's copy only. The game fills each view's uniform buffers in one function (main+0x6bc520): for each view it loads
the view's projection matrix (the view at x28, its projection's rows at +0x1a8) and writes it, the view-projection made from
it and more into the buffers the draws read. The sites are those loads: the jitter goes on the third column of the first two
rows (P[0][2], P[1][2]), in the registers, so the buffers get the jittered matrix and everything made from it, while the
game's own cameras and views keep the projection as it is.
The jitter used to go into sead's perspective projection itself (its frustum offset, +0xb4 and +0xb8), where the game's
CPU work saw it too: the shadow cascades are fitted to the view's frustum, so with the camera still the shadow map's origin
moved by a sixth of a shadow texel each frame (up to a third; 0.003 without the jitter), and with it the shadows' edges
(#841: a tester saw shadows flicker at DLAA). Here the cascades (and culling, and anything else on the CPU) see the view as
the game set it.
host/rx_backend.c hos_dlss_jitter_clip picks the jitter and checks the projection is one of the picture's: perspective, of
16:9 or the window's ratio (nx2/oot_wide.py). It is none (the game as shipped) unless DLSS or an FSR runs (the renderer's
rx_option HOS_OPT_DLSS_JITTER_W and _H are 0 otherwise), and the renderer reads the offset back from the camera's uniform
buffer. Which view is the scene's, the renderer learns from its draws, too late for
this: the other views of the picture's ratio (a reflection, a portrait) are drawn without DLSS and move by under a pixel
of the scene's size, too little to see.
Installed in every standalone lift (nx2/lift.py): FSR 3.1 needs no SDK, so a build without NVIDIA's has the jitter too.
"""
import oot_wide
# (main offset, the original instruction word there, and the registers whose third lane gets the jitter's x and y: None
# for one not loaded there). The view is x28 at all three; its projection's rows are at +0x1a8, 16 bytes apart.
SITES = (
# the view-projection block: ldp q19, q20, [x8] (rows 0 and 1), then this ldp q21, q22, [x8, #0x20] (rows 2 and 3)
(0x6bc708, 0xad415915, 19, 20),
# the second block: ldr q26, [x9] (row 0), ldr q6, [x9] (row 1)
(0x6bcf8c, 0x3dc0013a, 26, None),
(0x6bcf9c, 0x3dc00126, None, 6),
)
VIEW, PROJECTION = 28, 0x1a8
def _text(xreg, yreg):
add = "".join("{ float _o; memcpy(&_o, (uint8_t *)c->vreg[%d] + 8, 4); _o += _j[%d]; memcpy((uint8_t *)c->vreg[%d] + 8, &_o, 4); } "
% (r, axis, r) for axis, r in ((0, xreg), (1, yreg)) if r is not None)
return ("{ extern int hos_dlss_jitter_clip(uint64_t projection, float out[2]); float _j[2]; "
"if (hos_dlss_jitter_clip(c->x[%d] + %#x, _j)) { %s} } /* (nx2/oot_jitter.py: the upscalers' camera jitter) */"
% (VIEW, PROJECTION, add))
def install(lifter, img):
"""The sites into lifter.after, after what is there; a site whose word differs (another game version) is left out"""
n = 0
for off, word, xreg, yreg in SITES:
pc = oot_wide.MAIN + off
try:
have = img.word(pc)
except Exception:
have = None
if have != word:
print("dlss jitter: main+%#x holds %s, not %08x: left out" % (off, "?" if have is None else "%08x" % have, word))
continue
lifter.after[pc] = (lifter.after[pc] + "\n" if pc in lifter.after else "") + _text(xreg, yreg)
n += 1
print("dlss jitter: %d of %d sites" % (n, len(SITES)))
return n

View File

@@ -0,0 +1,97 @@
"""Native scene resolution for Ocarina of Time 1.1.1 (v131072): Resolution 100% renders the 3D at the picture's size.
From oscarrc91's PR #578 (native scene base scale) and i900gaming's PR #329 (stale fixed-resolution scale); both switches are
host globals (host/nn_vi.c, hos_main.c) instead of environment reads in the game's code.
- The graphics configuration has a scale table apart from the GPU-time controller: (1.0, 0.8, 2/3) for each axis, in two
profiles. The game runs in mode 1, so even the fixed path's 100% sized the scene at 0.8 of the picture (2048x1152 in
2560x1440). f_806702e0 picks the mode and tells the resources about a change: with hos_native_scene set it changes mode 1's
0.8 to 1.0 (after checking the whole stock table) and makes the mode compare unequal once, so the game's own notification
runs and the targets are made again at the new size.
- f_831f3200, the scene's update: the fixed path can ask for a resource refresh without marking the scene's cached scale
dirty, so the last dynamic scale stayed (0.75: 1536x864). With hos_fix_res set, a cached scale that differs from the one
asked for sets the scale-dirty bit (8) and the game's own code recomputes it.
Another game version leaves the functions out (and says so).
"""
import glob
import os
import whole_write # (rewrites written beside and moved over: issue 1197)
CONFIG_HEAD = "void f_806702e0(Ctx *__restrict A64_NOESC c) {"
CONFIG_ANCHOR = " c->x[14]=recomp_load8(c,c->x[0] + 1354);\n"
CONFIG_DECL = "extern volatile int hos_native_scene;\nextern void hos_log(const char *, ...);\n"
CONFIG_PATCH = """ /* (nx2/oot_native_resolution.py: mode 1's base scale 0.8 -> 1.0, PR #578) */
if (hos_native_scene) {
for (unsigned _profile = 0; _profile < 2; ++_profile) {
const uint64_t _table = c->x[0] + 724 + _profile * 520;
if (recomp_load64(c, _table) == 0x3f8000003f800000ULL &&
recomp_load64(c, _table + 8) == 0x3f4ccccd3f4ccccdULL &&
recomp_load64(c, _table + 16) == 0x3f2aaaab3f2aaaabULL) {
recomp_store64(c, _table + 8, 0x3f8000003f800000ULL);
c->x[14] = 0xff; /* (the old mode compares unequal once: the game's own refresh runs) */
hos_log("native-scene: profile %u mode 1 base scale 0.8 -> 1.0", _profile);
}
}
}
"""
SCENE_HEAD = "void f_831f3200(Ctx *__restrict A64_NOESC c) {"
SCENE_DECL = "extern volatile int hos_fixed_resolution;\nextern volatile int hos_fix_res;\nextern volatile uint32_t hos_fixed_scale_bits;\n"
SCENE_PATCH = """
/* (nx2/oot_native_resolution.py: the fixed scale marked dirty when the cached one differs, PR #329) */
if (hos_fixed_resolution && hos_fix_res) {
float _req, _base, _ref, _prev; uint32_t _u = hos_fixed_scale_bits; memcpy(&_req, &_u, 4);
_u = recomp_load32(c, c->x[0] + 3664); memcpy(&_base, &_u, 4);
_u = recomp_load32(c, c->x[0] + 3672); memcpy(&_ref, &_u, 4);
_u = recomp_load32(c, c->x[0] + 3676); memcpy(&_prev, &_u, 4);
if (_base > 0.0f && _ref > 0.0f && _req > 0.0f && _req <= 2.0f && _prev != _req * _ref / _base)
recomp_store32(c, c->x[0] + 3628, recomp_load32(c, c->x[0] + 3628) | 8u);
}
"""
HEADER = '#include "a64.h"\n'
def _body(s, head):
start = s.index(head)
end = s.find("\nvoid f_", start + len(head))
return start, s[start:end if end >= 0 else len(s)]
def patch_config(s):
"""None when the file has no such function; s unchanged when its code differs"""
if CONFIG_HEAD not in s:
return None
start, body = _body(s, CONFIG_HEAD)
if CONFIG_PATCH in body or body.count(CONFIG_ANCHOR) != 1 or "goto L_80670648;" not in body or s.count(HEADER) != 1:
return s
at = start + body.index(CONFIG_ANCHOR) + len(CONFIG_ANCHOR)
s = s[:at] + CONFIG_PATCH + s[at:]
return s.replace(HEADER, HEADER + CONFIG_DECL, 1)
def patch_scene(s):
if SCENE_HEAD not in s:
return None
if SCENE_PATCH in s or s.count(SCENE_HEAD) != 1 or s.count(HEADER) != 1:
return s
s = s.replace(SCENE_HEAD, SCENE_HEAD + SCENE_PATCH, 1)
return s.replace(HEADER, HEADER + SCENE_DECL, 1)
def apply(gen):
done = {"scene base scale (f_806702e0)": 0, "scene scale check (f_831f3200)": 0}
for path in sorted(glob.glob(os.path.join(glob.escape(gen), "code_*.c"))):
with open(path, encoding="utf-8", newline="") as f:
s = f.read()
out = s
for what, fn in (("scene base scale (f_806702e0)", patch_config), ("scene scale check (f_831f3200)", patch_scene)):
r = fn(out)
if r is not None and r != out:
out = r
done[what] += 1
print("oot_native_resolution: %s in %s" % (what, os.path.basename(path)))
if out != s:
whole_write.write_whole(path, out, newline="")
for what, n in done.items():
if n != 1:
print("oot_native_resolution: WARNING: the %s was found %d times" % (what, n))

37
nx2/oot_occlusion.py Normal file
View File

@@ -0,0 +1,37 @@
"""Occlusion Off for Ocarina of Time 1.1.1: C added after six instructions of the game (a64_lift.py Lifter.after).
The game decides at six places whether to make its z-prepass occlusion queries and its samples-passed queries, each by a
byte it loads (ldrb w8, [x0]). With hos_occlusion_off set (HOOKSHOT_OCCLUSION_OFF=1), that byte reads 0: the game makes
no occlusion queries, so it issues no occlusion reports and no conditional renders, and every candidate object is drawn.
Our renderer has no real occlusion counts (each draw adds 16 samples), so the game's visibility answers came from that guess.
The sites are cLohan's community patch (zortos293/Lichii#641), as Fritchii ships it (Frizti/Fritchii 1a46ba972,
"Occlusion Off", ldrb w8, [x0] -> mov w8, wzr). Each site's original word is checked: another game version leaves it out.
"""
MAIN = 0x80020000 # (main's base in this image: nx2/eden_image.py)
LDRB_W8_X0 = 0x39400008
OFFSETS = (0x11ACE68, 0x11B828C, 0x11B82A4, 0x11ACEC8, 0x11B82DC, 0x11B82F4)
TEXT = ("{ extern volatile int hos_occlusion_off; if (hos_occlusion_off) c->x[8] = 0; } "
"/* (nx2/oot_occlusion.py: no occlusion queries) */")
def install(lifter, img):
"""The sites into lifter.after; a site whose word differs (another game version) is left out"""
n = 0
for off in OFFSETS:
pc = MAIN + off
try:
have = img.word(pc)
except Exception:
have = None
if have != LDRB_W8_X0:
print("occlusion off: main+%#x holds %s, not %08x: left out" % (off, "?" if have is None else "%08x" % have, LDRB_W8_X0))
continue
if pc in lifter.after:
lifter.after[pc] += "\n" + TEXT
else:
lifter.after[pc] = TEXT
n += 1
print("occlusion off: %d of %d sites" % (n, len(OFFSETS)))
return n

454
nx2/oot_patches.py Normal file
View File

@@ -0,0 +1,454 @@
"""Changes to the lifted C for Ocarina of Time, applied by lift.py after it writes the code (and by hand: python -I nx2/oot_patches.py).
Each is a runtime switch where it can be, so the game's own path stays reachable.
- Dynamic resolution (Lichii's built-in "Max Resolution" patch, main+0x805BD0): the engine's config flag at +0x94 picks the path that
follows the GPU-time controller; with hos_fixed_resolution set it takes the other path, the 3D at full scale. The controller made
a new set of render targets at each size, ~11 images a frame at times, and held the GPU near its budget.
- Text speed (the Insert menu's "Text speed"): f_80b1e140 (main+0xafe140) steps a text pane's letter animation, the
LetterAnimControl of the panes with "LetterAnimOn" user data (dialogue, Navi, signs). Its time argument (s0) moves the
letters' clock (+0x60 += rate +0x4c * time, the rate from the message's speed tag) and the pauses' timer (+0x58 -= time);
hos_text_scale multiplies it, 1.0 as shipped. The auto-advance timer (+0x5c, before) keeps the game's time. A huge
scale shows a page at once (one frame per pause), as the game's own skip on A (+0x72 = 2, f_80794b00) without its flags.
- The fixed path's scale (hos_fixed_scale_bits: the Insert menu's Resolution) reaching the 3D, in the caller f_831f3200
(main+0x31d3200). It recomputes the 3D's scale (+0xe5c = scale * +0xe58 / +0xe50) only while +0xe2c bit 3 is set. The dynamic
path sets it on a level change; the fixed path's flag 1 calls f_831f41d0, whose bit 3 the apply block clears in the same frame,
before any recompute. So the fixed path kept the dynamic path's last scale (0.75 docked: 1536x864 at every setting). Flag 1 now
also asks for the recompute, and on the fixed path the flags +0xe20 bits 3/4 (a hardcoded 0.75 / 0.5 in place of the scale)
are not looked at.
"""
import glob
import os
import sys
import whole_write # (rewrites written beside and moved over: issue 1197)
DECL = "extern volatile int hos_fixed_resolution;\nextern volatile int hos_dynres_runs;\nextern volatile uint32_t hos_fixed_scale_bits;\n"
OLD = "if (!((c->x[8] >> 0) & 1)) { goto L_8082640c; }"
NEW = "if (hos_fixed_resolution || !((c->x[8] >> 0) & 1)) { goto L_8082640c; }\n hos_dynres_runs++;"
LABEL = "L_8082640c:;"
SCALE_OLD = "c->x[9] = 0x3f800000ULL;"
SCALE_NEW = "c->x[9] = hos_fixed_scale_bits;" # (the fixed path's scale, 1.0 as shipped)
DECL2 = "extern volatile int hos_fixed_resolution;\nextern volatile uint64_t hos_res_module;\nextern volatile uint32_t hos_fixed_scale_bits;\n"
APPLY_OLD = "L_831f3efc:;\n c->x[9]=recomp_load32(c,c->x[20] + 3628);"
APPLY_NEW = "L_831f3efc:;\n if (hos_fixed_resolution) { /* (only while +0xe5c is not the chosen scale: a recompute every frame remade the targets) */\n union { uint32_t u; float f; } now = {(uint32_t)recomp_load32(c,c->x[20] + 3676)}, want = {hos_fixed_scale_bits},\n e50 = {(uint32_t)recomp_load32(c,c->x[20] + 3664)}, e58 = {(uint32_t)recomp_load32(c,c->x[20] + 3672)};\n const float d = now.f - (e50.f != 0 ? want.f * e58.f / e50.f : want.f);\n if (d > 1e-4f || d < -1e-4f) c->x[8] |= 8;\n }\n c->x[9]=recomp_load32(c,c->x[20] + 3628);"
FLAGS_OLD = "L_831f3f08:;\n c->x[8]=recomp_load64(c,c->x[20] + 3616);\n"
FLAGS_NEW = FLAGS_OLD + " hos_res_module = c->x[20];\n if (hos_fixed_resolution) c->x[8] &= ~0x18ULL;\n"
def fixed_scale(t):
"""The fixed path's scale reaching the 3D (the Resolution row). (t, 1) when applied here"""
if APPLY_OLD not in t or FLAGS_OLD not in t:
return t, 0
t = t.replace(APPLY_OLD, APPLY_NEW, 1).replace(FLAGS_OLD, FLAGS_NEW, 1)
return t.replace('#include "a64.h"\n', '#include "a64.h"\n' + DECL2, 1), 1
TEXT_DECL = "extern volatile float hos_text_scale;\n"
LIFE_BUILD_ID = "e0f3ccfaf50065ae4ea01eb5aeb13b1a45da3348"
LIFE_HEAD = "void f_804bd8c0(Ctx *__restrict A64_NOESC c) {\n"
MOVE_HEAD = "void f_82c415e0(Ctx *__restrict A64_NOESC c) {\n"
MOVE_ANCHOR = ("if (((c->x[8] >> 1) & 1)) { goto L_82c41748; }\n")
MOVE_MATRIX = "L_82c41778:;\n"
WARP_HEAD = "void f_806d2530(Ctx *__restrict A64_NOESC c) {\n"
WARP_ANCHOR = "{ uint64_t _t = c->x[8]; c->x[30] = 0x806d25c8ULL; A64_ICALL(c, _t); }\n"
VIEW_HEAD = "void f_804e48c0(Ctx *__restrict A64_NOESC c) {\n"
# (the head of the view's loop: a bare label, since nx2/poll.py puts the interrupt poll before each backward goto; with the
# poll after the label in C lifted before that)
VIEW_ANCHORS = ("L_804e5068:;\n", "L_804e5068:; A64_POLL(c);\n")
LIFE_CAS = ("{ uint64_t _a = c->x[13]; uint32_t _e = (uint32_t)c->x[11]; "
"__atomic_compare_exchange_n((uint32_t *)A64_PTR(_a), &_e, (uint32_t)c->x[12], "
"0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); A64_MARK(_a, 4); c->x[11] = (uint64_t)_e; }")
def patch_life(source):
start = source.find(LIFE_HEAD)
if start < 0:
return None
end = source.find("\n}", start)
if end < 0 or source[start:end].count(LIFE_CAS) != 2:
return None
# x19 is the actor. Confirm its type and life pointer before protecting this write.
guard = (" if ((uint32_t)c->x[12] < (uint32_t)c->x[11] && hos_oot_hearts_enabled() &&\n"
" hos_oot_is_player(c, c->x[19]) &&\n"
" c->x[13] == recomp_load64(c,c->x[19] + 7184) + 8)\n"
" c->x[12] = c->x[11];\n")
body = source[start:end].replace(LIFE_CAS, guard + LIFE_CAS)
return (source[:start] + body + source[end:]).replace(
'#include "a64.h"\n', '#include "a64.h"\nextern int hos_oot_hearts_enabled(void);\n'
'extern int hos_oot_is_player(Ctx *c, uint64_t actor);\n', 1)
def patch_movement(source):
start = source.find(MOVE_HEAD)
if start < 0:
return None
end = source.find("\n}", start)
if end < 0 or source[start:end].count(MOVE_ANCHOR) != 1 or source[start:end].count(MOVE_MATRIX) != 1:
return None
bypass = (" hos_oot_lift_player(c, c->x[19]);\n"
+ MOVE_ANCHOR)
body = source[start:end].replace(MOVE_ANCHOR, bypass, 1).replace(
MOVE_MATRIX, MOVE_MATRIX + " hos_oot_flight_pose(c, c->x[19], c->x[25]);\n", 1)
source = source[:start] + body + source[end:]
return source.replace('#include "a64.h"\n', '#include "a64.h"\n'
'extern void hos_oot_lift_player(Ctx *c, uint64_t actor);\n'
'extern void hos_oot_flight_pose(Ctx *c, uint64_t actor, uint64_t matrix);\n', 1)
def patch_flight_rotation(source):
start = source.find(WARP_HEAD)
if start < 0:
return None
end = source.find("\n}", start)
body = source[start:end]
if end < 0 or body.count(WARP_ANCHOR) != 1:
return None
body = body.replace(WARP_ANCHOR, " hos_oot_flight_rotation(c, c->x[0], c->x[1]);\n" + WARP_ANCHOR)
return (source[:start] + body + source[end:]).replace(
'#include "a64.h"\n', '#include "a64.h"\n'
'extern void hos_oot_flight_rotation(Ctx *c, uint64_t controller, uint64_t matrix);\n', 1)
def patch_view(source):
start = source.find(VIEW_HEAD)
if start < 0:
return None
end = source.find("\n}", start)
body = source[start:end]
if end < 0:
return None
for anchor in VIEW_ANCHORS:
if body.count(anchor) == 1:
break
else:
return None
body = body.replace(anchor, anchor + " hos_oot_free_view(c, c->x[20]);\n")
return (source[:start] + body + source[end:]).replace(
'#include "a64.h"\n', '#include "a64.h"\nextern void hos_oot_free_view(Ctx *c, uint64_t view);\n', 1)
# Native mouse look (host/mouse_native.c): the player camera's update, f_80644c80 (a virtual method, the camera in x0), hands
# its camera to the host before anything else, on the game's thread, once a frame; with the menu's Mouse camera on the mouse's
# motion turns the camera's yaw (+0x1a4) and pitch (+0x18c) there, before the camera reads its input.
CAM_HEAD = "void f_80644c80(Ctx *__restrict A64_NOESC c) {\n"
def patch_mouse_camera(source):
start = source.find(CAM_HEAD)
if start < 0:
return None
if "hos_mouse_camera(c->x[0]);" in source: # (made before: a lift patched again)
return source
at = start + len(CAM_HEAD)
source = source[:at] + " hos_mouse_camera(c->x[0]); /* (nx2/oot_patches.py: native mouse look) */\n" + source[at:]
return source.replace('#include "a64.h"\n', '#include "a64.h"\nextern void hos_mouse_camera(uint64_t camera);\n', 1)
MENU_FOCUS_HEAD = "void f_802ed660(Ctx *__restrict A64_NOESC c) {\n"
def patch_menu_focus(source):
"""f_802ed660, a menu screen's handler of a pane getting the focus (host/oot_menu.c), calls
hos_oot_menu_focus first, with the screen, the pane and its caller's return address"""
start = source.find(MENU_FOCUS_HEAD)
if start < 0:
return None
if "hos_oot_menu_focus(c," in source: # (made before: a lift patched again)
return source
at = start + len(MENU_FOCUS_HEAD)
hook = " hos_oot_menu_focus(c, c->x[0], c->x[1], c->x[30]); /* (nx2/oot_patches.py: the menus' focus) */\n"
return (source[:at] + hook + source[at:]).replace(
'#include "a64.h"\n', '#include "a64.h"\nextern void hos_oot_menu_focus(Ctx *c, uint64_t screen, uint64_t pane, uint64_t lr);\n', 1)
MENU_TICK_HEAD = "void f_802f1380(Ctx *__restrict A64_NOESC c) {\n"
def patch_menu_tick(source):
"""f_802f1380, a menu screen's update (each frame, before the screen's UI controller: +56, its vtable's
+16), calls hos_oot_menu_tick first with the screen: the game's menus' mouse (host/oot_menu.c)"""
start = source.find(MENU_TICK_HEAD)
if start < 0:
return None
if "hos_oot_menu_tick(c," in source: # (made before: a lift patched again)
return source
at = start + len(MENU_TICK_HEAD)
hook = " hos_oot_menu_tick(c, c->x[0]); /* (nx2/oot_patches.py: the menus' mouse) */\n"
return (source[:at] + hook + source[at:]).replace(
'#include "a64.h"\n', '#include "a64.h"\nextern void hos_oot_menu_tick(Ctx *c, uint64_t screen);\n', 1)
TEXT_HEAD = "void f_80b1e140(Ctx *__restrict A64_NOESC c) {"
# (the letters' block: past the early returns, the auto-advance timer and the hidden / no-text checks; reached once a call)
TEXT_OLD = ("L_80b1e1b4:;\n"
" c->x[9]=recomp_load16(c,c->x[19] + 110);\n"
"if ((uint32_t)c->x[9] == 0) { goto L_80b1e18c; }\n")
# (the same block in a build with interrupt polls: nx2/poll.py puts the poll before each backward goto, as a block)
TEXT_OLD_POLLED = TEXT_OLD.replace("{ goto L_80b1e18c; }", "{ { A64_POLL(c); goto L_80b1e18c; } }")
TEXT_ADD = (" if (hos_text_scale != 1.0f) { float _t; uint32_t _u; memcpy(&_t,c->vreg[0],4); _t *= hos_text_scale; "
"memcpy(&_u,&_t,4); a64_vset64(c->vreg[0],_u); } /* (nx2/oot_patches.py: text speed) */\n")
TEXT_NEW = TEXT_OLD + TEXT_ADD
def patch_dynres(s):
if OLD not in s:
return None
s = s.replace(OLD, NEW, 1)
at = s.index(LABEL)
scale = s.index(SCALE_OLD, at)
s = s[:scale] + SCALE_NEW + s[scale + len(SCALE_OLD):]
return s.replace('#include "a64.h"\n', '#include "a64.h"\n' + DECL, 1)
# The frame's real time in place of the nominal rate, for a Frame rate cap above 60 (host/nn_vi.c hos_frame_seconds_bits; 0, as
# at a cap of 60: the game's own divide, nothing changes). The engine has two places where its integer nominal rate (+0x2c8 of
# the graphics module) becomes a time step, both a float divide "base / nominal":
# - f_80706be0: the frame's step, base 60.0 (30.0 in the half-rate mode), added to the game clock (the double at
# [x19+0xd58], in 1/60 s) and handed on as the frame's seconds;
# - f_83b6ea90: the global step (60.0 / nominal, a float in .data) the rest of the engine reads.
# The engine's own rate is 60 / <whole display intervals>, never above 60, and its controller puts 60 back over a higher value
# written from outside before either divide runs: above 60 fps every frame still stepped 1/60 s and the game ran fps / 60 times
# too fast. With the host's measured frame time the step is base * seconds, whatever the rate. hos_step_probe: f_80706be0's
# x19, which tells the host the hook is in this build and gives it the game clock for its log (HOS_FPS_LOG=1).
STEP_SITES = (("void f_80706be0(Ctx *__restrict A64_NOESC c) {\n", 8, "\n hos_step_probe = c->x[19];"),
("void f_83b6ea90(Ctx *__restrict A64_NOESC c) {\n", 0, ""))
STEP_OLD = " _v=a64_fpx_div32_div32(c,_a,_b);\n a64_vset64(c->vreg[%d],_v);}"
STEP_HOOK = (" _v=a64_fpx_div32_div32(c,_a,_b);\n"
" { const uint32_t _hs = hos_frame_seconds_bits; if (_hs) { float _base, _secs; uint32_t _w; memcpy(&_base,&_a,4); "
"memcpy(&_secs,&_hs,4); _base *= _secs; memcpy(&_w,&_base,4); _v = _w; } } /* (base * the frame's seconds: nx2/oot_patches.py) */\n"
" a64_vset64(c->vreg[%d],_v);}")
STEP_MARK = "hos_frame_seconds_bits; if (_hs)" # (once per divide with the hook)
STEP_DECL = "extern volatile uint32_t hos_frame_seconds_bits;\nextern volatile uint64_t hos_step_probe;\n"
def patch_frame_seconds(s):
"""The host's frame time at the step divides of f_80706be0 and f_83b6ea90: (s, the sites patched here). A site whose
function is not in the file, is patched already, or whose code differs (another game version) is left as it is."""
done = 0
for head_text, reg, extra in STEP_SITES:
head = s.find(head_text)
if head < 0:
continue
end = s.find("\nvoid f_", head + len(head_text))
body = s[head:end if end >= 0 else len(s)]
old, new = STEP_OLD % reg, STEP_HOOK % reg + extra
if new in body or body.count(old) != 1 or body.count("a64_fpx_div32_div32") != 1:
continue
at = head + body.index(old)
s = s[:at] + new + s[at + len(old):]
done += 1
if done and STEP_DECL not in s:
s = s.replace('#include "a64.h"\n', '#include "a64.h"\n' + STEP_DECL, 1)
return s, done
def unpatch_frame_seconds(s):
"""s without the hook, as the lift wrote it: for a lift where only one of the two divides was found (apply)"""
for _, reg, extra in STEP_SITES:
s = s.replace(STEP_HOOK % reg + extra, STEP_OLD % reg)
return s if STEP_MARK in s else s.replace(STEP_DECL, "", 1) # (a hook of another shape stays, with its declarations)
def patch_text_speed(s):
"""The scale at the head of the letters' block of f_80b1e140; None when the file has no such function, s unchanged when the
function is there but its code differs (another game version)."""
head = s.find(TEXT_HEAD)
if head < 0:
return None
end = s.find("\nvoid f_", head + len(TEXT_HEAD))
body = s[head:end if end >= 0 else len(s)]
for old in (TEXT_OLD_POLLED, TEXT_OLD): # (with the interrupt poll before the block's backward goto, or without: --no-polls)
if old + TEXT_ADD in body: # (applied already: a second run)
return s
if body.count(old) == 1:
break
else:
return s
if body.count("goto L_80b1e1b4;") != 1:
return s
at = head + body.index(old)
s = s[:at] + old + TEXT_ADD + s[at + len(old):]
return s.replace('#include "a64.h"\n', '#include "a64.h"\n' + TEXT_DECL, 1)
def apply(gen, *, protect_hearts=False, required=False):
"""required: the lift of the supported build (LIFE_BUILD_ID), whole: a missing output-size edit stops it (apply_output_size).
The other patches only warn when their place is not found once."""
import oot_native_resolution # (HOS_SCENE_NATIVE=1: the scene's base scale 1.0; off by default since #849's output resolution)
oot_native_resolution.apply(gen)
done = {"dynamic resolution branch": 0, "text speed block": 0, "fixed path's scale": 0}
if protect_hearts: # (the supported build's: these places are its functions)
done["menu focus hook"] = 0
done["menu tick hook"] = 0
done["player heart protection"] = 0
done["player flight"] = 0
done["player flight rotation"] = 0
done["free camera"] = 0
done["mouse camera"] = 0
step_sites, step_files = 0, [] # (the divides with the hook, and the files that have it)
for path in sorted(glob.glob(os.path.join(glob.escape(gen), "code_*.c"))):
with open(path, encoding="utf-8", newline="") as f:
s = f.read()
out = s
d = patch_dynres(out)
if d is not None:
out = d
done["dynamic resolution branch"] += 1
print("oot_patches: dynamic resolution switch in", os.path.basename(path))
t = patch_text_speed(out)
if t is not None and t != out:
out = t
done["text speed block"] += 1
print("oot_patches: text speed scale in", os.path.basename(path))
focus = patch_menu_focus(out) if protect_hearts else None
if focus is not None:
out = focus
done["menu focus hook"] += 1
print("oot_patches: the menus' focus hook in", os.path.basename(path))
tick = patch_menu_tick(out) if protect_hearts else None
if tick is not None:
out = tick
done["menu tick hook"] += 1
print("oot_patches: the menus' update hook in", os.path.basename(path))
out, n = fixed_scale(out)
if n:
done["fixed path's scale"] += 1
print("oot_patches: the fixed path's scale in", os.path.basename(path))
life = patch_life(out) if protect_hearts else None
if life is not None:
out = life
done["player heart protection"] += 1
print("oot_patches: player heart protection in", os.path.basename(path))
move = patch_movement(out) if protect_hearts else None
if move is not None:
out = move
done["player flight"] += 1
print("oot_patches: player flight in", os.path.basename(path))
rotation = patch_flight_rotation(out) if protect_hearts else None
if rotation is not None:
out = rotation
done["player flight rotation"] += 1
print("oot_patches: player flight rotation in", os.path.basename(path))
mouse = patch_mouse_camera(out) if protect_hearts else None
if mouse is not None:
out = mouse
done["mouse camera"] += 1
print("oot_patches: native mouse camera in", os.path.basename(path))
view = patch_view(out) if protect_hearts else None
if view is not None:
out = view
done["free camera"] += 1
print("oot_patches: free camera in", os.path.basename(path))
out, sites = patch_frame_seconds(out)
if STEP_MARK in out: # (made now, or by an earlier run over this file)
step_sites += out.count(STEP_MARK)
step_files.append(path)
if sites:
print("oot_patches: the frame's time at %d step divide(s) in %s" % (sites, os.path.basename(path)))
if out != s:
whole_write.write_whole(path, out, newline="")
for what, n in done.items():
if n != 1:
print("oot_patches: WARNING: the %s was found %d times" % (what, n))
if step_sites != len(STEP_SITES):
# (both or neither: with one divide timed and the other stepping 1/60 s the two clocks of the game would run apart. Without
# the hook the host keeps the rate up to the cap, as before: host/nn_vi.c, hos_step_probe)
for path in step_files:
with open(path, encoding="utf-8", newline="") as f:
s = f.read()
whole_write.write_whole(path, unpatch_frame_seconds(s), newline="")
print("oot_patches: WARNING: the frame's time was found at %d of %d step divides, none patched (a Frame rate cap above 60 "
"keeps the game's own step)" % (step_sites, len(STEP_SITES)))
apply_output_size(gen, required=required)
# Output size (host/nn_applet.c hos_output_size: the Insert menu's Output resolution, 2560 x 1440 as shipped):
# - nnMain (f_81222ce0) puts the display size in the framework's create arg (+20/+24, the framework's +156/+160); the window
# textures are made at it.
# - f_810a7b70 sets up the display-size object (B = [[0x84ca29c0]+216]) from a config (x1): its docked size list (+8, first
# 2560 x 1440) is the frame's base (B+244; the frame is the base times the graphics scale's factor, f_806702e0, capped by the
# scale's display class: docked 3840 x 2160 at scale 0). Its first entry asks the host when it differs.
OUT_DECL = "uint32_t hos_output_size(int height);\nuint64_t hos_rt_pool_size(uint64_t game);\n"
OUT_EDITS = [
# (the picture's 2560 x 1440 at 0x81202ee4, not f_831f3200's copy at +0xb68; nx2/oot_wide.py's widescreen size, after it,
# stays over it: host/hos_main.c makes that one at this height)
(" c->x[8] = 0xa00ULL;\n c->x[8] = (c->x[8] & ~(0xFFFFULL<<32)) | (0x5a0ULL<<32);\n"
"{ extern volatile uint64_t hos_wide_out; const uint64_t _s = hos_wide_out; if (_s) c->x[8] = _s; } /* (nx2/oot_wide.py: hos_wide_out) */\n"
" { uint64_t _b=c->x[21]; int64_t _o=20; recomp_store64(c,(_b+_o),c->x[8]); }",
" c->x[8] = hos_output_size(0) | ((uint64_t)hos_output_size(1) << 32);\n"
"{ extern volatile uint64_t hos_wide_out; const uint64_t _s = hos_wide_out; if (_s) c->x[8] = _s; } /* (nx2/oot_wide.py: hos_wide_out) */\n"
" { uint64_t _b=c->x[21]; int64_t _o=20; recomp_store64(c,(_b+_o),c->x[8]); }"),
(" { uint64_t _a=c->x[0], _b=724ULL; uint64_t _r=_a+_b; c->x[21]=_r; }\n"
" { uint64_t _r = ((uint64_t)0 | ((c->x[1]) << 0)); c->x[20] = _r; }\n",
" { uint64_t _a=c->x[0], _b=724ULL; uint64_t _r=_a+_b; c->x[21]=_r; }\n"
" { uint64_t _r = ((uint64_t)0 | ((c->x[1]) << 0)); c->x[20] = _r; }\n"
" if (hos_output_size(0) != 2560 && recomp_load32(c, c->x[20] + 8) == 2560) { recomp_store32(c, c->x[20] + 8, hos_output_size(0)); recomp_store32(c, c->x[20] + 12, hos_output_size(1)); }\n"),
# - f_80ea2f30: the 3D's largest size, ResolutionParam's BaseResolution (or 1920 x 1080 as constants), 3/4 of 2560 x 1440;
# above it the game takes a smaller upscale network (1/3: 1280 x 720 at 4K). Scaled with the output.
("c->x[30] = 0x80ea2ffcULL; f_831cca50(c);\n"
" { uint64_t _b=c->x[0]; int64_t _o=0; { uint64_t _p0,_p1; recomp_ldp32(c,(_b+_o),&_p0,&_p1); c->x[24]=_p0; c->x[25]=_p1; } }\n",
"c->x[30] = 0x80ea2ffcULL; f_831cca50(c);\n"
" { uint64_t _b=c->x[0]; int64_t _o=0; { uint64_t _p0,_p1; recomp_ldp32(c,(_b+_o),&_p0,&_p1); c->x[24]=_p0; c->x[25]=_p1; } }\n"
" if (hos_output_size(0) != 2560) { c->x[24] = (uint32_t)c->x[24] * hos_output_size(0) / 2560; c->x[25] = (uint32_t)c->x[25] * hos_output_size(1) / 1440; }\n"),
(" c->x[8] = 0x44f00000ULL;\n c->x[8] &= 0xFFFFFFFFULL;\n recomp_store32(c,c->x[19] + 3668,c->x[8]);\n"
" c->x[8] = 0x44870000ULL;\n c->x[8] &= 0xFFFFFFFFULL;\n",
" c->x[8] = hos_output_size(0) * 3 / 4;\n { float _f = (float)c->x[8]; uint32_t _u; memcpy(&_u, &_f, 4); c->x[8] = _u; }\n"
" recomp_store32(c,c->x[19] + 3668,c->x[8]);\n"
" c->x[8] = hos_output_size(1) * 3 / 4;\n { float _f = (float)c->x[8]; uint32_t _u; memcpy(&_u, &_f, 4); c->x[8] = _u; }\n"),
# - The render targets' GPU pool: 0x13100000 bytes (the graphics config's +1464, read in f_836ce9d0, which takes its storage from
# the "Graphics" heap and makes a physical pool and a virtual one over it: f_80c4a1c0 -> f_80c49070 -> f_80c49380 -> f_811c9ba0).
# Sized for 2560 x 1440: a 3840x2160 3D at the 4K output ran out of it and the next target came back null (crash in f_8005ed00,
# the pool's allocator, its +0x28 0x13100000). Scaled with the output's pixels (host/nn_applet.c
# hos_rt_pool_size: 687 MB at 3840 x 2160), at the config, so the storage and the pool agree.
("L_836cea54:;\n",
"L_836cea54:;\n"
" if (recomp_load64(c, c->x[19] + 1464) == 0x13100000ULL) recomp_store64(c, c->x[19] + 1464, hos_rt_pool_size(0x13100000ULL));\n"),
# - The "Graphics" heap holds it, made by f_83b7b340 with the other module heaps (twice: 0x34900000 bytes in f_83b7a2c0's first
# pass, then the heap kept, 0x348ffe20; it had under 16 MB to spare): grown by as much as the pool.
("c->x[30] = 0x83b7b440ULL; f_809df710(c);\n",
"if (c->x[0] == 0x34900000ULL || c->x[0] == 0x348ffe20ULL) c->x[0] += hos_rt_pool_size(0x13100000ULL) - 0x13100000ULL;\n"
"c->x[30] = 0x83b7b440ULL; f_809df710(c);\n"),
# - f_83b7b340 then takes each module heap's use (its vtable +120) off the module's budget (the configured size, a table of slots)
# and aborts below zero (f_8351c280): the Graphics heap's use counts as the shipped one.
("{ uint64_t _t = c->x[8]; c->x[30] = 0x83b7b4e0ULL; A64_ICALL(c, _t); }\n",
"{ uint64_t _t = c->x[8]; c->x[30] = 0x83b7b4e0ULL; A64_ICALL(c, _t); }\n"
"{ const uint64_t _b = recomp_load64(c, c->x[21] + (c->x[22] << 3)), _g = hos_rt_pool_size(0x13100000ULL) - 0x13100000ULL; "
"if ((_b == 0x34900000ULL || _b == 0x348ffe20ULL) && c->x[0] > _g) c->x[0] -= _g; }\n"),
# - f_80b72340 (from the graphics init f_836cf4b0) makes a heap of what the Graphics heap can still give, which took the growth as
# well (0x1b6cb48 bytes as shipped, 0x1996cb48 grown) and left too little for the pool: it keeps its shipped size.
("c->x[30] = 0x80b723acULL; f_809df700(c);\n",
"if (c->x[0] > hos_rt_pool_size(0x13100000ULL) - 0x13100000ULL) c->x[0] -= hos_rt_pool_size(0x13100000ULL) - 0x13100000ULL;\n"
"c->x[30] = 0x80b723acULL; f_809df700(c);\n"),
]
def apply_output_size(gen, required=False):
"""required (the supported build's whole lift): each edit must be made, or found made, exactly once, else SystemExit: without
one the Insert menu's Output resolution is ignored, or a 4K output runs out of the render targets' pool. Elsewhere (another
build, part of a lift, by hand) a warning"""
found = [0] * len(OUT_EDITS)
for path in sorted(glob.glob(os.path.join(glob.escape(gen), "code_*.c"))):
with open(path, encoding="utf-8", newline="") as f:
s = f.read()
hit = False
for i, (old, new) in enumerate(OUT_EDITS):
# (every place: each made one, and each old text outside a made one; a new text can hold its old one. A file with
# both is counted for both, and its old ones are left: a second made place or a stray raw one is not once)
made = s.count(new)
raw = (s.replace(new, "\0") if made else s).count(old)
found[i] += made + raw
if raw and not made:
s = s.replace(old, new)
hit = True
if hit:
if OUT_DECL not in s:
s = s.replace('#include "a64.h"\n', '#include "a64.h"\n' + OUT_DECL, 1)
whole_write.write_whole(path, s, newline="")
print("oot_patches: output size in", os.path.basename(path))
if found != [1] * len(OUT_EDITS):
if required:
raise SystemExit("oot_patches: the output size places were found %s times, not once each (OUT_EDITS %s): the "
"lifted C no longer has the text they match" % (found, [i for i, n in enumerate(found) if n != 1]))
print("oot_patches: WARNING: the output size places were found %s times" % found)
if __name__ == "__main__":
apply(sys.argv[1] if len(sys.argv) > 1 else os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "work", "sa", "gen"))

109
nx2/oot_wide.py Normal file
View File

@@ -0,0 +1,109 @@
"""Widescreen and ultrawide for Ocarina of Time 1.1.1: C added after a few instructions of the game (a64_lift.py Lifter.after).
The game draws a 2560x1440 picture whatever the screen and renders its scene at 1920x1080; its cameras use a 16:9 ratio.
With these patches the picture takes the window's shape (the height stays the game's, the width grows on an ultrawide
screen and shrinks on a 16:10 or 4:3 one), the 16:9 cameras take the picture's ratio, the HUD and menus keep their 16:9
shape in the middle of a wider picture (a narrower one: its whole width), pre-rendered movies play in that middle 16:9,
and the picture a save keeps shows that middle 16:9 (a narrower picture: its middle band, at the texture's shape). Every patch reads host globals
(host/hos_wide.c) that stay 0 for a 16:9 window, so the game runs as shipped then.
The sites and what each does come from NX2navi's recomp/patches-v111.txt, where they are credited to Ivan0xFF (MIT,
"The matrix-update, UI and movie sites: Copyright (c) 2026 Ivan0xFF"); the picture and scene size sites are the same
constants that file names; the save picture's site is Hookshot's own (issue 952). Here each is written as C after the
original instruction instead of a replacement of it.
Each site's original instruction word is checked: another game version leaves the site out (and says so).
"""
MAIN = 0x80020000 # (main's base in this image: nx2/eden_image.py)
UI_M00, UI_M11 = "(2.0f / 3840.0f)", "(2.0f / 2160.0f)" # (the UI's full-screen orthographic projection, 3840x2160 grid)
# (the screens whose UI keeps the whole picture: the item-wheel and pause / system menu backgrounds redraw a capture of
# the whole picture; the event and loading fades cover it. Bit k of host/hos_main.c's hos_wide_keep_mask keeps screen k
# whole; the Insert menu's Ultrawide menus row clears the menus' bits 1 and 3 (vk_framegen.cpp): their layout then
# keeps its 16:9 shape in the middle, as the HUD's does, instead of being stretched over the whole picture. Bit 0, the
# item wheel's background (the last frame of play, blurred, HUD and all), and bit 2, the fades, keep covering it all: bit 0
# cleared, that capture went into the middle at twice its width)
KEEP_VTABLES = (MAIN + 0x4bed098, MAIN + 0x4be4fb0, MAIN + 0x4be61c8, MAIN + 0x4be7b30)
def _ratio(vreg, base=None):
"""A ratio just loaded into s<vreg>: when it is 16:9 (within 0.01), the picture's; with base, also stored back at +0xb0
(sead's perspective projection keeps it there, and culling reads it)."""
store = ("recomp_store32(c, c->x[%d] + 0xb0, _u); " % base) if base is not None else ""
return ("{ extern volatile float hos_wide_ratio; const float _w = hos_wide_ratio; if (_w > 0.0f) { float _r; "
"memcpy(&_r, c->vreg[%d], 4); const float _d = _r - (16.0f / 9.0f); if (_d <= 0.01f && _d >= -0.01f) { "
"uint32_t _u; memcpy(&_u, &_w, 4); a64_vset64(c->vreg[%d], _u); %s} } } /* (nx2/oot_wide.py: camera ratio) */"
% (vreg, vreg, store))
def _size(name):
return ("{ extern volatile uint64_t %s; const uint64_t _s = %s; if (_s) c->x[8] = _s; } /* (nx2/oot_wide.py: %s) */"
% (name, name, name))
HUD = ("{ extern volatile float hos_wide_squeeze; const float _q = hos_wide_squeeze; if (_q > 0.0f) { float _m00, _m11; "
"memcpy(&_m00, c->vreg[1], 4); memcpy(&_m11, (const uint8_t *)c->vreg[4] + 4, 4); "
"if (_m00 == %s && _m11 == %s) { const uint64_t _vt = recomp_load64(c, c->x[0] - 0x1b8); "
"extern volatile uint32_t hos_wide_keep_mask; const uint32_t _kb = %s; "
"if (!(_kb & hos_wide_keep_mask)) { float _row[4]; memcpy(_row, c->vreg[1], 16); for (int _k = 0; _k < 4; _k++) _row[_k] *= _q; "
"memcpy(c->vreg[1], _row, 16); } } } } /* (nx2/oot_wide.py: HUD in the middle 16:9) */"
% (UI_M00, UI_M11, " : ".join("_vt == 0x%xULL ? %du" % (v, 1 << k) for k, v in enumerate(KEEP_VTABLES)) + " : 0u"))
MOVIE = ("{ extern volatile float hos_wide_margin; const float _m = hos_wide_margin; if (_m > 0.0f) { "
"const uint64_t _vp = c->x[29] - 40; uint32_t _ua = (uint32_t)recomp_load32(c, _vp + 8), _ub = (uint32_t)recomp_load32(c, _vp + 16); "
"float _a, _b; memcpy(&_a, &_ua, 4); memcpy(&_b, &_ub, 4); const float _in = (_b - _a) * _m; _a += _in; _b -= _in; "
"memcpy(&_ua, &_a, 4); memcpy(&_ub, &_b, 4); recomp_store32(c, _vp + 8, _ua); recomp_store32(c, _vp + 16, _ub); } }"
" /* (nx2/oot_wide.py: movies in the middle 16:9) */")
# The save's picture (issue 952): the pause menu draws the whole picture into a 543 x 306 texture, the save's JPEG, through the
# game's viewport-and-scissor helper (f_802a46e0: SetScissor, then SetViewport(x, y, w, h) in w1-w4 from [x8], called by blr x8).
# Wider than 16:9, the whole picture was squeezed into it. That one viewport is made wider than the texture by the picture's
# ratio over 16:9, centred: the middle 16:9 fills the texture (the scissor keeps the texture's), as a 16:9 window would show.
# Narrower than 16:9 (4:3, 16:10) it is made taller instead, by 16:9 over the picture's ratio: the picture's middle band fills
# the texture at its own shape, neither squeezed nor with bars.
THUMB = ("{ extern volatile float hos_wide_ratio; const float _r = hos_wide_ratio; if (_r > 0.0f && (uint32_t)c->x[1] == 0 "
"&& (uint32_t)c->x[2] == 0 && (uint32_t)c->x[3] == 543 && (uint32_t)c->x[4] == 306) { "
"if (_r > 16.0f / 9.0f) { const int _w = (int)(543.0f * _r / (16.0f / 9.0f) + 0.5f); "
"c->x[1] = (uint32_t)((543 - _w) / 2); c->x[3] = (uint32_t)_w; } "
"else { const int _h = (int)(306.0f * (16.0f / 9.0f) / _r + 0.5f); "
"c->x[2] = (uint32_t)((306 - _h) / 2); c->x[4] = (uint32_t)_h; } } }"
" /* (nx2/oot_wide.py: the save's picture, the middle 16:9) */")
# (main offset, the original instruction word there, the C after it)
SITES = (
# the picture: the graphics setup and the display settings object build {2560, 1440} in x8 (mov, then this movk)
(0x1202ee4, 0xf2c0b408, _size("hos_wide_out")),
(0x31f39f8, 0xf2c0b408, _size("hos_wide_out")),
# the scene the game's upscaler takes in, {1920, 1080}
(0x31f3a04, 0xf2c08708, _size("hos_wide_scene")),
# the camera update copies the camera's ratio (ldr s0, [x19, #0x140]) into the projection the renderer reads
(0x4c5334, 0xbd414260, _ratio(0)),
# sead's perspective projection's matrix update (ldp s0, s1, [x0, #0xac]) and its three inlined copies
(0x3485cc0, 0x2d558400, _ratio(1, 0)),
(0x4f1e8, 0x2d558ea0, _ratio(3, 21)),
(0x23b9a8, 0x2d558c00, _ratio(3, 0)),
(0x33b82c, 0x2d558440, _ratio(1, 2)),
# the 2D canvas setup: the layout's projection rows (ldp q1, q4, [x2]); the canvas is x0, its screen 0x1b8 before it
(0x774368, 0xad401041, HUD),
# the movie layer's draw: its viewport (min x at x29 - 32, max x at x29 - 24) after ldp x9, x8, [x19, #0x18]
(0x3b5c758, 0xa941a269, MOVIE),
# the viewport helper's SetViewport, after the call's address is loaded (ldr x8, [x8]), before blr x8
(0x284824, 0xf9400108, THUMB),
)
def install(lifter, img):
"""The sites into lifter.after; a site whose word differs (another game version) is left out"""
n = 0
for off, word, text in SITES:
pc = MAIN + off
try:
have = img.word(pc)
except Exception:
have = None
if have != word:
print("widescreen: main+%#x holds %s, not %08x: left out" % (off, "?" if have is None else "%08x" % have, word))
continue
lifter.after[pc] = text
n += 1
print("widescreen: %d of %d sites" % (n, len(SITES)))
return n

43
nx2/pgo_pieces.py Normal file
View File

@@ -0,0 +1,43 @@
"""The profile from the whole code_hot.c made to fit its pieces (nx2/split_hot.py).
python -I nx2/pgo_pieces.py <in.prof> <out.prof> [pieces, default 8]
A file's static functions go by "<file>:<name>" in a profile, so the helpers of a64.h would match only in a file of the same name. Each
such record is written once per piece. The rest go by name alone and stay as they are.
"""
import os
import subprocess
import sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import kit # noqa: E402,F401
import toolchain # noqa: E402
PREFIX = "code_hot.c:"
def main():
src, dst = sys.argv[1:3]
pieces = int(sys.argv[3]) if len(sys.argv) > 3 else 8
tool = toolchain.need(toolchain.find("llvm-profdata"))
text = subprocess.run([tool, "merge", "--text", "-o", "-", src], capture_output=True, text=True, encoding="utf-8", check=True).stdout
head, _, rest = text.partition("\n\n")
records = rest.split("\n\n")
out = []
moved = 0
for r in records:
if r.startswith(PREFIX):
moved += 1
out.extend("code_hot_%d.c:%s" % (i, r[len(PREFIX):]) for i in range(pieces))
elif r.strip():
out.append(r)
tmp = dst + ".txt"
with open(tmp, "w", encoding="utf-8", newline="\n") as f:
f.write(head + "\n\n" + "\n\n".join(out) + "\n")
subprocess.run([tool, "merge", "--text-format" if False else "--binary", "-o", dst, tmp], check=True)
os.remove(tmp)
print("pgo_pieces: %d records, %d helpers for %d pieces" % (len(records), moved, pieces))
if __name__ == "__main__":
main()

67
nx2/pipeline_states.py Normal file
View File

@@ -0,0 +1,67 @@
"""The pipeline state lists (gpu/vk/vk_draw.cpp: triforce_pipeline_states.bin, written as the game is played): show one, or merge
several into the list a release ships (work/pipeline_states.bin, copied by nx2/make_game.py beside the game as
triforce_pipeline_states_shipped.bin, whose pipelines the first start's shader preparation builds before the title).
python -I nx2/pipeline_states.py <list>... what they hold
python -I nx2/pipeline_states.py --out work/pipeline_states.bin <list>... merged, each pipeline once
A list holds no game code: each record is a pipeline's stage code keys (hashes of the game's shaders, which the player's own
triforce_shaders.bin turns back into the shaders), the program variant's hash, and the formats and draw state baked into the pipeline.
<list>: "HKSTATE1", u32 record size (528), then the records:
u64 code[4] (vertex or mesh, fragment, task, the header-only stage), u64 variant,
u32 mesh, topology, targets, samples, depth format, has depth, u32 color formats[8], u8 color types[8],
u64 blend[8], u32 vertex attributes[16], u32 attribute count, u32 strides[16], u32 divisors[16],
u32 blend enables, channel mask, polygon mode, depth clamp, alpha to coverage, pad,
u32 answers, u32 questions, u32 answers[24] (stage, descriptor, image, type), u8 questions[32] (each answer as asked)
"""
import argparse
import collections
import struct
import sys
MAGIC = b"HKSTATE1"
SIZE = 528
def read(path):
with open(path, "rb") as f:
data = f.read()
if data[:8] != MAGIC or struct.unpack_from("<I", data, 8)[0] != SIZE:
raise SystemExit("%s: not a pipeline state list" % path)
body = data[12:]
return [body[i:i + SIZE] for i in range(0, len(body) - SIZE + 1, SIZE)]
def describe(recs):
programs = collections.Counter()
kinds = collections.Counter()
for r in recs:
code = struct.unpack_from("<4Q", r, 0)
mesh = struct.unpack_from("<I", r, 40)[0]
programs[(code, mesh)] += 1
kinds["mesh" if mesh else "graphics"] += 1
return "%d pipelines (%s) of %d programs" % (len(recs), ", ".join("%s %d" % kv for kv in sorted(kinds.items())), len(programs))
def main():
p = argparse.ArgumentParser(description=__doc__.split("\n")[0])
p.add_argument("lists", nargs="+")
p.add_argument("--out")
a = p.parse_args()
merged, seen = [], set()
for path in a.lists:
recs = read(path)
print("%s: %s" % (path, describe(recs)))
for r in recs:
if r not in seen:
seen.add(r)
merged.append(r)
if a.out:
with open(a.out, "wb") as f:
f.write(MAGIC + struct.pack("<I", SIZE) + b"".join(merged))
print("%s: %s" % (a.out, describe(merged)))
if __name__ == "__main__":
sys.exit(main())

73
nx2/poll.py Normal file
View File

@@ -0,0 +1,73 @@
"""Interrupt polls in lifted code: `A64_POLL(c);` at every loop back-edge (right before each backward goto).
a64-kit's own runtime gives every guest thread a host thread, so the host OS preempts a guest spin loop. Inside
Eglantine guest threads are fibers multiplexed on the emulated cores: a lifted loop that never makes an SVC would
keep its core forever (MKW spins on GPU fences in memory and on CNTPCT, NX2_NOTES rows 35 and 127), and Eden's
preemption, which halts dynarmic at block boundaries, would never get in. So every backward branch target polls the
context's interrupt flag, the same points where dynarmic would see a halt request between blocks.
A64_POLL is defined by the generated guest model (rtgen.py): nothing unless A64_POLLING, else one relaxed load of
c->poll and, when it is set, a store of 0 to it and a call of a64_poll_cur() (the backend's, which hands the core on
and returns when the thread runs again).
Where the poll sits and what it is handed decide whether the compiler can keep a loop's guest registers in host
registers (measured on a lifted-shaped loop, clang 24, -O1 and -O2; nx2/rtgen.py has the macro):
- a call handed the context (a64_poll(c)) may change any of it, so every loop body reloaded and stored its registers;
- a call handed nothing cannot touch the noescape context, but at the loop header it still blocked the promotion: C
cannot say the call returns (LLVM's willreturn), so the body's stores after it were not guaranteed to execute;
- at the back-edge, after the body's stores, with the call handed nothing: the registers and the invariant loads
stay in host registers across the loop (4 loads an iteration gone in the sample; the two stores stay).
So the poll goes before each backward goto. One poll an iteration either way. a64_poll_cur reads no registers, so
polls work with A64_LOCAL_REGS too. The emulator-hosted DLL build keeps a64_poll(c), whose host's poll takes the
context (A64_POLL_CTX, build.py).
install() wraps Lifter.emit_function; functions with no loop are unchanged.
"""
import re
import a64_lift
LABEL = re.compile(r"^L_([0-9a-f]{8}):;", re.M)
GOTO = re.compile(r"goto L_([0-9a-f]{8});")
def add_polls(text):
"""A64_POLL before every backward goto (the loop's back-edge), as a block: `{ A64_POLL(c); goto L_x; }` is a
statement wherever the goto was. Returns the text and the number of polls added."""
labels = {m.group(1): m.end() for m in LABEL.finditer(text)}
out = []
last = n = 0
for m in GOTO.finditer(text):
at = labels.get(m.group(1))
if at is None or at > m.start():
continue
out.append(text[last:m.start()])
out.append("{ A64_POLL(c); " + m.group(0) + " }")
last = m.end()
n += 1
if not n:
return text, 0
out.append(text[last:])
return "".join(out), n
_installed = False
stats = {"functions": 0, "polls": 0}
def install():
global _installed
if _installed:
return
_installed = True
base = a64_lift.Lifter.emit_function
def emit_function(self, e, texts):
text, refs = base(self, e, texts)
text, n = add_polls(text)
if n:
stats["functions"] += 1
stats["polls"] += n
return text, refs
a64_lift.Lifter.emit_function = emit_function

963
nx2/prompt_assets.py Normal file
View File

@@ -0,0 +1,963 @@
"""(hookshot) The button prompts of other controllers: the game's button-icon font redrawn per controller face, from the user's RomFS
and Kenney's Input Prompts (CC0, third_party/kenney_input).
python -I nx2/prompt_assets.py --romfs <dir> --out <game folder>/prompts
Writes <out>/<face>/External.bfotf for each face (switch, xbox, ps4, ps5, steamdeck). Every text in the game draws its button icons
from Font2/Scalable/External.*.bfotf (dialogue, the control hints, menus): A B X Y at U+E040..E043, L R ZL ZR at E044..E047, Plus
and Minus at E04B and E04C, the stick presses at E056 and E057, the d-pad at E060..E063. Each written font has one icon per pad
input at U+F700 + HOS_PAD_* (gpu/vk/vk_api.h), and each of those Switch buttons' code points in a cmap segment of its own: the
game (host/prompts.c) points each one at the icon of the pad input bound to that button when it serves the font, so the icons
follow the controls, changed ones too.
The textures with button icons (the ocarina's notes, the letters on black, the controller in the pause menu) the same way, with
Pillow, astc_encoder and texture2ddecoder: <out>/Common.bntx is the game's UI/Layout2Archive/Common bntx decompressed, and
<face>/textures.bin the encoded blocks to write over it for each Switch button on each pad input, which the game picks by the
controls when it serves the file. The Talk ring's buttons and the ocarina hint's icons (the layouts of UI/Layout2Archive/Common,
<out>/Common.blarc) the same way: <out>/ring.bin and <face>/hint.bin say what to write over them for the controls.
Each face's images (the notes, the letters, the controller picture drawn by nx2/prompt_draw.py, the leader lines) are made as
layers first: --dump <dir> writes them as PNG, and a PNG of the same name in nx2/prompt_faces/<face>/ takes a layer's place, to
rework one by hand (nx2/prompt_faces/README.md).
The faces' fonts are larger than the game's, and the game drops a file larger than its entry in the resource size table: <out>/
rsizetable.zs is the game's table with the font's entry raised to fit the largest, served in its place.
A .bfotf is an OpenType font behind an 8-byte header and XORed per little-endian word: the header is the magic 0x18029a7f and the
font's size (big-endian), each XORed with the same key as the font. The .zs files are zstd frames made with the dictionary zs.zsdic
of Pack/ZsDic.pack.zs.
"""
import argparse
import io
import os
import struct
import sys
HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE) # (python -I leaves the script folder off)
from prompt_faces import FACES, override, parts # noqa: E402 (the faces: their icons, drawings and their parts)
KENNEY = os.path.join(os.path.dirname(HERE), "third_party", "kenney_input")
FONT = "Font2/Scalable/External.Nin_Ounce_NVN2.bfotf.zs"
MAGIC = 0x18029A7F
PAD_BASE = 0xF700
WIDE = 1.6 # (how much wider than the game's icon a wide Kenney icon may be)
# the HOS_PAD_* inputs (gpu/vk/vk_api.h), in order
PADS = ["south", "east", "west", "north", "back", "guide", "start", "lstick", "rstick", "lshoulder", "rshoulder", "up", "down", "left",
"right", "ltrigger", "rtrigger"]
# the Switch buttons' code points, in HOS_SW_* order (A B X Y LS RS L R ZL ZR Plus Minus Left Up Right Down), and hookshot's
# default controls for them (host/nn_hid.c pad_default: by position, A the right face button)
SWITCH = [0xE040, 0xE041, 0xE042, 0xE043, 0xE056, 0xE057, 0xE044, 0xE045, 0xE046, 0xE047, 0xE04B, 0xE04C, 0xE062, 0xE060, 0xE063,
0xE061]
DEFAULT = ["east", "south", "north", "west", "lstick", "rstick", "lshoulder", "rshoulder", "ltrigger", "rtrigger", "start", "back",
"left", "up", "right", "down"]
# the game's own icon for each pad input: what the Switch face shows
SWITCH_ICON = {"south": 0xE041, "east": 0xE040, "west": 0xE043, "north": 0xE042, "back": 0xE04C, "guide": 0xE04A, "start": 0xE04B,
"lstick": 0xE056, "rstick": 0xE057, "lshoulder": 0xE044, "rshoulder": 0xE045, "up": 0xE060, "down": 0xE061,
"left": 0xE062, "right": 0xE063, "ltrigger": 0xE046, "rtrigger": 0xE047}
# the icons that do not follow the controls, by the Kenney glyph's name after the face's prefix: the whole d-pad (the sticks keep the
# game's icons, an L or R cap with its arrows: right for any controller, where Kenney's are drawn in perspective)
STATIC = {0xE049: "dpad", 0xE064: "dpad_vertical_outline", 0xE065: "dpad_horizontal_outline"}
def zs_dictionary(romfs):
"""zs.zsdic, out of Pack/ZsDic.pack.zs (a SARC, itself zstd without a dictionary)"""
import zstandard
with open(os.path.join(romfs, "Pack", "ZsDic.pack.zs"), "rb") as f:
b = zstandard.ZstdDecompressor().decompressobj().decompress(f.read())
data, = struct.unpack_from("<I", b, 12)
hl, = struct.unpack_from("<H", b, 4)
n, = struct.unpack_from("<H", b, hl + 6)
names = hl + 12 + 16 * n + 8
for i in range(n):
attr, start, end = struct.unpack_from("<III", b, hl + 12 + 16 * i + 4)
o = names + (attr & 0xFFFFFF) * 4
if b[o:b.index(b"\0", o)] == b"zs.zsdic":
return b[data + start:data + end]
raise SystemExit("prompts: no zs.zsdic in Pack/ZsDic.pack.zs")
def read_zs(romfs, rel, dictionary):
import zstandard
d = zstandard.ZstdDecompressor(dict_data=zstandard.ZstdCompressionDict(dictionary))
with open(os.path.join(romfs, rel), "rb") as f:
return d.decompressobj().decompress(f.read())
def size_table(romfs, dictionary, game_size, font_size):
"""the game's resource size table (System/Resource/ResourceSizeTable.*.rsizetable.zs: "RESTBL", u32 version, u32 name length,
u32 hashed entries, u32 named entries; the hashed ones u32 CRC-32 of the path, u32 size, by hash) with the External font's
entry raised from the game's font (game_size bytes) to fit one of font_size: the game drops a file larger than its entry. None
when it fits already"""
import zlib
folder = os.path.join(romfs, "System", "Resource")
name = next((n for n in sorted(os.listdir(folder)) if n.startswith("ResourceSizeTable.") and n.endswith(".rsizetable.zs")),
None)
if not name:
raise SystemExit("prompts: no System/Resource/ResourceSizeTable.*.rsizetable.zs")
b = bytearray(read_zs(romfs, f"System/Resource/{name}", dictionary))
if b[:6] != b"RESTBL":
raise SystemExit(f"prompts: {name} is not a RESTBL table")
hashed, = struct.unpack_from("<I", b, 14)
want = zlib.crc32(FONT[:-3].encode())
for i in range(hashed):
at = 22 + 8 * i
h, size = struct.unpack_from("<II", b, at)
if h == want:
fit = font_size + size - game_size # (the entry's allowance over the file kept: 288 bytes)
if fit <= size:
return None
struct.pack_into("<I", b, at + 4, fit)
return bytes(b)
raise SystemExit(f"prompts: {name} has no entry for {FONT[:-3]}")
def unwrap(blob):
key = struct.unpack_from("<I", blob, 0)[0] ^ MAGIC
size = struct.unpack(">I", struct.pack("<I", struct.unpack_from("<I", blob, 4)[0] ^ key))[0]
body = b"".join(struct.pack("<I", w ^ key) for w in struct.unpack_from(f"<{(len(blob) - 8) // 4}I", blob, 8))
return key, body[:size]
def wrap(key, otf):
otf += b"\0" * (-len(otf) % 4)
head = struct.pack("<II", MAGIC ^ key, struct.unpack("<I", struct.pack(">I", len(otf)))[0] ^ key)
return head + b"".join(struct.pack("<I", w ^ key) for w in struct.unpack(f"<{len(otf) // 4}I", otf))
def cmap4(mapping, single):
"""a cmap table of one format 4 subtable (for Unicode BMP, platforms 0 and 3) from {code: glyph id}: runs of codes with
consecutive glyphs share a segment, and each code of `single` has one of its own; every segment by idDelta alone"""
segs = []
for c in sorted(mapping):
g = mapping[c]
if segs and c not in single and segs[-1][1] == c - 1 and segs[-1][1] not in single and \
(segs[-1][2] + c) & 0xFFFF == g:
segs[-1][1] = c
else:
segs.append([c, c, (g - c) & 0xFFFF])
segs.append([0xFFFF, 0xFFFF, 1])
n = len(segs)
search = 2 * (1 << (n.bit_length() - 1))
sub = struct.pack(">HHHHHHH", 4, 16 + 8 * n, 0, 2 * n, search, (search // 2).bit_length() - 1, 2 * n - search)
sub += b"".join(struct.pack(">H", s[1]) for s in segs) + b"\0\0"
sub += b"".join(struct.pack(">H", s[0]) for s in segs)
sub += b"".join(struct.pack(">H", s[2]) for s in segs)
sub += b"\0\0" * n
return struct.pack(">HHHHIHHI", 0, 2, 0, 3, 20, 3, 1, 20) + sub
def face_font(otf, face):
"""the game's font (decrypted) with this face's icons: a pad icon per input at PAD_BASE + its number, and the Switch buttons'
code points on the default controls' icons"""
from fontTools.pens.boundsPen import BoundsPen
from fontTools.pens.t2CharStringPen import T2CharStringPen
from fontTools.pens.transformPen import TransformPen
from fontTools.ttLib import TTFont
from fontTools.ttLib.tables.DefaultTable import DefaultTable
game = TTFont(io.BytesIO(otf))
gset, cmap = game.getGlyphSet(), dict(game.getBestCmap())
top = game["CFF "].cff.topDictIndex[0]
chars = top.CharStrings
order = list(game.getGlyphOrder())
def bounds(glyphset, name):
p = BoundsPen(glyphset)
glyphset[name].draw(p)
return p.bounds
def add(name, charstring, width):
chars.charStringsIndex.append(charstring)
chars.charStrings[name] = len(chars.charStringsIndex) - 1
top.charset.append(name)
order.append(name)
game["hmtx"].metrics[name] = (width, 0)
if face == "switch":
icon = {p: cmap[SWITCH_ICON[p]] for p in PADS}
for i, p in enumerate(PADS):
src = chars[icon[p]]
src.decompile()
add(f"pad_{p}", src, game["hmtx"][icon[p]][0])
else:
f = FACES[face]
prefix, names = f["prefix"], f["icons"]
kenney = TTFont(os.path.join(KENNEY, f["font"]))
kset = kenney.getGlyphSet()
def kname(s):
return s[1:] if s.startswith("/") else prefix + s
def draw(target, kn):
"""the Kenney glyph scaled into the box of the game's glyph `target` (as a square around its centre); a wide one (the
Menu and View pills on the square Plus and Minus) as high as the box, up to WIDE times as wide, the glyph widened"""
gb, kb = bounds(gset, target), bounds(kset, kn)
side = max(gb[2] - gb[0], gb[3] - gb[1])
s = min(side * WIDE / (kb[2] - kb[0]), side / (kb[3] - kb[1]))
over = max(0, s * (kb[2] - kb[0]) - side)
dx = (gb[0] + gb[2] + over) / 2 - s * (kb[0] + kb[2]) / 2
dy = (gb[1] + gb[3]) / 2 - s * (kb[1] + kb[3]) / 2
width = game["hmtx"][target][0] + round(over)
pen = T2CharStringPen(width, gset)
kset[kn].draw(TransformPen(pen, (s, 0, 0, s, dx, dy)))
cs = chars[target]
return pen.getCharString(private=cs.private, globalSubrs=cs.globalSubrs), width
for p in PADS:
kn = kname(names[p]) if p in names else None
if kn not in kset: # (no such button on this face, the PlayStation's PS button: the game's own icon)
kn = None
target = cmap[SWITCH_ICON[p]]
if kn:
add(f"pad_{p}", *draw(target, kn))
else:
src = chars[target]
src.decompile()
add(f"pad_{p}", src, game["hmtx"][target][0])
for c, s in STATIC.items():
kn = prefix + s
if c in cmap and kn in kset:
cs, _ = draw(cmap[c], kn)
chars[cmap[c]] = cs
game.setGlyphOrder(order)
gid = {n: i for i, n in enumerate(order)}
mapping = {c: gid[n] for c, n in cmap.items() if c <= 0xFFFF}
for i, p in enumerate(PADS):
mapping[PAD_BASE + i] = gid[f"pad_{p}"]
for c, p in zip(SWITCH, DEFAULT):
mapping[c] = gid[f"pad_{p}"]
t = DefaultTable("cmap")
t.data = cmap4(mapping, set(SWITCH))
game["cmap"] = t
out = io.BytesIO()
game.save(out)
return out.getvalue()
# ---- the textures with button icons (UI/Layout2Archive/Common.*.bntx: 878 textures, ASTC 4x4 and BC4, the Switch 2's block-linear
# layout). prompts/Common.bntx is the game's file, decompressed; prompts/<face>/textures.bin the blocks to write over it, for each
# Switch button and each pad input it can be bound to (host/prompts.c writes those of the controls in use):
# "HPTX", u32 count, then count times: u8 Switch button (HOS_SW_*), u8 pad input (HOS_PAD_*), u16 0, u32 runs,
# runs times: u32 offset in the file, u32 length, the bytes
# Each face's images are made first, as layers (face_layers), then encoded: --dump writes the layers as PNG, and a PNG of the same
# name in nx2/prompt_faces/<face>/ takes a layer's place (prompt_faces.override). A layer's name: the texture's, then for a Switch
# button's image on a pad input ".<pad input>" (the notes, the letters; the leader lines ".<Switch button>.<pad input>": only the
# line, drawn over the texture's lines that stay), ".png".
TEXTURES = "UI/Layout2Archive/Common.Product.100.Nin_Ounce_NVN2.bntx.zs"
SW = {n: i for i, n in enumerate(["a", "b", "x", "y", "ls", "rs", "l", "r", "zl", "zr", "plus", "minus", "left", "up", "right", "down"])}
# the ocarina's notes (white icons, dark letters) and the white letters on black (KeyTexFont, BC4)
NOTES = {"OcarinaButtonA_00": "a", "OcarinaButtonB_00": "b", "OcarinaButtonX_00": "x", "OcarinaButtonY_00": "y",
"OcarinaButtonL_00": "l", "OcarinaButtonR_00": "r"}
LETTERS = {"KeyTexFontA_00": "a", "KeyTexFontB_00": "b", "KeyTexFontX_00": "x", "KeyTexFontY_00": "y"}
# the Pro Controller in the pause menu's controls page (System): the picture, the page's copy of it (tinted: PICTURE_TINT) and its
# shadow (PICTURE_SHADOW); a face's own picture (nx2/prompt_faces.py) takes their place, else the face buttons' discs (centre,
# radius in pixels) by the pad input there get the face's labels (a picture of the controller: its own labels, whatever the controls)
PICTURES = ("Controller_Pro", "Controller_Pro1")
PICTURE_TINT, PICTURE_SHADOW = "Controller_Pro1", "ControllerSh_Pro1"
PICTURE_BUTTONS = {"east": (869, 227, 31), "south": (800, 286, 31), "north": (800, 168, 31), "west": (732, 227, 31)}
# the page's leader lines (BC4, white on black, drawn over the picture: GUIDE_SCALE and GUIDE_AT take its pixels to theirs), from
# the labels' bars (BARS: the left ones' right edge, the right ones' left edge, kept from the game's) to the picture's parts
# (nx2/prompt_faces.py): each line's texture, bar (0 left, 1 right), row (its top; 6 pixels thick) and end: a Switch button (the
# part of the pad input bound to it: one line per pad input, chosen by the controls; a face button or the d-pad: a frame round the
# group, as the game's round its face buttons, the label's icon telling which) or a pad input (fixed: the sticks)
GUIDES = ("ControllerGuide_Pro_00", "ControllerGuide_Pro_01")
GUIDE_SCALE, GUIDE_AT, BARS = 1.1136, (131.3, 462.6), (147, 1407)
GUIDE_LINES = [(0, 0, 333, "zl"), (1, 0, 561, "minus"), (0, 0, 667, "lstick"), (0, 1, 558, "plus"), (0, 1, 713, "a"),
(0, 1, 1225, "rstick")]
def bntx_textures(b):
"""{name: (BRTI offset, format, width, height, GOBs per block, data offset)} of a Switch 2 BNTX (platform "Ounc")"""
count, = struct.unpack_from("<I", b, 0x24)
info, = struct.unpack_from("<Q", b, 0x28)
out = {}
for i in range(count):
p, = struct.unpack_from("<Q", b, info + 8 * i)
no, = struct.unpack_from("<Q", b, p + 0x60)
nl, = struct.unpack_from("<H", b, no)
fmt, = struct.unpack_from("<I", b, p + 0x1C)
w, h = struct.unpack_from("<II", b, p + 0x24)
bh, = struct.unpack_from("<I", b, p + 0x34)
mips, = struct.unpack_from("<Q", b, p + 0x70)
data, = struct.unpack_from("<Q", b, mips)
out[b[no + 2:no + 2 + nl].decode()] = (p, fmt, w, h, 1 << bh, data)
return out
def gob_offset(x, y, gobs, pitch):
"""a byte's place in a block-linear texture (x in bytes, y in rows of blocks): the Switch 2's GOB, the Tegra X1's with
offset bits 5 and 6 swapped (gpu/render/rx_nvn.cpp bl_offset)"""
a = (y // (8 * gobs)) * 512 * gobs * pitch + (x // 64) * 512 * gobs + (y % (8 * gobs) // 8) * 512
return a + (((x & 63) >> 5) << 8 | ((y & 7) >> 2) << 7 | ((x & 31) >> 4) << 6 | ((y >> 1) & 1) << 5 | (y & 1) << 4 | (x & 15))
def decode(b, t):
"""a texture as a PIL image (RGBA, or L for BC4)"""
import texture2ddecoder
from PIL import Image
_, fmt, w, h, gobs, data = t
bpe = 16 if fmt >> 8 == 0x2D else 8
bw, bh = (w + 3) // 4, (h + 3) // 4
pitch = (bw * bpe + 63) // 64
lin = b"".join(b[data + o:data + o + bpe] for o in (gob_offset(x * bpe, y, gobs, pitch) for y in range(bh) for x in range(bw)))
if bpe == 16:
return Image.frombytes("RGBA", (bw * 4, bh * 4), texture2ddecoder.decode_astc(lin, bw * 4, bh * 4, 4, 4), "raw",
"BGRA").crop((0, 0, w, h))
return Image.frombytes("L", (bw * 4, bh * 4), lin, "bcn", 4).crop((0, 0, w, h))
def encode_bc4(im):
"""BC4 by each block's darkest and lightest, the 6 levels between"""
import numpy as np
v = np.asarray(im, dtype=np.int32)
v = v.reshape(im.height // 4, 4, im.width // 4, 4).transpose(0, 2, 1, 3).reshape(-1, 16)
hi, lo = v.max(1), v.min(1)
span = np.maximum(hi - lo, 1)[:, None]
t = np.where((hi > lo)[:, None], np.rint((v - lo[:, None]) * 7 / span), 7).astype(np.uint64)
code = np.where(t == 7, 0, np.where(t == 0, 1, 8 - t)).astype(np.uint64) # (0 the lightest, 1 the darkest, 2..7 between)
idx = (code << (np.arange(16, dtype=np.uint64) * 3)).sum(1, dtype=np.uint64)
out = np.zeros((len(v), 8), np.uint8)
out[:, 0], out[:, 1] = hi, lo
out[:, 2:] = idx.astype("<u8").view(np.uint8).reshape(-1, 8)[:, :6]
return out.tobytes()
def encode_astc(im, srgb):
import astc_encoder as A
ctx = A.ASTCContext(A.ASTCConfig(A.ASTCProfile.LDR_SRGB if srgb else A.ASTCProfile.LDR, 4, 4, 1, A.ASTCQualityPreset.THOROUGH))
return ctx.compress(A.ASTCImage(A.ASTCType.U8, im.width, im.height, 1, im.tobytes()), A.ASTCSwizzle.from_str("RGBA"))
def runs(t, im, box=None, mask=None):
"""the file's bytes for the blocks of im (the whole texture's size) inside box (pixels; all of it by default), or those with
any of mask (L, im's size) in them, as (offset, bytes) runs"""
from PIL import Image
_, fmt, w, h, gobs, data = t
bpe = 16 if fmt >> 8 == 0x2D else 8
bw, bh = (w + 3) // 4, (h + 3) // 4
x0, y0, x1, y1 = box or (mask and mask.getbbox()) or (0, 0, w, h)
bx0, by0, bx1, by1 = max(0, x0 // 4), max(0, y0 // 4), min(bw, (x1 + 3) // 4), min(bh, (y1 + 3) // 4)
part = Image.new(im.mode, (bw * 4, bh * 4)) # (only the box's blocks are encoded)
part.paste(im, (0, 0))
part = part.crop((bx0 * 4, by0 * 4, bx1 * 4, by1 * 4))
blocks = encode_astc(part.convert("RGBA"), (fmt & 0xFF) == 6) if bpe == 16 else encode_bc4(part.convert("L"))
pw = bx1 - bx0
pitch = (bw * bpe + 63) // 64
if mask: # (the blocks with any of the mask)
import numpy as np
m = np.zeros((bh * 4, bw * 4), bool)
m[:h, :w] = np.asarray(mask) > 0
cover = m.reshape(bh, 4, bw, 4).any((1, 3))
pieces = sorted((data + gob_offset(x * bpe, y, gobs, pitch),
blocks[((y - by0) * pw + x - bx0) * bpe:((y - by0) * pw + x - bx0 + 1) * bpe])
for y in range(by0, by1) for x in range(bx0, bx1) if not mask or cover[y, x])
out = []
for off, piece in pieces: # (blocks next to each other in the file: one run)
if out and out[-1][0] + len(out[-1][1]) == off:
out[-1][1] += piece
else:
out.append([off, bytearray(piece)])
return out
class Icons:
"""a face's icon for each pad input, drawn from a font (the game's own for the Switch face, else Kenney's)"""
def __init__(self, face, otf):
from fontTools.ttLib import TTFont
if face == "switch":
self.path, self.code = io.BytesIO(otf), {p: SWITCH_ICON[p] for p in PADS}
else:
f = FACES[face]
prefix, names = f["prefix"], f["icons"]
self.path = os.path.join(KENNEY, f["font"])
rev = {n: c for c, n in TTFont(self.path).getBestCmap().items()}
self.code = {}
for p in PADS:
n = names.get(p)
n = n and (n[1:] if n.startswith("/") else prefix + n)
if n in rev:
self.code[p] = rev[n]
def masks(self, pad, size):
"""(the icon, its silhouette with the holes filled, the holes) as L masks of size x size, the icon centred in it"""
from PIL import Image, ImageChops, ImageDraw, ImageFilter, ImageFont
if isinstance(self.path, io.BytesIO):
self.path.seek(0)
f = ImageFont.truetype(self.path, size * 4)
g = Image.new("L", (size * 8, size * 8), 0)
ImageDraw.Draw(g).text((size * 2, size * 2), chr(self.code[pad]), font=f, fill=255)
g = g.crop(g.getbbox())
s = size / max(g.size)
g = g.resize((max(1, round(g.width * s)), max(1, round(g.height * s))), Image.LANCZOS)
icon = Image.new("L", (size, size), 0)
icon.paste(g, ((size - g.width) // 2, (size - g.height) // 2))
outside = Image.new("L", (size + 4, size + 4), 255) # (a margin: every side's outside joins the corner's)
outside.paste(ImageChops.invert(icon.point(lambda v: 255 if v > 127 else 0)), (2, 2))
ImageDraw.floodfill(outside, (0, 0), 128) # (what the corner reaches: the outside)
shape = outside.crop((2, 2, size + 2, size + 2)).point(lambda v: 0 if v == 128 else 255)
inner = shape.filter(ImageFilter.MinFilter(5)) # (the soft rim of the icon is not a hole)
holes = ImageChops.multiply(inner, ImageChops.invert(icon))
return icon, shape, holes
class Layer:
"""an image of a texture's: written whole, or its box (pixels) of it, or (over: a layer's name) drawn over that layer where
it has any of itself; for a Switch button (0xff: always) on a pad input"""
def __init__(self, name, texture, sw, pad, im, box=None, over=None):
self.name, self.texture, self.sw, self.pad, self.im, self.box, self.over = name, texture, sw, pad, im, box, over
def face_layers(b, tex, face, otf):
"""this face's images, as layers (in textures.bin's order), each a hand-made one if nx2/prompt_faces/<face>/ has it"""
from PIL import Image, ImageDraw
import prompt_draw
icons = Icons(face, otf)
layers = []
def keep(sw, pad): # (the Switch face's default controls: the game's own textures)
return pad in icons.code and not (face == "switch" and DEFAULT[SW[sw]] == pad)
for name, sw in NOTES.items():
t = tex[name]
w, h = t[2], t[3]
for pad in PADS:
if not keep(sw, pad):
continue
icon, shape, holes = icons.masks(pad, int(min(w, h) * 0.9))
im = Image.new("RGBA", (w, h), (0, 0, 0, 0))
o = ((w - icon.width) // 2, (h - icon.height) // 2)
im.paste(Image.new("RGBA", icon.size, (255, 255, 255, 255)), o, shape)
im.paste(Image.new("RGBA", icon.size, (0, 0, 0, 255)), o, holes)
layers.append(Layer(f"{name}.{pad}.png", name, SW[sw], PADS.index(pad), im))
for name, sw in LETTERS.items():
t = tex[name]
w, h = t[2], t[3]
for pad in PADS:
if not keep(sw, pad):
continue
icon, shape, holes = icons.masks(pad, int(min(w, h) * 0.9))
letter = holes if holes.getbbox() else icon # (an icon with no letter in it: its shape)
im = Image.new("L", (w, h), 0)
im.paste(letter, ((w - letter.width) // 2, (h - letter.height) // 2))
layers.append(Layer(f"{name}.{pad}.png", name, SW[sw], PADS.index(pad), im))
own = None
if face != "switch":
hand = override(face, PICTURES[0] + ".png")
own = Image.open(hand).convert("RGBA") if hand else prompt_draw.picture(
face, decode(b, tex[PICTURES[0]]).convert("RGBA"), Icons("switch", otf))
if own:
layers += picture_layers(b, tex, own)
for name in PICTURES if face != "switch" and not own else ():
t = tex[name]
im = decode(b, t)
px = im.load()
for pad, (cx, cy, r) in PICTURE_BUTTONS.items():
fill, ink = px[cx + r - 6, cy], (205, 205, 205, 255)
ImageDraw.Draw(im).ellipse((cx - r, cy - r, cx + r, cy + r), fill=fill)
size = int(2 * r * 0.92)
icon, shape, holes = icons.masks(pad, size)
disc = sum(i * n for i, n in enumerate(shape.histogram())) / 255 / size / size > 0.7 # (a disc with a letter in it: the letter)
mark = holes if disc else icons.masks(pad, int(size * 0.7))[0]
im.paste(Image.new("RGBA", mark.size, ink), (cx - mark.width // 2, cy - mark.height // 2), mark)
xs, ys = [c[0] for c in PICTURE_BUTTONS.values()], [c[1] for c in PICTURE_BUTTONS.values()]
r = max(c[2] for c in PICTURE_BUTTONS.values())
layers.append(Layer(name + ".png", name, 0xFF, 0, im, (min(xs) - r, min(ys) - r, max(xs) + r + 1, max(ys) + r + 1)))
layers += guide_layers_of(b, tex, face)
for layer in layers:
hand = override(face, layer.name)
if hand:
im = Image.open(hand).convert(layer.im.mode)
if im.size != layer.im.size:
raise SystemExit(f"prompts: {hand} is {im.size[0]}x{im.size[1]}, the texture {layer.im.size[0]}x{layer.im.size[1]}")
layer.im = im
return layers
def face_textures(b, tex, layers):
"""textures.bin's entries for these layers: [(Switch button, pad input, runs)], Switch button 0xff for those always written"""
from PIL import ImageChops
entries, made = [], {}
for layer in layers:
t, im = tex[layer.texture], layer.im
made[layer.name] = im
if layer.over:
entries.append((layer.sw, layer.pad, runs(t, ImageChops.lighter(made[layer.over], im), mask=im)))
elif layer.box:
x0, y0, x1, y1 = layer.box
if im.getbbox(): # (a hand-made image may reach further)
bx0, by0, bx1, by1 = im.getbbox()
x0, y0, x1, y1 = min(x0, bx0), min(y0, by0), max(x1, bx1), max(y1, by1)
entries.append((layer.sw, layer.pad, runs(t, im, (x0, y0, x1, y1))))
else:
entries.append((layer.sw, layer.pad, runs(t, im)))
return entries
# the groups a line frames, ending on the frame, when its pad input is one of them; the face buttons' labels (Switch A's line, under
# one bar with B, X and Y's) always frame one: the d-pad when A is on it, else the face buttons
GUIDE_GROUPS = (("north", "west", "east", "south"), ("up", "left", "right", "down"))
GUIDE_FACE = "a"
# the lines' router: a grid of GUIDE_GRID pixels; a bend costs GUIDE_BEND pixels of length, crossing a line that stays GUIDE_CROSS,
# coming to a part other than from above GUIDE_SIDE; a line keeps GUIDE_CLEAR pixels off the other parts (the sticks GUIDE_CLEAR / 2)
GUIDE_GRID, GUIDE_BEND, GUIDE_CROSS, GUIDE_SIDE, GUIDE_CLEAR = 8, 64, 40, 48, 16
def guide_at(part):
"""a part (picture pixels: x, y, r) in the lines' pixels"""
return part[0] * GUIDE_SCALE + GUIDE_AT[0], part[1] * GUIDE_SCALE + GUIDE_AT[1], part[2] * GUIDE_SCALE
def guide_frame_at(where, group):
"""a group's frame: its centre and its corners' distance from it (the lines' pixels)"""
pts = [guide_at(where[p]) for p in group]
cx, cy = sum(p[0] for p in pts) / 4, sum(p[1] for p in pts) / 4
return cx, cy, max(max(abs(x - cx), abs(y - cy)) + r for x, y, r in pts) + 8 # (from the centre to a corner, as the game's)
def guide_route(size, bar, row, where, pad, stays, frame=None):
"""the way (points, the lines' pixels) of a line from a bar's row to the pad input's part, round the picture's other parts and
with the fewest bends, crossing the lines that stay (L) as little as it can: it comes to a stick's centre, to a part just off
its edge (above it best), or to a frame's corner (frame: centre, corners' distance)"""
import heapq
import math
import numpy as np
from PIL import ImageFilter
g = GUIDE_GRID
x0, y0 = BARS[0] if bar == 0 else BARS[1], row + 3
ilo, ihi = (0, (BARS[1] - x0) // g) if bar == 0 else (-((x0 - BARS[0]) // g), 0)
jlo, jhi = -((y0 - 250) // g), (min(size[1] - 20, 1420) - y0) // g
xs, ys = x0 + g * np.arange(ilo, ihi + 1), y0 + g * np.arange(jlo, jhi + 1)
X, Y = np.meshgrid(xs, ys, indexing="ij")
blocked = np.zeros(X.shape, bool)
own = set(next((grp for grp in GUIDE_GROUPS if pad in grp), ())) if frame else {pad}
for p, part in where.items():
if len(part) == 4: # (a box to keep off)
(bx0, by0, _), (bx1, by1, _) = guide_at(part[:2] + (0,)), guide_at(part[2:] + (0,))
m = GUIDE_CLEAR / 2
blocked |= (X > bx0 - m) & (X < bx1 + m) & (Y > by0 - m) & (Y < by1 + m)
continue
x, y, r = guide_at(part)
if p in own:
continue
clear = GUIDE_CLEAR / 2 if p in ("lstick", "rstick") else GUIDE_CLEAR
blocked |= (X - x) ** 2 + (Y - y) ** 2 < (r + clear) ** 2
for grp in GUIDE_GROUPS: # (a group: inside its frame, which another line may draw, its own inside it)
if all(p in where for p in grp):
cx, cy, e = guide_frame_at(where, grp)
if frame and pad in grp:
blocked |= (X - cx) ** 2 + (Y - cy) ** 2 < (e - 8) ** 2
else: # (the frame: a square on its corner, its sides 1.15 corners' distance off in x + y)
blocked |= np.abs(X - cx) + np.abs(Y - cy) < 1.15 * e + GUIDE_CLEAR
blocked |= ((np.abs(X - BARS[0]) < 14) | (np.abs(X - BARS[1]) < 14)) & (Y != y0) # (off the bars but on its own row)
st = np.asarray(stays.filter(ImageFilter.MaxFilter(9)))
pen = np.where(st[np.clip(Y, 0, size[1] - 1), np.clip(X, 0, size[0] - 1)] > 0, GUIDE_CROSS, 0)
dirs = ((1, 0), (-1, 0), (0, 1), (0, -1))
goals = {} # ((i, j, dir): (the end, extra cost)): the cell two before the end, coming the way it ends
def goal(ex, ey, d, extra):
i, j = round((ex - x0) / g) - 2 * dirs[d][0], round((ey - y0) / g) - 2 * dirs[d][1]
cells = [(i + k * dirs[d][0], j + k * dirs[d][1]) for k in range(3)]
if all(ilo <= ci <= ihi and jlo <= cj <= jhi and not blocked[ci - ilo, cj - jlo] for ci, cj in cells[:2]):
goals[(i, j, d)] = ((ex, ey), extra)
x, y, r = guide_at(where[pad])
if frame: # (a corner, or beside it on the grid's nearest line, along the frame's side)
cx, cy, e = frame
for k in (0, 1):
sy, sx = y0 + g * (math.floor((cy - y0) / g) + k) - cy, x0 + g * (math.floor((cx - x0) / g) + k) - cx
for (ex, ey), d, extra in (((cx + e - abs(sy), cy + sy), 1, 0), ((cx + sx, cy - e + abs(sx)), 2, GUIDE_SIDE),
((cx - e + abs(sy), cy + sy), 0, GUIDE_SIDE),
((cx + sx, cy + e - abs(sx)), 3, GUIDE_SIDE)):
goal(ex, ey, d, extra)
elif pad in ("lstick", "rstick"):
for d in range(4):
goal(x, y, d, 0)
else:
for (ex, ey), d, extra in (((x, y - r - 13), 2, 0), ((x, y + r + 13), 3, GUIDE_SIDE), ((x - r - 13, y), 0, GUIDE_SIDE),
((x + r + 13, y), 1, GUIDE_SIDE)):
if r or d >= 2:
goal(ex, ey, d, extra)
start = (0, 0, 0 if bar == 0 else 1)
best, came, heap = {start: 0}, {}, [(0, start)]
while heap:
c, (i, j, d) = heapq.heappop(heap)
if c > best.get((i, j, d), 1e18):
continue
if (i, j, d) in goals:
(ex, ey), _ = goals[(i, j, d)]
way, k = [], (i, j, d)
while k:
way.append(k)
k = came.get(k)
way.reverse()
pts = [(x0 + g * way[0][0], y0 + g * way[0][1])]
for p0, p1 in zip(way, way[1:]):
if p0[2] != p1[2]:
pts.append((x0 + g * p0[0], y0 + g * p0[1]))
pts.append((ex, ey) if len(pts) > 1 or abs(ey - y0) > g else (ex, y0)) # (one leg: on its row)
if len(pts) > 2: # (the last leg's corner moved onto the end's line: its leg before only longer or shorter)
cx_, cy_ = pts[-2]
pts[-2] = (cx_, ey) if dirs[d][0] else (ex, cy_)
return _square(pts)
for nd in range(4):
if nd == d ^ 1 or (nd != d and j == 0 and abs(i) < 5): # (no turning back; the label's stub first)
continue
ni, nj = i + dirs[nd][0], j + dirs[nd][1]
if not (ilo <= ni <= ihi and jlo <= nj <= jhi) or blocked[ni - ilo, nj - jlo]:
continue
nc = c + g + pen[ni - ilo, nj - jlo] + (GUIDE_BEND if nd != d else 0) + goals.get((ni, nj, nd), (0, 0))[1]
if nc < best.get((ni, nj, nd), 1e18):
best[(ni, nj, nd)], came[(ni, nj, nd)] = nc, (i, j, d)
heapq.heappush(heap, (nc, (ni, nj, nd)))
return None
def _square(pts):
"""a way's points with every leg level or upright (each point shares x or y with the one before)"""
out = [pts[0]]
for x, y in pts[1:]:
px, py = out[-1]
if (x, y) == (px, py):
continue
if x != px and y != py:
out.append((px, y) if len(out) > 1 and out[-2][0] == px else (x, py))
out.append((x, y))
return out
def guide_draw(size, way, dot):
"""a line along its way (6 pixels thick, square corners), ending on a dot"""
from PIL import Image, ImageDraw
im = Image.new("L", size, 0)
d = ImageDraw.Draw(im)
for (x0, y0), (x1, y1) in zip(way, way[1:]):
x0, x1, y0, y1 = round(min(x0, x1)), round(max(x0, x1)), round(min(y0, y1)), round(max(y0, y1))
d.rectangle((x0 - 3, y0 - 3, x1 + 2, y1 + 2), fill=255)
if dot:
d.regular_polygon((round(way[-1][0]), round(way[-1][1]), 11), 5, rotation=18, fill=255)
return im
def guide_line(size, bar, row, where, pad, stays):
"""a leader line (L, size) from a bar's row to the pad input's part (where: the picture's parts), ending on a dot"""
way = guide_route(size, bar, row, where, pad, stays)
if not way: # (no way round: straight along the row, then up or down)
x, y, r = guide_at(where[pad])
y += 0 if pad in ("lstick", "rstick") else -(r + 13) if row < y else r + 13
way = [(BARS[0] if bar == 0 else BARS[1], row + 3), (x, row + 3), (x, y)]
return guide_draw(size, way, True)
def guide_frame(size, bar, row, where, group, stays):
"""a group's frame (L, size; a square on its corner, rounded, round its parts) and its line from a bar's row"""
import math
from PIL import Image, ImageChops, ImageDraw
cx, cy, e = guide_frame_at(where, group)
ss, round_ = 4, 0.36 * e
a = (e + (math.sqrt(2) - 1) * round_) / math.sqrt(2)
n = int(2 * a * ss) + 8
sq = Image.new("L", (n, n), 0)
ImageDraw.Draw(sq).rounded_rectangle((4, 4, n - 4, n - 4), radius=round_ * ss, outline=255, width=6 * ss)
sq = sq.rotate(45, Image.BICUBIC, expand=True)
sq = sq.resize((sq.width // ss, sq.height // ss), Image.LANCZOS)
im = Image.new("L", size, 0)
im.paste(sq, (round(cx - sq.width / 2), round(cy - sq.height / 2)))
way = guide_route(size, bar, row, where, group[0], stays, (cx, cy, e - 3)) or \
[(BARS[0] if bar == 0 else BARS[1], row + 3), (cx + e - 3 if bar else cx - e + 3, row + 3)]
return ImageChops.lighter(im, guide_draw(size, way, False))
def guide_layers(game, face):
"""the page's leader lines for this face's picture, from the game's (L images, GUIDES): each texture's lines that stay (the
Switch face: the game's), and {texture: [(Switch button, {pad input: its line})]}"""
from PIL import Image, ImageChops, ImageFilter
where, size = parts(face), game[0].size
keep = [] # (the game's bars, or on the Switch face all the game's but the lines that move)
for im in game:
k = Image.new("L", size, 0)
if face == "switch":
k.paste(im)
else:
k.paste(im.crop((0, 0, BARS[0], size[1])), (0, 0))
k.paste(im.crop((BARS[1], 0, size[0], size[1])), (BARS[1], 0))
keep.append(k)
for tx, bar, row, end in GUIDE_LINES: # (the lines that stay first: those that move keep off them)
if end not in SW and face != "switch":
keep[tx] = ImageChops.lighter(keep[tx], guide_line(size, bar, row, where, end, keep[tx]))
moving = {}
for tx, bar, row, end in GUIDE_LINES:
if end not in SW:
continue
stays, frames = keep[tx], {}
for group in GUIDE_GROUPS:
if all(p in where for p in group):
im = guide_frame(size, bar, row, where, group, stays)
frames.update((p, im) for p in group)
lines = {pad: frames.get(pad) or (frames["south"] if end == GUIDE_FACE and "south" in frames else
guide_line(size, bar, row, where, pad, stays)) for pad in PADS if pad in where}
if face == "switch": # (the game's own line where the pad input's is hookshot's default's: taken out of what stays)
own = lines[DEFAULT[SW[end]]]
around = Image.new("L", size, 0)
grown = own.filter(ImageFilter.MaxFilter(25 if DEFAULT[SW[end]] in sum(GUIDE_GROUPS, ()) else 15))
around.paste(grown.crop((BARS[0] + 1, 0, BARS[1], size[1])), (BARS[0] + 1, 0))
game_own = ImageChops.multiply(game[tx], around)
for pad, im in lines.items():
if im is own:
lines[pad] = game_own
keep[tx] = ImageChops.subtract(keep[tx], around)
moving.setdefault(tx, []).append((end, lines))
return keep, moving
def guide_layers_of(b, tex, face):
"""the layers of the page's leader lines: each texture's lines that stay (0xff), then each Switch button's line for each pad
input, drawn over them (host/prompts.c writes the entries in order)"""
from PIL import Image, ImageChops
if not parts(face):
return []
game = [decode(b, tex[n]) for n in GUIDES]
keep, moving = guide_layers(game, face)
layers = []
for tx, name in enumerate(GUIDES):
box = Image.new("L", game[tx].size, 0) # (every place a line is or may be)
for im in [game[tx], keep[tx]] + [im for _, lines in moving.get(tx, []) for im in lines.values()]:
box = ImageChops.lighter(box, im)
layers.append(Layer(name + ".png", name, 0xFF, 0, keep[tx], box.getbbox()))
for end, lines in moving.get(tx, []):
for pad, im in lines.items():
layers.append(Layer(f"{name}.{end}.{pad}.png", name, SW[end], PADS.index(pad), im, over=name + ".png"))
return layers
def picture_layers(b, tex, im):
"""the layers putting a face's controller picture (1160x680 RGBA, drawn like Controller_Pro) in place of the Pro Controller:
the picture, the page's tinted copy (each channel a line of the grey level, fitted on the game's pair) and the shadow (the
outline's left part squeezed into ControllerSh_Pro1 and blurred, fitted on the game's)"""
import numpy as np
from PIL import Image, ImageFilter
plain, tinted = (np.asarray(decode(b, tex[n]).convert("RGBA")).astype(float) for n in (PICTURES[0], PICTURE_TINT))
solid = plain[..., 3] > 250
grey = plain[solid][:, :3].sum(1)
lines = [np.polyfit(grey, tinted[solid][:, c], 1) for c in range(3)]
px = np.asarray(im).astype(float)
level = px[..., :3].sum(2)
tint = np.dstack([np.clip(np.polyval(lines[c], level), 0, 255) for c in range(3)] + [px[..., 3]])
sh = tex[PICTURE_SHADOW]
shadow = im.getchannel("A").crop((40, 0, 480, 680)).resize((sh[2], sh[3])).filter(ImageFilter.GaussianBlur(24))
shadow = shadow.point(lambda v: round(v * 0.7))
return [Layer(PICTURES[0] + ".png", PICTURES[0], 0xFF, 0, im),
Layer(PICTURE_TINT + ".png", PICTURE_TINT, 0xFF, 0, Image.fromarray(tint.astype("uint8"))),
Layer(PICTURE_SHADOW + ".png", PICTURE_SHADOW, 0xFF, 0, shadow)]
def write_textures(path, entries):
with open(path, "wb") as f:
f.write(b"HPTX" + struct.pack("<I", len(entries)))
for sw, pad, rs in entries:
f.write(struct.pack("<BBHI", sw, pad, 0, len(rs)))
for off, data in rs:
f.write(struct.pack("<II", off, len(data)) + bytes(data))
# ---- the Talk ring (DoCommand_00 in UI/Layout2Archive/Common.*.blarc, a BFFH layout, four copies): one group per Switch button
# an action can be on, N_A/B/X/Y_00, each its base, its big button and the three others' letters (KeyTexFont) at the Switch's
# places. prompts/Common.blarc is the game's file decompressed, prompts/ring.bin what to write over it to put each one where its
# pad input is (host/prompts.c picks the places by the controls): "HPRG", u32 count, then count times: u8 group (HOS_SW_A..Y),
# u8 the Switch button placed (the group's own: its big button and base), u8 place (RING_PLACES), u8 0, u32 runs, the runs
# (u32 offset, u32 length, the bytes)
LAYOUTS = "UI/Layout2Archive/Common.Product.100.Nin_Ounce_NVN2.blarc.zs"
RING_PLACES = {"east": (80.0, 0.0), "south": (0.0, -80.0), "north": (0.0, 80.0), "west": (-80.0, 0.0)}
# each group's panes (in the layout's order): its base, the others' letters, its big button (then its five children); the group
# whose big button is at each place gives the base's look (CrossBase_A or _B, mirrored or not) and the button's banner
RING_GROUPS = {"a": (5, {"b": 6, "x": 7, "y": 8}, 9), "b": (16, {"a": 17, "x": 18, "y": 19}, 20),
"x": (27, {"a": 28, "b": 29, "y": 30}, 31), "y": (38, {"a": 39, "b": 40, "x": 41}, 42)}
RING_AT = {"east": "a", "south": "b", "north": "x", "west": "y"}
def layout_copies(blarc, name):
"""the (start, end) of each BFFH layout in the archive that is the layout `name`"""
starts, at = [], 0
while (at := blarc.find(b"BFFH", at)) >= 0:
starts.append(at)
at += 4
return [(a, z) for a, z in zip(starts, starts[1:] + [len(blarc)]) if blarc.find(b"\0" + name.encode() + b"\0RootPane\0", a, z) > 0]
def ring_entries(blarc):
"""ring.bin's entries: [(group, Switch button placed, place, runs)], or None when the layout is not the one known"""
def pane(k): # (a pane's record: its transform (rotation, scale, translation) first, then parent, size, material)
return 0x968 + k * 0x44 - 0x28
copies = [a for a, _ in layout_copies(blarc, "DoCommand_00")]
if not copies:
return None
lay = blarc[copies[0]:copies[0] + 0x2000]
if any(struct.unpack_from("<2f", lay, pane(base) + 0x2c) != (320.0, 320.0) for base, _, _ in RING_GROUPS.values()):
return None
out = []
for group, (base, small, btn) in RING_GROUPS.items():
for place, xy in RING_PLACES.items():
tb, _, tbtn = RING_GROUPS[RING_AT[place]]
parts = [(pane(base) + 0x10, lay[pane(tb) + 0x10:pane(tb) + 0x18]), # (the base's scale, its material)
(pane(base) + 0x3c, lay[pane(tb) + 0x3c:pane(tb) + 0x40]),
(pane(btn) + 0x1c, lay[pane(tbtn) + 0x1c:pane(tbtn) + 0x24])] # (the button, its banner and text)
parts += [(pane(btn + c), lay[pane(tbtn + c):pane(tbtn + c) + 0x24]) for c in range(1, 6)]
out.append((SW[group], SW[group], list(RING_PLACES).index(place),
[[c + o, bytearray(v)] for c in copies for o, v in parts]))
for other, k in small.items():
out.append((SW[group], SW[other], list(RING_PLACES).index(place),
[[c + pane(k) + 0x1c, bytearray(struct.pack("<2f", *xy))] for c in copies]))
return out
def write_ring(path, entries):
with open(path, "wb") as f:
f.write(b"HPRG" + struct.pack("<I", len(entries)))
for group, placed, place, rs in entries:
f.write(struct.pack("<BBBBI", group, placed, place, 0, len(rs)))
for off, data in rs:
f.write(struct.pack("<II", off, len(data)) + bytes(data))
# ---- the ocarina's hints ("Play a note" and the notes' icons: PaGuideKeyOcarinaInput_00 and _01, and PaGuideKeyCapture_00 while a
# song is taught to the scarecrow, in the same archive, four copies each): each icon a text pane of its own, drawn from its left,
# 90 apart, room for the game's icons. A wider one (the Xbox's LB and RB pills) runs into the next, so host/prompts.c moves each
# icon on by the extra width of the icons bound before it: the last one stays, the others and the text move left. prompts/<face>/hint.bin: "HPHN", u32 count of PADS, each pad input's icon's extra width (f32, layout units), u32
# panes, then each: u32 offset (its x in prompts/Common.blarc), u8 slot (0-4 the notes' icons in the hint's order, 5 the text),
# u8 0 x3, f32 its x
HINTS = ("PaGuideKeyOcarinaInput_00", "PaGuideKeyOcarinaInput_01", "PaGuideKeyCapture_00")
HINT_PANES = ["T_Icon_00", "T_Icon_01", "T_Icon_02", "T_Icon_03", "T_Icon_04", "T_Guide_00"]
HINT_FITS = 720 # (the widest advance the game's spacing holds: its own shoulder icons; font units)
HINT_SCALE = 0.1 # (layout units per font unit: the icons' panes draw the font at 100 per 1000 em)
def layout_panes(lay):
"""a BFFH layout's panes: {name: its record's offset} (the record: rotation, scale at +0x10, translation at +0x1c, size at
+0x2c); the header gives the offsets of the names' table (+0x118), the records' table (+0x120) and the count (+0x130)"""
names, recs = struct.unpack_from("<QQ", lay, 0x118)
n = struct.unpack_from("<I", lay, 0x130)[0]
out = {}
for k in range(n):
name = struct.unpack_from("<Q", lay, names + 8 * k)[0]
out[lay[name:lay.index(b"\0", name)].decode()] = struct.unpack_from("<Q", lay, recs + 8 * k)[0]
return out
def hint_panes(blarc):
"""the hint's panes in every copy: [(offset of an x, slot, x)], or None when the layouts are not the ones known. Each pane's
x, and the keys of its x in the layout's animation (its "Type" sets the text's and the first two icons' x, the same in every
state): a curve's values are an array (u64 offset of the floats, u32 count, u32 4, the floats) of (value, slope) pairs, taken
when every value is a pane's x and every slope 0"""
out = []
for name in HINTS:
for a, z in layout_copies(blarc, name):
lay = blarc[a:z]
panes = layout_panes(lay)
if any(p not in panes for p in HINT_PANES):
return None
xs = {}
for slot, p in enumerate(HINT_PANES):
x = struct.unpack_from("<f", lay, panes[p] + 0x1c)[0]
xs.setdefault(x, slot)
out.append((a + panes[p] + 0x1c, slot, x))
for o in range(0, len(lay) - 16, 8):
at, n, size = struct.unpack_from("<QII", lay, o)
if at != o + 16 or size != 4 or not 2 <= n <= 64 or n % 2 or o + 16 + 4 * n > len(lay):
continue
v = struct.unpack_from(f"<{n}f", lay, at)
if v[0] in xs and all(k == v[0] for k in v[::2]) and not any(v[1::2]):
out += [(a + at + 8 * k, xs[v[0]], v[0]) for k in range(n // 2)]
return out or None
def write_hint(path, otf, panes):
from fontTools.ttLib import TTFont
hmtx = TTFont(io.BytesIO(otf))["hmtx"]
with open(path, "wb") as f:
f.write(b"HPHN" + struct.pack("<I", len(PADS)))
f.write(b"".join(struct.pack("<f", round(max(0, hmtx[f"pad_{p}"][0] - HINT_FITS) * HINT_SCALE)) for p in PADS))
f.write(struct.pack("<I", len(panes)) + b"".join(struct.pack("<IB3xf", at, slot, x) for at, slot, x in panes))
def dump(out, face, b, tex, layers):
"""the face's layers as PNG, and the game's textures they go into (once)"""
os.makedirs(os.path.join(out, face), exist_ok=True)
os.makedirs(os.path.join(out, "game"), exist_ok=True)
for layer in layers:
layer.im.save(os.path.join(out, face, layer.name))
game = os.path.join(out, "game", layer.texture + ".png")
if not os.path.exists(game):
decode(b, tex[layer.texture]).save(game)
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--romfs", required=True)
ap.add_argument("--out", required=True)
ap.add_argument("--faces", default=",".join(["switch"] + list(FACES)))
ap.add_argument("--no-textures", action="store_true")
ap.add_argument("--dump", help="also write every face's images as PNG into <dir>/<face>/ (the names nx2/prompt_faces/<face>/ "
"takes in their place), and the game's textures they go into into <dir>/game/")
a = ap.parse_args()
dictionary = zs_dictionary(a.romfs)
game_font = read_zs(a.romfs, FONT, dictionary)
key, otf = unwrap(game_font)
textures = None
if not a.no_textures:
try:
import astc_encoder # noqa: F401
import PIL # noqa: F401
import texture2ddecoder # noqa: F401
textures = read_zs(a.romfs, TEXTURES, dictionary)
except ImportError:
print("prompts: the textures keep the Switch's icons (no astc_encoder, Pillow or texture2ddecoder)", file=sys.stderr)
if textures:
os.makedirs(a.out, exist_ok=True)
with open(os.path.join(a.out, "Common.bntx"), "wb") as f:
f.write(textures)
tex = bntx_textures(textures)
blarc = read_zs(a.romfs, LAYOUTS, dictionary)
hint = hint_panes(blarc)
if not hint:
print("prompts: the ocarina's hint keeps the game's spacing (its layout is not the one known)", file=sys.stderr)
largest = 0
for face in a.faces.split(","):
d = os.path.join(a.out, face)
os.makedirs(d, exist_ok=True)
plain = face_font(otf, face)
font = wrap(key, plain)
if hint:
write_hint(os.path.join(d, "hint.bin"), plain, hint)
largest = max(largest, len(font))
with open(os.path.join(d, "External.bfotf"), "wb") as f:
f.write(font)
if textures:
layers = face_layers(textures, tex, face, otf)
write_textures(os.path.join(d, "textures.bin"), face_textures(textures, tex, layers))
if a.dump:
dump(a.dump, face, textures, tex, layers)
print(f"prompts: {face}", file=sys.stderr)
ring = ring_entries(blarc)
if ring or hint:
with open(os.path.join(a.out, "Common.blarc"), "wb") as f:
f.write(blarc)
if ring:
write_ring(os.path.join(a.out, "ring.bin"), ring)
print("prompts: the Talk ring", file=sys.stderr)
else:
print("prompts: the Talk ring keeps the Switch's places (its layout is not the one known)", file=sys.stderr)
table, path = size_table(a.romfs, dictionary, len(game_font), largest), os.path.join(a.out, "rsizetable.zs")
if table:
import zstandard
with open(path, "wb") as f:
f.write(zstandard.ZstdCompressor(level=3).compress(table))
print(f"prompts: the size table lets the font have {largest} bytes", file=sys.stderr)
elif os.path.exists(path):
os.remove(path)
if __name__ == "__main__":
main()

161
nx2/prompt_draw.py Normal file
View File

@@ -0,0 +1,161 @@
"""(hookshot) Each face's controller for the pause menu's controls page, drawn like the game's Pro Controller (Controller_Pro: flat
greys, a black outline), at the build from the user's RomFS: nx2/prompt_assets.py calls picture(). Its look is the drawing in each
face's entry of nx2/prompt_faces.py; a hand-made picture in nx2/prompt_faces/<face>/Controller_Pro.png takes its place.
The body is the Kenney font's controller glyph (third_party/kenney_input; the Steam Deck: deck() below, Kenney's is a shorter one),
filled, outlined and with its bumpers raised (they stop short of a touchpad); the sticks and the d-pad are the Pro Controller's own,
scaled; the face and small buttons blank discs with the face's labels (the Home button's the game's own icon). All of it drawn SS
times as large and scaled down: smooth edges.
python -I nx2/prompt_draw.py --romfs <dir> --out <dir> (each face's picture as <face>.png, to look at)
"""
import os
import sys
HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
import prompt_assets as P # noqa: E402
from prompt_faces import FACES # noqa: E402
BODY, PART, INK, EDGE = (51, 51, 51, 255), (100, 100, 100, 255), (205, 205, 205, 255), (0, 0, 0, 255)
# the Pro Controller's stick and d-pad (centre, half size in its picture's pixels) and the box the body is fitted in
STICK, DPAD = (347, 228, 74), (453, 348, 70)
BOX, DECK_BOX = (151, 43, 1009, 651), (70, 105, 1088, 573)
SS = 4 # (the drawing's scale before the picture's size)
def sprite(pro, cx, cy, r):
return pro.crop((cx - r, cy - r, cx + r + 1, cy + r + 1))
def blank_disc(r, k=1):
"""a button of the Pro, without its label (its outline k times as thick: drawn k times as large)"""
from PIL import Image, ImageDraw
m = 3 * k
s = Image.new("RGBA", (2 * r + 2 * m + 1, 2 * r + 2 * m + 1), (0, 0, 0, 0))
ImageDraw.Draw(s).ellipse((m, m, 2 * r + m, 2 * r + m), fill=PART, outline=EDGE, width=m)
return s
def grown(mask, d):
"""an L mask (0/255) grown by d pixels all round (a blur's foot: far quicker than a max filter that size)"""
from PIL import ImageFilter
return mask.filter(ImageFilter.GaussianBlur(d / 2)).point(lambda v: 255 if v > 6 else 0)
def silhouette(fname, glyph, h):
"""a Kenney glyph, filled (its holes too), as an L mask"""
from fontTools.ttLib import TTFont
from PIL import Image, ImageChops, ImageDraw, ImageFont
path = os.path.join(P.KENNEY, fname)
rev = {n: c for c, n in TTFont(path).getBestCmap().items()}
im = Image.new("L", (h * 5, h * 4), 0)
ImageDraw.Draw(im).text((h, h), chr(rev[glyph]), font=ImageFont.truetype(path, h * 2), fill=255)
im = im.crop(im.getbbox()).point(lambda v: 255 if v > 127 else 0)
out = Image.new("L", (im.width + 4, im.height + 4), 255)
out.paste(ImageChops.invert(im), (2, 2))
ImageDraw.floodfill(out, (0, 0), 0)
return ImageChops.lighter(im, out.crop((2, 2, im.width + 2, im.height + 2)))
def deck(w):
"""the Steam Deck's body: 2.55 times as wide as high, the grips rounder below"""
from PIL import Image, ImageDraw
h = round(w / 2.55)
m = Image.new("L", (w, h), 0)
d = ImageDraw.Draw(m)
d.rounded_rectangle((0, 0, w - 1, int(h * .92)), radius=int(h * .16), fill=255)
d.ellipse((int(w * .01), int(h * .30), int(w * .27), h - 1), fill=255)
d.ellipse((int(w * .73), int(h * .30), int(w * .99), h - 1), fill=255)
d.rectangle((int(w * .14), int(h * .5), int(w * .86), int(h * .92)), fill=255)
return m
def picture(face, pro, game_icons):
"""the face's controller (RGBA, the size of pro: the game's Controller_Pro), or None for a face with no drawing"""
from PIL import Image, ImageChops, ImageDraw, ImageFilter
drawing = FACES.get(face, {}).get("drawing")
if not drawing:
return None
fname, glyph, controls = drawing
W, H = pro.size[0] * SS, pro.size[1] * SS
stick = sprite(pro, *STICK)
dpad = sprite(pro, *DPAD)
cross = Image.new("L", dpad.size, 0) # (the cross alone: the square crop takes in the left stick's ring)
ImageDraw.Draw(cross).rectangle((0, 42, 140, 98), fill=255)
ImageDraw.Draw(cross).rectangle((44, 0, 100, 140), fill=255)
dpad.putalpha(ImageChops.multiply(dpad.getchannel("A"), cross))
sil = silhouette(fname, glyph, 600 * SS) if fname else deck(1600 * SS)
box = [v * SS for v in (BOX if fname else DECK_BOX)]
s = min((box[2] - box[0]) / sil.width, (box[3] - box[1]) / sil.height) * 0.97
sil = sil.resize((round(sil.width * s), round(sil.height * s)), Image.LANCZOS).point(lambda v: 255 if v > 127 else 0)
ox, oy = (W - sil.width) // 2, box[1] + (box[3] - box[1] - sil.height) // 2
mask = Image.new("L", (W, H), 0)
mask.paste(sil, (ox, oy))
im = Image.new("RGBA", (W, H), (0, 0, 0, 0))
if fname: # (the bumpers: the body raised, an ellipse over each top corner)
up = Image.new("L", (W, H), 0)
up.paste(mask, (0, -18 * SS))
top = Image.new("L", (W, H), 0)
for c in (.25, .75):
ImageDraw.Draw(top).ellipse((ox + int(sil.width * (c - .17)), oy - int(sil.height * .12), ox + int(sil.width * (c + .17)),
oy + int(sil.height * .16)), fill=255)
for kind, cx, _, size, _ in controls:
if kind == "pad": # (a touchpad's top edge is no bumper)
ImageDraw.Draw(top).rectangle((ox + int(sil.width * (cx - size[0] / 2 - .03)), 0,
ox + int(sil.width * (cx + size[0] / 2 + .03)), H), fill=0)
bump = ImageChops.multiply(up, top).filter(ImageFilter.GaussianBlur(SS)).point(lambda v: 255 if v > 127 else 0)
im.paste(EDGE, (0, 0), grown(bump, 3 * SS))
im.paste(PART, (0, 0), bump)
im.paste(EDGE, (0, 0), grown(mask, 3 * SS))
im.paste(BODY, (0, 0), mask)
icons = P.Icons(face, None)
d = ImageDraw.Draw(im)
for kind, cx, cy, size, what in controls:
x, y = ox + cx * sil.width, oy + cy * sil.height
if kind in ("stick", "dpad"):
r = round(size * sil.width / 2)
im.alpha_composite((stick if kind == "stick" else dpad).resize((2 * r + 1, 2 * r + 1), Image.LANCZOS),
(round(x) - r, round(y) - r))
elif kind == "screen":
w2, h2 = size[0] * sil.width / 2, size[1] * sil.height / 2
d.rounded_rectangle((x - w2, y - h2, x + w2, y + h2), radius=8 * SS, fill=(27, 27, 27, 255), outline=EDGE, width=3 * SS)
d.rectangle((x - w2 + 22 * SS, y - h2 + 22 * SS, x + w2 - 22 * SS, y + h2 - 22 * SS), fill=(77, 77, 77, 255),
outline=EDGE, width=2 * SS)
elif kind == "pad":
w2, h2 = size[0] * sil.width / 2, size[1] * sil.height / 2
d.rounded_rectangle((x - w2, y - h2, x + w2, y + h2), radius=round(min(w2, h2) * .35), fill=PART, outline=EDGE,
width=3 * SS)
else: # ("face", "small": a disc with the face's label, the Home button the game's icon)
r = round(size * sil.width / 2)
im.alpha_composite(blank_disc(r, SS), (round(x) - r - 3 * SS, round(y) - r - 3 * SS))
ic = game_icons if what == "guide" else icons
if what in ic.code:
n = int(2 * r * (.92 if kind == "face" else .80))
_, shape, holes = ic.masks(what, n)
full = sum(i * k for i, k in enumerate(shape.histogram())) / 255 / n / n > 0.7 # (a disc with a letter in it)
mark = holes if full else ic.masks(what, int(n * (.7 if kind == "face" else .85)))[0]
im.paste(INK, (round(x) - mark.width // 2, round(y) - mark.height // 2), mark)
return im.resize(pro.size, Image.BOX)
def main():
import argparse
ap = argparse.ArgumentParser()
ap.add_argument("--romfs", required=True)
ap.add_argument("--out", required=True)
a = ap.parse_args()
dictionary = P.zs_dictionary(a.romfs)
b = P.read_zs(a.romfs, P.TEXTURES, dictionary)
pro = P.decode(b, P.bntx_textures(b)["Controller_Pro"]).convert("RGBA")
game_icons = P.Icons("switch", P.unwrap(P.read_zs(a.romfs, P.FONT, dictionary))[1])
os.makedirs(a.out, exist_ok=True)
for face in FACES:
im = picture(face, pro, game_icons)
if im:
im.save(os.path.join(a.out, f"{face}.png"))
print(f"prompts: {face}.png", file=sys.stderr)
if __name__ == "__main__":
main()

110
nx2/prompt_faces.py Normal file
View File

@@ -0,0 +1,110 @@
"""(hookshot) The button-prompt faces: everything that differs between controllers, in one place (nx2/prompt_assets.py builds them).
To add a face: an entry in FACES below, its HOS_PROMPT_* in gpu/vk/vk_api.h (in this order, after the Switch), its name in
host/prompts.c's faces[], and the Insert menu's names for it (gpu/vk/vk_menu.cpp tx_prompts, pad_name;
tests/test_prompt_font_bind.py checks the order). To change one: its entry. To rework a texture by hand: nx2/prompt_assets.py --dump <dir> writes
every image the build makes, as PNG, under the names it looks for in nx2/prompt_faces/<face>/: a PNG put there takes that image's
place (nx2/prompt_faces/README.md).
Each face:
font Kenney's Input Prompts font its icons come from (third_party/kenney_input)
prefix the glyph names' prefix in that font
icons each pad input's glyph name after the prefix ("/name": the whole name); an input left out keeps the game's icon. No
Guide: the makers' logos are their trademarks, every face keeps the game's Home icon.
drawing the controller in the pause menu's controls page (System), drawn at the build like the game's Pro Controller
(nx2/prompt_draw.py): (Kenney font, its controller glyph: the body; None, None: the Steam Deck's own shape) and each
control (kind, centre x, y in the body's box 0..1, size: a fraction of the body's width, or (w, h) for a "pad" or
"screen", the pad input whose label it shows, None: none): "stick", "dpad" (the Pro's own, scaled), "face", "small" (a
disc with the label; none: a blank button), "pad" (a touchpad), "screen". The build makes the page's tinted copy and
the shadow from it. None: the game's Pro Controller with this face's labels on its face buttons.
parts where each pad input is in the picture, for the page's leader lines (picture pixels): (x, y, r), a stick's centre (its
line ends on it), a button's centre and radius (its line ends just off the edge it comes from), a shoulder's top
edge (r 0). The four face buttons are framed. Not a pad input's: boxes the lines keep off (x0, y0, x1, y1: a
screen, a touchpad). SWITCH_PARTS: the game's Pro Controller's.
"""
import os
OVERRIDES = os.path.join(os.path.dirname(os.path.abspath(__file__)), "prompt_faces")
# the d-pad's arms, the same names in each of Kenney's fonts (the outlined pad with the arm meant filled in)
_DPAD = {"up": "dpad_up_outline", "down": "dpad_down_outline", "left": "dpad_left_outline", "right": "dpad_right_outline"}
def _parts(axis, sticks, dpad, face, small, home, shoulders, **boxes):
"""a picture's parts: axis (x + its mirror's x), sticks ((x, y, r) left, right), dpad (x, y, an arm's distance, its r), face
((x, y) north, west, east, south, r), small ((x, y, r) back; start mirrored), home (x, y, r), shoulders ((x, y) trigger,
bumper; the right ones mirrored), boxes (the lines keep off)"""
(ls, rs), (dx, dy, arm, ar) = sticks, dpad
(n, w, e, s, fr), (bx, by, br) = face, small
(tx, ty), (sx, sy) = shoulders
return dict(lstick=ls, rstick=rs, up=(dx, dy - arm, ar), down=(dx, dy + arm, ar), left=(dx - arm, dy, ar),
right=(dx + arm, dy, ar), north=n + (fr,), west=w + (fr,), east=e + (fr,), south=s + (fr,), back=(bx, by, br),
start=(axis - bx, by, br), guide=home, ltrigger=(tx, ty, 0), lshoulder=(sx, sy, 0), rtrigger=(axis - tx, ty, 0),
rshoulder=(axis - sx, sy, 0), **boxes)
SWITCH_PARTS = _parts(1159, ((348, 227, 48), (683, 348, 48)), (455, 348, 45, 17),
((800, 168), (731, 227), (869, 227), (800, 286), 32), (477, 161, 19), (631, 227, 21), ((375, 39), (290, 60)))
FACES = {
"xbox": dict(
font="kenney_input_xbox_series.otf", prefix="xbox_",
drawing=("kenney_input_xbox_series.otf", "controller_xboxseries", [
("stick", .215, .262, .158, "lstick"), ("dpad", .345, .530, .160, None), ("stick", .640, .495, .158, "rstick"),
("face", .700, .275, .080, "west"), ("face", .775, .375, .080, "south"), ("face", .775, .175, .080, "north"),
("face", .850, .275, .080, "east"), ("small", .498, .120, .090, "guide"), ("small", .420, .275, .052, "back"),
("small", .576, .275, .052, "start"), ("small", .498, .355, .044, None)]),
parts=_parts(1156, ((343, 210, 43), (696, 344, 43)), (451, 364, 44, 16), ((809, 160), (746, 218), (871, 218), (809, 275), 31),
(513, 218, 19), (578, 129, 34), ((330, 48), (470, 57)), share=(558, 243, 598, 283)),
icons=dict(_DPAD, south="button_a", east="button_b", west="button_x", north="button_y", back="button_view",
start="button_menu", lshoulder="lb", rshoulder="rb", ltrigger="lt", rtrigger="rt", lstick="ls", rstick="rs")),
"ps4": dict(
font="kenney_input_playstation_series.otf", prefix="playstation_",
drawing=("kenney_input_playstation_series.otf", "controller_playstation4", [
("dpad", .195, .290, .180, None), ("stick", .340, .520, .150, "lstick"), ("stick", .660, .520, .150, "rstick"),
("pad", .499, .200, (.320, .280), None), ("face", .742, .285, .072, "west"), ("face", .814, .175, .072, "north"),
("face", .814, .395, .072, "south"), ("face", .886, .285, .072, "east"), ("small", .298, .130, .036, "back"),
("small", .702, .130, .036, "start"), ("small", .500, .510, .046, "guide")]),
parts=_parts(1160, ((447, 357, 40), (713, 357, 40)), (326, 237, 49, 18), ((841, 177), (781, 234), (901, 234), (841, 292), 28),
(412, 153, 12), (580, 352, 16), ((320, 67), (455, 92)), touchpad=(446, 117, 712, 263)),
icons=dict(_DPAD, south="button_cross", east="button_circle", west="button_square", north="button_triangle",
back="/playstation4_button_share", start="/playstation4_button_options", lshoulder="trigger_l1",
rshoulder="trigger_r1", ltrigger="trigger_l2", rtrigger="trigger_r2", lstick="button_l3", rstick="button_r3")),
"ps5": dict(
font="kenney_input_playstation_series.otf", prefix="playstation_",
drawing=("kenney_input_playstation_series.otf", "controller_playstation5", [
("dpad", .170, .265, .190, None), ("stick", .320, .490, .145, "lstick"), ("stick", .680, .490, .145, "rstick"),
("pad", .498, .175, (.400, .300), None), ("face", .765, .265, .078, "west"), ("face", .840, .150, .078, "north"),
("face", .840, .380, .078, "south"), ("face", .915, .265, .078, "east"), ("small", .248, .080, .036, "back"),
("small", .750, .080, .036, "start"), ("small", .496, .480, .046, "guide"), ("small", .496, .575, .030, None)]),
parts=_parts(1158, ((430, 341, 39), (730, 341, 39)), (305, 217, 51, 19), ((863, 153), (800, 217), (925, 217), (863, 281), 30),
(370, 114, 12), (577, 336, 16), ((321, 52), (445, 64)), touchpad=(412, 84, 744, 250), mute=(563, 375, 591, 403)),
icons=dict(_DPAD, south="button_cross", east="button_circle", west="button_square", north="button_triangle",
back="/playstation5_button_create", start="/playstation5_button_options", lshoulder="trigger_l1",
rshoulder="trigger_r1", ltrigger="trigger_l2", rtrigger="trigger_r2", lstick="button_l3", rstick="button_r3")),
"steamdeck": dict(
font="kenney_input_steam_deck.otf", prefix="steamdeck_",
drawing=(None, None, [ # (the real Deck's proportions: Kenney's icon is a shorter one)
("screen", .500, .465, (.580, .860), None), ("dpad", .072, .200, .078, None), ("stick", .158, .220, .085, "lstick"),
("stick", .842, .220, .085, "rstick"), ("pad", .140, .510, (.112, .270), None), ("pad", .860, .510, (.112, .270), None),
("face", .935, .090, .036, "north"), ("face", .935, .270, .036, "south"), ("face", .900, .180, .036, "west"),
("face", .970, .180, .036, "east"), ("small", .125, .075, .024, "back"), ("small", .875, .075, .024, "start"),
("small", .150, .755, .030, "guide"), ("small", .850, .755, .030, None)]),
parts=_parts(1159, ((242, 230, 27), (917, 230, 27)), (157, 222, 25, 9), ((1009, 180), (974, 215), (1043, 215), (1009, 249), 16),
(209, 174, 9), (234, 437, 12), ((154, 144), (229, 142)), screen=(294, 159, 866, 491),
ltrackpad=(169, 290, 279, 394), rtrackpad=(880, 290, 990, 394), more=(908, 420, 942, 454)),
icons=dict(_DPAD, south="button_a", east="button_b", west="button_x", north="button_y", back="button_view",
start="button_options", lshoulder="button_l1", rshoulder="button_r1", ltrigger="button_l2",
rtrigger="button_r2")),
}
def override(face, name):
"""the path of a hand-made image taking the place of the build's (name: as prompt_assets.py --dump writes it), or None"""
path = os.path.join(OVERRIDES, face, name)
return path if os.path.exists(path) else None
def parts(face):
"""the face's picture's parts (the Switch face: the game's Pro Controller), or None"""
return SWITCH_PARTS if face == "switch" else FACES[face].get("parts")

View File

@@ -0,0 +1,44 @@
# Button-prompt faces: reworking their images
Every image a face shows in the game is made at the build, from the user's RomFS and Kenney's Input Prompts
(`third_party/kenney_input`), by `nx2/prompt_assets.py`. Nothing from the game is kept in git. To change an image, either change
what it is made from, or draw it by hand and put it here.
## What is made, and from what
| Image | What it is | Made from |
| --- | --- | --- |
| `OcarinaButton<A B X Y L R>_00.<pad input>.png` | the ocarina's notes: the note of that Switch button when it is on that pad input | the face's icon for the pad input (`icons` in `nx2/prompt_faces.py`, a glyph of the face's Kenney font) |
| `KeyTexFont<A B X Y>_00.<pad input>.png` | the white letters on black (the Talk ring's small buttons) | the icon's letter (its holes), or its shape when it has none |
| `Controller_Pro.png` | the controller in the pause menu's controls page (System) | the face's `drawing` in `nx2/prompt_faces.py`, drawn by `nx2/prompt_draw.py` |
| `Controller_Pro1.png`, `ControllerSh_Pro1.png` | the page's tinted copy of the controller, and its shadow | `Controller_Pro.png` (yours if you made one) |
| `ControllerGuide_Pro_0<0 1>.png` | the page's leader lines that stay, whatever the controls | the face's `parts` in `nx2/prompt_faces.py` (where each control is in the picture) and the router in `prompt_assets.py` (`GUIDE_*`) |
| `ControllerGuide_Pro_0<0 1>.<Switch button>.<pad input>.png` | the line of that Switch button's label when it is on that pad input, drawn over the lines that stay | the same |
The text icons (dialogue, the control hints, the menus) are not images: they are the face's font, the game's
`External.bfotf` with the face's Kenney glyphs, picked by `icons` in `nx2/prompt_faces.py`.
The Switch face keeps the game's own images for its default controls; only the other bindings get images.
Pad inputs: `south east west north back guide start lstick rstick lshoulder rshoulder up down left right ltrigger rtrigger`.
Switch buttons: `a b x y ls rs l r zl zr plus minus left up right down`.
## Drawing one by hand
1. Write out what the build makes (and the game's own textures, to draw over):
python -I nx2/prompt_assets.py --romfs <romfs> --out /tmp/prompts --dump /tmp/prompts-png
`/tmp/prompts-png/<face>/` has each image under the name above, `/tmp/prompts-png/game/` the game's texture each goes into.
2. Edit the image (keep its size and its mode: RGBA for the notes and the controller, greyscale for the letters, the
lines and the shadow), and save it as `nx2/prompt_faces/<face>/<the same name>`.
3. Build the game (or only the prompts, as in step 1 without `--dump`, into `<game folder>/prompts`): your image takes the
place of the made one. A controller picture of yours also makes the tinted copy and the shadow, unless you give those too.
An image of the wrong size stops the build with its name. A leader line of yours is written only where it has any of
itself: draw it on black, the lines that stay are kept under it.
When a face's picture changes, its `parts` must follow it (the lines end on them): the pictures' pixels, `(x, y, r)` for each
control, see `_parts` in `nx2/prompt_faces.py`.

View File

@@ -0,0 +1,9 @@
# (hookshot) What nx2/prompt_assets.py draws the button prompts with, at build time only: the icon font (fontTools), the
# controller pictures and leader lines (Pillow), and the textures' ASTC blocks (astc-encoder-py, texture2ddecoder). Without
# them the game is built all the same and shows the Switch's icons. The tools' Python carries them: scripts/linux/deps.sh
# (the AppImage) and scripts/release_parts.py package --python-packages (the Windows builder's Python part).
fonttools==4.68.0
pillow==13.0.0
astc-encoder-py==0.1.12
archspec==0.2.7 # astc-encoder-py's runtime dependency (the Windows package installs with --no-deps)
texture2ddecoder==1.0.6

63
nx2/rt/eden_shim.c Normal file
View File

@@ -0,0 +1,63 @@
/* The recompiled game's side of the emulator ABI (mkw_host.h): a64-kit's OS hooks routed to the host table.
a64-kit's runtime (rt/a64_rt.c) leaves a64_svc to the OS layer and makes a64_on_trap, a64_call_miss and
a64_cntpct_host weak; this file is the "OS layer" when the game runs inside Eglantine, and adds a64_poll for the
loop-header polls (nx2/poll.py). Built into mkw_game.dll with the lifted code (nx2/build.py --dll). */
#include "a64.h"
#include "mkw_host.h"
#include <stdio.h>
#include <stdlib.h>
extern const uint32_t a64_fn_count;
extern const uint64_t mkw_image_base;
extern const char mkw_main_build_id[];
#define SAME(f) _Static_assert(offsetof(GuestContext, f) == offsetof(MkwGuestContext, f), "MkwGuestContext." #f)
SAME(x); SAME(pc); SAME(n); SAME(v); SAME(halted); SAME(vreg); SAME(tpidr_el0); SAME(tpidrro_el0); SAME(fpcr);
SAME(fpsr); SAME(excl_addr); SAME(excl_hi); SAME(excl_size); SAME(excl_valid); SAME(thread); SAME(poll);
_Static_assert(sizeof(GuestContext) == sizeof(MkwGuestContext), "MkwGuestContext size");
static MkwHost host;
static int attached;
void a64_svc(Ctx *c, uint32_t imm) { host.svc(host.user, c, imm); }
void a64_poll(Ctx *c) {
__atomic_store_n(&c->poll, 0, __ATOMIC_RELAXED);
host.poll(host.user, c);
}
void a64_on_trap(Ctx *c, uint64_t pc, uint32_t insn, const char *why) {
if (attached) host.trap(host.user, c, pc, insn, why);
fprintf(stderr, "[mkw] %s at %#llx (insn %08x)\n", why, (unsigned long long)pc, insn);
abort();
}
int a64_call_miss(Ctx *c, uint64_t target) { return attached && host.call_miss(host.user, c, target); }
uint64_t a64_cntpct_host(void) { return attached ? host.cntpct(host.user) : 0; }
MKW_EXPORT int mkw_recomp_attach(const MkwHost *h, MkwGameInfo *info) {
if (info) {
info->version = MKW_HOST_VERSION;
info->ctx_size = (uint32_t)sizeof(GuestContext);
info->mem_base = (uint64_t)(uintptr_t)A64_MEM;
info->image_base = mkw_image_base;
info->functions = a64_fn_count;
#if defined(A64_POLLING)
info->polls = 1;
#else
info->polls = 0;
#endif
info->main_build_id = mkw_main_build_id;
}
if (!h || h->version != MKW_HOST_VERSION || h->ctx_size != sizeof(GuestContext))
return 1;
if (!h->svc || !h->poll || !h->trap || !h->call_miss || !h->cntpct)
return 2;
host = *h;
attached = 1;
return 0;
}
MKW_EXPORT MkwFn mkw_recomp_lookup(uint64_t guest_addr) { return (MkwFn)a64_lookup(guest_addr); }

82
nx2/rt/mkw_host.h Normal file
View File

@@ -0,0 +1,82 @@
/* The C ABI between the recompiled game (mkw_game.dll: lifted code + a64-kit's runtime + eden_shim.c) and the
emulator that hosts it (Eglantine's ArmRecomp backend, eglantine/src/core/arm/recomp/).
The DLL knows nothing about Eden: every way out of lifted code (SVC, interrupt poll, trap, an indirect call to an
address with no lifted function, the counter) goes through the MkwHost table the backend hands to
mkw_recomp_attach. The backend knows nothing about the lifted code beyond this header: it looks functions up by
guest address and calls them with a context.
Guest memory is the 64-bit window at A64_HOST_BASE (nx2/rtgen.py): the backend must have placed the process's
fastmem arena there (attach checks it). */
#ifndef MKW_HOST_H
#define MKW_HOST_H
#include <stdint.h>
#ifdef __cplusplus
extern "C" {
#endif
#define MKW_HOST_VERSION 1
/* The lifted code's context (a64-kit rt/a64.h GuestContext with nx2/rtgen.py's poll flag), as the host sees it.
eden_shim.c checks every offset against the real one at compile time. */
typedef struct MkwGuestContext {
uint64_t x[32]; /* x[31] is SP */
uint64_t pc; /* written where a thread stops (traps); otherwise the thread's start address */
uint8_t n, z, c, v;
int32_t halted;
uint64_t vreg[32][2];
uint64_t tpidr_el0;
uint64_t tpidrro_el0;
uint64_t fpcr, fpsr;
uint64_t excl_addr, excl_lo, excl_hi;
uint32_t excl_size;
int32_t excl_valid;
void *thread; /* the host's per-thread object */
uint32_t poll; /* interrupt request: the next loop header calls MkwHost::poll */
} MkwGuestContext;
struct GuestContext;
typedef void (*MkwFn)(struct GuestContext *c);
typedef struct MkwHost {
uint32_t version; /* MKW_HOST_VERSION */
uint32_t ctx_size; /* sizeof(GuestContext) as the host sees it */
void *user;
/* SVC #imm: the context holds the arguments; returns when the thread runs again, results in the context */
void (*svc)(void *user, struct GuestContext *c, uint32_t imm);
/* c->poll was set (an interrupt): returns when the thread runs again */
void (*poll)(void *user, struct GuestContext *c);
/* the thread cannot continue (UDF, BRK, a branch to a non-function): does not return */
void (*trap)(void *user, struct GuestContext *c, uint64_t pc, uint32_t insn, const char *why);
/* BLR / BR / a thread start at an address with no lifted function: run it some other way (the JIT) until it
returns to c->x[30] (calls) or forever (jumps); returns 1 if it did */
int (*call_miss)(void *user, struct GuestContext *c, uint64_t target);
/* CNTPCT_EL0 */
uint64_t (*cntpct)(void *user);
} MkwHost;
#if defined(_WIN32)
#define MKW_EXPORT __declspec(dllexport)
#else
#define MKW_EXPORT __attribute__((visibility("default")))
#endif
/* exported by mkw_game.dll */
typedef struct MkwGameInfo {
uint32_t version; /* MKW_HOST_VERSION */
uint32_t ctx_size; /* sizeof(GuestContext) in the DLL */
uint64_t mem_base; /* A64_HOST_BASE: where the guest window must be */
uint64_t image_base; /* the guest address the image was lifted at (rtld's load address) */
uint32_t functions; /* lifted entries */
uint32_t polls; /* 1 if built with A64_POLLING */
const char *main_build_id; /* hex, the main module's NSO build id */
} MkwGameInfo;
typedef int (*PFN_mkw_recomp_attach)(const MkwHost *host, MkwGameInfo *info); /* 0: ok */
typedef MkwFn (*PFN_mkw_recomp_lookup)(uint64_t guest_addr); /* NULL: no lifted function */
#ifdef __cplusplus
}
#endif
#endif

96
nx2/rtgen.py Normal file
View File

@@ -0,0 +1,96 @@
"""a64-kit's guest model (rt/a64.h, rt/a64_rt.c) with a 64-bit guest window, written next to the lifted code.
a64-kit maps all guest memory below 4 GB (`A64_MEM + (uint32_t)addr`): its own loader picks every address. MKW
does not fit: a 39-bit address space, ~9.5 GiB of planned memory on the 12 GB layout, 2x 5.7 GiB sparse GPU
regions. With A64_ADDR64 (defined by the generated header) an access is `A64_MEM + addr` over a reservation of the
whole 39-bit space (Eden's fastmem arena, or our own), and lookups of code addresses refuse anything above 4 GB
instead of truncating it (code always lies below: Eden loads it at 0x80000000).
It also adds the interrupt flag and A64_POLL that nx2/poll.py's loop-header polls use.
The changes are text substitutions on the vendored files, each required to match exactly once, so a refreshed
a64-kit that moved these lines fails here loudly instead of silently keeping the 32-bit window.
"""
import os
import kit
POLL_DECL = r"""
/* (nx2/poll.py) loop back-edge interrupt polls: A64_POLLING builds clear c->poll and call a64_poll_cur, the backend's,
when it is set. The call is handed nothing on purpose: the context parameter is noescape (A64_NOESC) and never
captured, so a call with no pointer argument cannot touch *c, and the compiler keeps guest registers in host registers
across the loop. (a64_poll(c), the old form, made every loop body reload and store them: a call that takes c may change
any of it.) The poll sits before the backward goto, after the body's stores: at the loop header, a call C cannot mark
as returning (LLVM's willreturn) left the stores after it not guaranteed to execute, and nothing was promoted.
a64_poll_cur finds the thread's context itself (host/mkw_glue.c, hos_ctx). A64_POLL_CTX keeps the old form for the
emulator-hosted DLL (nx2/rt/eden_shim.c), whose host's poll takes the context. */
void a64_poll(Ctx *A64_NOESC c);
void a64_poll_cur(void);
#if defined(A64_POLLING) && defined(A64_POLL_CTX)
#define A64_POLL(c) do { if (__builtin_expect(__atomic_load_n(&(c)->poll, __ATOMIC_RELAXED) != 0, 0)) a64_poll(c); } while (0)
#elif defined(A64_POLLING)
#define A64_POLL(c) do { if (__builtin_expect(__atomic_load_n(&(c)->poll, __ATOMIC_RELAXED) != 0, 0)) { \
__atomic_store_n(&(c)->poll, 0, __ATOMIC_RELAXED); a64_poll_cur(); } } while (0)
#else
#define A64_POLL(c) ((void)0)
#endif"""
ADDR64 = r"""#ifndef A64_ADDR64
#define A64_ADDR64 1 /* (nx2/rtgen.py: 64-bit guest window) */
#endif
#if A64_ADDR64
#define A64_PTR(a) (A64_MEM + (uint64_t)(a))
#define A64_DIRTY_PAGES (1ull << 27) /* (nx2/rtgen.py) A64_WRITE_TRACK: a mark per 4 KB page of the 512 GB window */
#else
#define A64_DIRTY_PAGES (1ull << 20)
#define A64_PTR(a) (A64_MEM + (uint32_t)(a))
#endif"""
RET_MISMATCH = ("void a64_ret_mismatch(Ctx *A64_NOESC c, uint64_t pc); "
"/* RET to something other than the caller (debug builds) */")
THREAD = " void *thread; /* the OS layer's thread object */\n} GuestContext;"
LOOKUP = " uint32_t a = (uint32_t)addr, lo = 0, hi = a64_fn_count;"
# (the soft write watch's table at the window's size, the address masked into it: a stray pointer marks a wrong
# page, never memory past the table)
MARK_NATIVE = r"""extern uint8_t a64_dirty[1u << 20];
/* (Tried 2026-10-21: reading a mark first and writing it only when clear, against cache lines moving between cores:
game threads +1.1 ms a frame, the code 18 MB bigger. Not kept.) */
#define A64_MARK(a, n) (__atomic_store_n(&a64_dirty[(uint32_t)(a) >> 12], 1, __ATOMIC_RELEASE), \
__atomic_store_n(&a64_dirty[(uint32_t)((a) + (n) - 1) >> 12], 1, __ATOMIC_RELEASE))"""
MARK_NATIVE_64 = r"""extern uint8_t a64_dirty[A64_DIRTY_PAGES];
/* (Tried 2026-10-21: reading a mark first and writing it only when clear, against cache lines moving between cores:
game threads +1.1 ms a frame, the code 18 MB bigger. Not kept.) */
#define A64_DIRTY_AT(a) (((uint64_t)(a) >> 12) & (A64_DIRTY_PAGES - 1))
#define A64_MARK(a, n) (__atomic_store_n(&a64_dirty[A64_DIRTY_AT(a)], 1, __ATOMIC_RELEASE), \
__atomic_store_n(&a64_dirty[A64_DIRTY_AT((a) + (n) - 1)], 1, __ATOMIC_RELEASE))"""
SUBST = {
"a64.h": [
(THREAD, THREAD.replace("\n} GuestContext;", "\n uint32_t poll; "
"/* (nx2/rtgen.py) interrupt request: A64_POLL clears it and calls a64_poll_cur when set */\n} GuestContext;")),
(RET_MISMATCH, RET_MISMATCH + POLL_DECL),
("#define A64_PTR(a) (A64_MEM + (uint32_t)(a))", ADDR64),
(MARK_NATIVE, MARK_NATIVE_64),
],
"a64_rt.c": [
(LOOKUP, "#if A64_ADDR64\n if (addr >> 32) return NULL; /* (code lies below 4 GB: never a truncated match) */\n"
"#endif\n" + LOOKUP),
("uint8_t a64_dirty[1u << 20];", "uint8_t a64_dirty[A64_DIRTY_PAGES];"),
],
}
def write(out_dir):
os.makedirs(out_dir, exist_ok=True)
for name, subs in SUBST.items():
text = open(os.path.join(kit.KIT, "rt", name), encoding="utf-8").read()
for old, new in subs:
n = text.count(old)
if n != 1:
raise SystemExit("rtgen: %r found %d times in a64-kit rt/%s (expected once): update nx2/rtgen.py"
% (old[:60], n, name))
text = text.replace(old, new)
head = "/* Generated by mkw-recomp nx2/rtgen.py from a64-kit rt/%s (64-bit guest window). Do not edit. */\n" % name
with open(os.path.join(out_dir, name), "w", newline="\n", encoding="utf-8") as f:
f.write(head + text)
return out_dir

23
nx2/samap.py Normal file
View File

@@ -0,0 +1,23 @@
"""Triforce.exe offset -> lifted function (f_<guest address>) + offset, from work/sa/build/Triforce.map (build.py --standalone).
Usage: python -I nx2/samap.py <hex offset> [...]
"""
import bisect
import os
import re
import sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import kit
rows = []
for line in open(os.path.join(kit.WORK, "sa", "build", "Triforce.map"), errors="replace"):
m = re.match(r"\s*[0-9a-f]{4}:[0-9a-f]{8}\s+(\S+)\s+([0-9a-f]{16})", line)
if m:
rows.append((int(m.group(2), 16), m.group(1)))
rows.sort()
keys = [r[0] for r in rows]
for a in sys.argv[1:]:
t = 0x140000000 + int(a, 16)
i = bisect.bisect_right(keys, t) - 1
print("%s: %s+%#x" % (a, rows[i][1], t - rows[i][0]))

38
nx2/sasym.py Normal file
View File

@@ -0,0 +1,38 @@
"""Symbolize guest addresses by the image's symbol names: a standalone run's crash backtrace (work/runs/<run>/log.txt),
or addresses given on the command line.
Usage: python -I nx2/sasym.py <run> | <hex address> ...
"""
import bisect
import os
import subprocess
import sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import kit
import image as eden_image
import toolchain
def main():
if sys.argv[1].startswith("0x"):
addrs = [int(a, 16) for a in sys.argv[1:]]
else:
log = open(os.path.join(kit.WORK, "runs", sys.argv[1], "log.txt"), errors="replace").read()
if "guest backtrace: lr " not in log:
sys.exit("no crash in that run")
addrs = [int(x, 16) for x in log.split("guest backtrace: lr ")[1].split("\n")[0].split() if x != "0"]
img = eden_image.EdenImage(kit.need_exefs())
ks = sorted(img.names)
rows = []
for a in addrs:
j = bisect.bisect_right(ks, a) - 1
rows.append((a, img.names[ks[j]] if j >= 0 else "?", a - ks[j] if j >= 0 else 0))
dm = subprocess.run([toolchain.find("llvm-cxxfilt")], input="\n".join(r[1] for r in rows), capture_output=True,
text=True).stdout.splitlines()
for (a, _, off), d in zip(rows, dm):
print("%#x %s +%#x" % (a, d[:110], off))
if __name__ == "__main__":
main()

298
nx2/shader_census.py Normal file
View File

@@ -0,0 +1,298 @@
"""Every shader program in the game's archives, for the precompile before the first start.
python -I nx2/shader_census.py <romfs> [--out <file>]
Walks romfs/Shader/*.bfsha.zs (and the shader packages inside other archives), and each BNSH inside them. A BNSH's variations
are programs: which stages go together, so what comes out are real pairs, not a heap of stages. --out writes them for the back
end to translate (gpu/vk/vk_shader.cpp); nothing here goes into the repository: the file is made from the player's own game.
The game's compute programs, and many of its graphics ones, are not in a BNSH: agl's shader archives (BAHS, in Lib/ and in
Shader/ApplicationPackage), agl's networks (NDSR, Lib/agl), the movie upscaler's (NDVE, Movie/Ndve) and the particle sets
(Effect/*.vsetb.byml.zs, each a BNSH inside a byml) are walked too, each in its own way (below). A stage is its control section
(0xd8 bytes) and the code area it sizes, the bytes the game hands the GPU: those are what the back end keys by.
After tools/shader_census.py of TotK Recomp (github.com/valentderah/totk-pc-recomp, MIT), itself after MacNativeGames'
prepare_shader_seed.py (MIT): the BNSH layout and the walk of its variations. BAHS, NDSR and NDVE were read from the stages a
play run dumped (RX_VK_DUMP_STAGES), after the first reading of BAHS (the code offset 0x68 bytes before a control section,
where the control section follows its own stage record).
<out>: "HKSEED01", u32 programs, u32 stages; then the stages, each once:
u64 key, u32 stage, u32 control bytes, u32 code bytes, control, code
then the programs: u32 count, u32 stage indices
"""
import argparse
import collections
import hashlib
import os
import re
import struct
import sys
import time
CONTROL_MAGIC = b"\x34\x12\x76\x98"
ZSTD = b"\x28\xb5\x2f\xfd"
# the stages a variation's program points at (a 0 offset: none), as far as the structure reaches
SLOTS = (8, 16, 24, 32, 40, 48, 56, 64, 72)
def dictionaries(romfs, zs):
"""The game's zstd dictionaries: the files of a SARC, Pack/ZsDic.pack.zs"""
raw = zs.ZstdDecompressor().decompress(open(os.path.join(romfs, "Pack", "ZsDic.pack.zs"), "rb").read(), max_output_size=1 << 28)
start = struct.unpack_from("<I", raw, 0x0c)[0]
out = []
for k in range(struct.unpack_from("<H", raw, 0x1a)[0]):
_, _, st, end = struct.unpack_from("<IIII", raw, 0x20 + k * 16)
out.append(raw[start + st:start + end])
return out
def unpack(data, dicts, zs):
if data[:4] != ZSTD:
return data
try:
return zs.ZstdDecompressor().decompress(data, max_output_size=1 << 31)
except zs.ZstdError:
pass
for d in dicts:
try:
return zs.ZstdDecompressor(dict_data=zs.ZstdCompressionDict(d, dict_type=zs.DICT_TYPE_FULLDICT)).decompress(
data, max_output_size=1 << 31)
except zs.ZstdError:
continue
raise ValueError("no dictionary fits")
def bnsh_programs(data, stats):
"""Each variation of a BNSH (its offsets are its own) as [(stage, control, code)...]"""
if data[:4] != b"BNSH" or data[12:14] != b"\xff\xfe" or data[96:100] != b"grsc":
raise ValueError("unsupported BNSH header")
size = struct.unpack_from("<I", data, 28)[0]
data = data[:size]
count, variations = struct.unpack_from("<IQ", data, 124)
for i in range(count):
program = struct.unpack_from("<Q", data, variations + i * 64 + 16)[0]
if not program or program + 80 > size:
stats["no program"] += 1
continue
stages = []
for rel in SLOTS:
offset = struct.unpack_from("<Q", data, program + rel)[0]
if not offset or offset + 32 > size:
continue
control_off, code_off, code_size, control_size = struct.unpack_from("<QQII", data, offset + 8)
if not 0x60 <= control_size <= 0x1000 or control_off + control_size > size or code_off + code_size > size:
continue
control = data[control_off:control_off + control_size]
if control[:4] != CONTROL_MAGIC:
continue
stage = struct.unpack_from("<H", control, 0x56)[0]
stats["slot %d, stage %d" % (rel, stage)] += 1
stages.append((stage, control, data[code_off:code_off + code_size]))
if stages:
yield stages
def control_at(data, c):
"""The 0xd8-byte NVN2 control section at c, and its code area's size (+0x44), or None"""
control = data[c:c + 0xd8]
if len(control) < 0xd8 or control[:4] != CONTROL_MAGIC or struct.unpack_from("<H", control, 0x56)[0] > 7:
return None, 0
return control, struct.unpack_from("<I", control, 0x44)[0]
def stage(control, code):
return struct.unpack_from("<H", control, 0x56)[0], control, code
def bahs_programs(data):
"""agl's shader archive (.sharcb, magic BAHS: gsys, agl, terrain and ApplicationPackage's AglShader / FoliageShader).
Header +0x14 u32 the data start, +0x18 the code blob's size, +0x1c its start. At data +0x150: u32 the program table (from
there), u32 the stage count; the stages follow from data +0x158, each {u32 bytes to the next, u32 kind (0 vertex, 1 fragment,
3 compute, 4 task, 5 mesh), u32, u32, u64 its code area's offset in the blob, u32 control bytes, s32 its control section
from there}. Stages share control sections (one control, several codes). The program table: u32 bytes, u32 programs, each
{u32 bytes, u32 name bytes, u32 its kinds (a bit each), u32 its first stage, name, options}; a program's variations are its
run of stages up to the next program's first, one stage of each of its kinds per variation."""
base, _, blob = struct.unpack_from("<III", data, 0x14)
table, count = struct.unpack_from("<II", data, base + 0x150)
at, stages = base + 0x158, []
for _ in range(count):
step = struct.unpack_from("<I", data, at)[0]
if step < 32:
raise ValueError("stage at %#x" % at)
offset, _, rel = struct.unpack_from("<QIi", data, at + 16)
control, end = control_at(data, at + 16 + rel)
if control is None or blob + offset + end > len(data):
raise ValueError("stage at %#x" % at)
stages.append(stage(control, data[blob + offset:blob + offset + end]))
at += step
table += base + 0x150
at, heads = table + 8, []
for _ in range(struct.unpack_from("<I", data, table + 4)[0]):
size, _, kinds, first = struct.unpack_from("<4I", data, at)
heads.append((first, bin(kinds).count("1")))
at += size
heads.sort()
for i, (first, width) in enumerate(heads):
last = heads[i + 1][0] if i + 1 < len(heads) else count
for v in range(first, last, max(width, 1)):
yield stages[v:v + width]
def ndsr_programs(data, base):
"""agl's networks (agl_ndsr*.pck, magic NDSR; also inside NDVE at +0x28): u32 +0x0c the item count, from +0x14 each item
{u32 offset from the NDSR, u32 bytes << 4 | type}. Types 0 / 3 / 6 are a vertex / fragment / compute control section, the item
after each (1 / 4 / 7) its code; a vertex and the fragment after it are a program, a compute stage is one alone."""
items = [struct.unpack_from("<II", data, base + 0x14 + 8 * i) for i in range(struct.unpack_from("<I", data, base + 0x0c)[0])]
graphics = []
for (c, ct), (x, xt) in zip(items, items[1:]):
if ct & 15 not in (0, 3, 6) or xt & 15 != (ct & 15) + 1:
continue
control, end = control_at(data, base + c)
if control is None or base + x + end > len(data):
continue
s = stage(control, data[base + x:base + x + end])
if ct & 15 == 6:
yield [s]
elif ct & 15 == 0:
if graphics:
yield graphics
graphics = [s]
elif graphics:
graphics.append(s)
if graphics:
yield graphics
def ndve_programs(data):
"""The movie upscaler's networks (Movie/Ndve, magic NDVE): an NDSR at +0x28, then tables {u32 stages, u32 blobs, blobs x {u32
offset, u32 bytes}, stages x {u32 control section from the table, u32 code from the control, u32 code bytes, u32 0}}. Each
control's entry is found from its code bytes (the pair "bytes, 0") and checked against its table's header."""
if data[0x28:0x2c] == b"NDSR":
yield from ndsr_programs(data, 0x28)
for m in re.finditer(re.escape(CONTROL_MAGIC), data):
c = m.start()
control, end = control_at(data, c)
if control is None:
continue
for e in re.finditer(re.escape(struct.pack("<II", end, 0)), data):
entry = e.start() - 8
if entry < 0 or entry % 4:
continue
to_control, to_code = struct.unpack_from("<II", data, entry)
table = c - to_control
if not to_control or table < 0 or table % 4 or c + to_code + end > len(data):
continue
n, blobs = struct.unpack_from("<II", data, table)
first = table + 8 + 8 * blobs
if 0 < n < 256 and blobs < 16 and (entry - first) % 16 == 0 and 0 <= (entry - first) // 16 < n:
yield [stage(control, data[c + to_code:c + to_code + end])]
break
def bnsh_all(data, stats):
"""Each BNSH inside the data (bfsha, and Effect's particle sets: a BNSH inside a byml)"""
for m in re.finditer(b"BNSH", data):
if data[m.start() + 96:m.start() + 100] == b"grsc":
yield from bnsh_programs(data[m.start():m.start() + struct.unpack_from("<I", data, m.start() + 28)[0]], stats)
def programs_in(data, stats):
if data[:4] == b"BAHS":
return bahs_programs(data)
if data[:4] == b"NDSR":
return ndsr_programs(data, 0)
if data[:4] == b"NDVE":
return ndve_programs(data)
return bnsh_all(data, stats)
def archives(romfs):
"""The files that hold shaders: Shader/* (bfsha programs; ApplicationPackage's agl archives), Layout2Shader/*, Lib/** (agl,
gsys and terrain archives, agl's networks), Movie/** (the upscaler's networks), Effect/* (particle sets)"""
for folder in ("Shader", "Layout2Shader", "Lib", "Movie", "Effect"):
for top, dirs, names in os.walk(os.path.join(romfs, folder)):
dirs.sort()
for name in sorted(names):
if name.endswith(".zs") or ".bfsha" in name or ".sarc" in name or ".pck" in name:
yield os.path.join(top, name)
def sarc_files(data):
start = struct.unpack_from("<I", data, 0x0c)[0]
count = struct.unpack_from("<H", data, 0x1a)[0]
for k in range(count):
_, _, st, end = struct.unpack_from("<IIII", data, 0x20 + k * 16)
yield data[start + st:start + end]
def blobs(data, dicts, zs):
"""The data itself, or each file when it is an archive"""
if data[:4] == b"SARC":
for inner in sarc_files(data):
yield from blobs(unpack(inner, dicts, zs), dicts, zs)
else:
yield data
def main():
p = argparse.ArgumentParser(description=__doc__.split("\n")[0])
p.add_argument("romfs")
p.add_argument("--out")
a = p.parse_args()
import zstandard as zs
dicts = dictionaries(a.romfs, zs)
stats = collections.Counter()
stage_index, stage_list, programs, seen = {}, [], [], set()
started = time.time()
effect = collections.Counter()
for path in archives(a.romfs):
made, found = 0, []
try: # (a file or container this doesn't read is told and skipped)
for k, data in enumerate(blobs(unpack(open(path, "rb").read(), dicts, zs), dicts, zs)):
try:
found += list(programs_in(data, stats))
except Exception as e:
print("%-64s container %d %r skipped: %s" % (os.path.basename(path)[:64], k, data[:4], e), flush=True)
except Exception as e:
print("%-64s skipped: %s" % (os.path.basename(path)[:64], e), flush=True)
for stages in found:
ids = []
for stage, control, code in stages:
# (what the back end reads of a stage: the 0xd8 control bytes and the code area they size, so the same stage
# from two archives is one)
end = struct.unpack_from("<I", control, 0x44)[0]
control, code = control[:0xd8], code[:end] if end <= len(code) else code
key = int.from_bytes(hashlib.sha1(control + code, usedforsecurity=False).digest()[:8], "little")
if key not in stage_index:
stage_index[key] = len(stage_list)
stage_list.append((key, stage, control, code))
ids.append(stage_index[key])
ids = tuple(sorted(set(ids)))
if ids not in seen:
seen.add(ids)
programs.append(ids)
made += 1
if os.path.basename(os.path.dirname(path)) == "Effect": # (1400 particle sets: one line for them all)
effect["files"] += 1
effect["programs"] += made
else:
print("%-64s %6d programs" % (os.path.basename(path)[:64], made), flush=True)
print("%-64s %6d programs" % ("Effect (%d particle sets)" % effect["files"], effect["programs"]))
print("%.0f s" % (time.time() - started))
kinds = collections.Counter(tuple(sorted(stage_list[i][1] for i in ids)) for ids in programs)
print("%d programs from %d stages" % (len(programs), len(stage_list)))
print("by stages:", kinds.most_common(12))
print("slots:", sorted(stats.items()))
if a.out:
with open(a.out, "wb") as f:
f.write(b"HKSEED01" + struct.pack("<II", len(programs), len(stage_list)))
for key, stage, control, code in stage_list:
f.write(struct.pack("<QIII", key, stage, len(control), len(code)) + control + code)
for ids in programs:
f.write(struct.pack("<I%dI" % len(ids), len(ids), *ids))
print("wrote", a.out, os.path.getsize(a.out) >> 20, "MB")
if __name__ == "__main__":
sys.exit(main())

86
nx2/split_hot.py Normal file
View File

@@ -0,0 +1,86 @@
"""code_hot.c in pieces that compile side by side: 83 MB in one file took 79 minutes on one core, and every full build waited for it.
python -I nx2/split_hot.py [gen folder] [pieces, default 8]
Functions that call each other stay together, heaviest calls first, so what used to inline still can. The pieces cost about the
same to compile (chunks.fn_cost). code_hot.c stays as it was, renamed code_hot.c.whole, and build.py compiles code_hot_*.c at the
hot flags. A profile from the whole file fits the pieces: its counters go by function name and shape.
"""
import collections
import glob
import os
import re
import sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # (python -I leaves the script folder off)
import kit # noqa: E402,F401 (a64-kit on the path, for chunks)
import chunks # noqa: E402
import lifted_c # noqa: E402
HEAD = lifted_c.HEAD
CALL = re.compile(r"\b(f_[0-9a-f]{8})\(c\)")
def split(gen, pieces):
path = os.path.join(gen, "code_hot.c")
whole = path + ".whole"
if os.path.exists(path):
os.replace(path, whole)
for old in glob.glob(os.path.join(glob.escape(gen), "code_hot_*.c")):
os.remove(old)
with open(whole, encoding="utf-8", newline="") as f:
text = f.read()
marks = list(HEAD.finditer(text))
if not marks:
raise SystemExit("split_hot: no functions in %s" % whole)
preamble = text[:marks[0].start()]
bodies = {m.group(1): text[m.start():(marks[i + 1].start() if i + 1 < len(marks) else len(text))] for i, m in enumerate(marks)}
# (groups joined along the calls: small callees first, those are what the optimiser inlines)
limit = sum(len(b) for b in bodies.values()) * 1.1 / pieces
parent = {f: f for f in bodies}
size = {f: len(b) for f, b in bodies.items()}
def root(f):
while parent[f] != f:
parent[f] = parent[parent[f]]
f = parent[f]
return f
edges = collections.Counter()
for f, b in bodies.items():
for callee in CALL.findall(b):
if callee in bodies and callee != f:
edges[(f, callee)] += 1
for (f, callee), n in sorted(edges.items(), key=lambda e: (len(bodies[e[0][1]]) / e[1], e[0])):
a, b = root(f), root(callee)
if a != b and size[a] + size[b] <= limit:
parent[b] = a
size[a] += size[b]
groups = collections.defaultdict(list)
for f in bodies:
groups[root(f)].append(f)
# (the groups dealt out by their compile cost, not their size: a piece's time goes with the square of its functions' sizes, and by
# size one piece took 333 s and another 60)
cost = {f: chunks.fn_cost(len(b)) for f, b in bodies.items()}
bins = [[0, []] for _ in range(pieces)]
for g in sorted(groups.values(), key=lambda g: (-sum(cost[f] for f in g), min(g))):
b = min(bins, key=lambda x: x[0])
b[0] += sum(cost[f] for f in g)
b[1].extend(g)
costs = {}
for i, (c, names) in enumerate(bins):
with open(os.path.join(gen, "code_hot_%d.c" % i), "w", encoding="utf-8", newline="") as f:
f.write(preamble)
for name in sorted(names):
f.write(bodies[name])
costs["code_hot_%d.c" % i] = len(preamble) + c
print("split_hot: piece %d: %d functions, %d MB, cost %d MB" % (i, len(names), sum(len(bodies[f]) for f in names) >> 20, c >> 20))
chunks.write_costs(gen, costs)
kept = sum(n for (f, c), n in edges.items() if root(f) == root(c))
print("split_hot: %d of %d calls stay in one group" % (kept, sum(edges.values())))
if __name__ == "__main__":
here = os.path.dirname(os.path.abspath(__file__))
split(sys.argv[1] if len(sys.argv) > 1 else os.path.join(here, "..", "work", "sa", "gen"), int(sys.argv[2]) if len(sys.argv) > 2 else 8)

201
nx2/standalone.py Normal file
View File

@@ -0,0 +1,201 @@
"""The standalone build's lift-time pieces (lift.py --standalone): which SDK functions are replaced by the native
runtime in host/, the stubs for the OS-facing ones it does not have yet, and the boot constants hos_main.c reads.
Adapted from a64-kit's build.py (TOTKPort e88744a: implemented, gen_stubs, OS_NAMESPACES, nested_name, os_facing,
thread_hooks, write_image_c), for MKW's Eden-layout image in the 64-bit window.
The model is TOTKPort's: the SDK is replaced at the nn:: boundary where it talks to system services. A function the
runtime defines as `HLE(<mangled name>)` / `void imp_<mangled name>(Ctx *c)` is hooked where the game imports it and
at the sdk's own definition. Every other free function of an OS-facing namespace the game imports gets a generated
stub that logs its first call and returns 0. The rest of the SDK (libc, nn::util, nn::mem, std, the NVN driver)
runs as lifted code.
"""
import os
import re
import struct
import subprocess
import sys
import kit
import toolchain
HOST = os.path.join(kit.HERE, "..", "host")
# a64-kit's list (TOTK), plus the Switch 2 SDK's service namespaces MKW imports (imports.py)
OS_NAMESPACES = {"audio", "hid", "oe", "prepo", "nfp", "time", "account", "vi", "swkbd", "pl", "err", "codec", "diag",
"detail", "settings", "ns", "applet", "album", "friends", "nifm", "ssl", "socket", "ec", "bcat",
"web", "ldn", "irsensor", "aoc", "olsc", "pctl", "apm", "ro",
"chat", "camera", "gs", "movie", "npns", "news", "fatal", "lbl", "capture", "syncpt"}
def implemented(host=HOST):
"""C names of the HLE functions in host/*.c: `void imp_x(Ctx` definitions and `HLE(x)` macros"""
names = set()
for f in os.listdir(host):
if f.endswith(".c"):
src = open(os.path.join(host, f), errors="replace").read()
names |= set(re.findall(r"^void (imp_\w+)\(Ctx", src, re.M))
names |= {"imp_" + n for n in re.findall(r"^HLE\((\w+)\)", src, re.M)}
return names
def nested_name(sym):
"""The components of an Itanium nested name (_ZN[K]<len><id>...E), or None if it is not that simple shape"""
m = re.match(r"_ZNK?", sym)
if not m:
return None
i, parts = m.end(), []
while i < len(sym) and sym[i].isdigit():
j = i
while sym[j].isdigit():
j += 1
n = int(sym[i:j])
parts.append(sym[j:j + n])
i = j + n
return parts if i < len(sym) and sym[i] == "E" else None
def os_facing(sym):
"""A free function of an OS-facing namespace: nn::<OS namespace>::f (class members and data stay lifted)"""
parts = nested_name(sym)
# (the Switch 2 SDK keeps many service calls a level down: nn::camera::sdk::CreateCameraContext)
return bool(parts) and parts[0] == "nn" and parts[1:2] and parts[1] in OS_NAMESPACES and (
len(parts) == 3 or (len(parts) == 4 and parts[2] == "sdk"))
def nvn_exports(modules):
"""Every NVN function the sdk defines (the console driver's C entry points): all are hooked to host/nvn2.c's
procs (gen/nvn2_exports.c), so the driver never runs, whether the game reaches a function through
nvnDeviceGetProcAddress or imports it (nvnNativeImageInitialize, for one)"""
sdk = next(m for m in modules if m.name == "sdk")
have = {n[4:] for n in implemented()}
return sorted(s["name"] for s in sdk.symbols if s["defined"] and s["name"] and nvn_proc_name(s["name"])
and s["type"] == "func" and s["name"] not in have)
def nvn_proc_name(sym):
"""The NVN proc a symbol is: a C name (nvnFoo), or a C++ one (_Z<len>nvnFoo<args>: the Switch 2 driver exports
some, nvnNativeImageInitialize among them); None for anything else"""
if sym.startswith("nvn"):
return sym
m = re.match(r"_Z(\d+)(nvn\w+)", sym)
return m.group(2)[:int(m.group(1))] if m else None
def write_nvn_exports(gen, names):
lines = ["/* Generated by nx2/lift.py --standalone: the sdk's NVN functions, routed to host/nvn2.c. */",
'#include "a64.h"', "void nvn2_dispatch(Ctx *c, const char *name, void **cache);", ""]
for n in names:
lines.append('void imp_%s(Ctx *A64_NOESC c) { static void *k[2]; nvn2_dispatch(c, "%s", k); }'
% (n, nvn_proc_name(n)))
text = "\n".join(lines) + "\n"
path = os.path.join(gen, "nvn2_exports.c")
if not os.path.exists(path) or open(path).read() != text:
open(path, "w", newline="\n").write(text)
def hle_set(modules):
"""(HLE'd symbols, stubbed symbols): the runtime's imp_ functions some module has, the sdk's NVN functions, and
the OS-facing functions the game modules import that it does not define"""
hle = {n[4:] for n in implemented()}
known = {s["name"] for m in modules for s in m.symbols if s["name"]}
# (TOTK's SDK functions that MKW's newer SDK no longer has: their HLE stays unused; a typo shows here too)
missing = sorted(hle - known)
if missing:
print("host/ defines %d imp_ functions for symbols no module has (unused):\n %s"
% (len(missing), "\n ".join(missing)))
hle -= set(missing)
kind = {s["name"]: s["type"] for m in modules for s in m.symbols if s["defined"] and s["name"]}
stubbed = set()
for m in modules:
if m.name not in ("sdk", "rtld"): # (every game module: main and the subsdk* the exefs has)
stubbed |= {s["name"] for s in m.symbols if not s["defined"] and s["name"] and os_facing(s["name"])
and kind.get(s["name"], "func") == "func"}
return hle | set(nvn_exports(modules)), stubbed - hle
def demangle(syms):
try:
out = subprocess.run([toolchain.find("llvm-cxxfilt")], input="\n".join(syms), capture_output=True,
text=True).stdout.splitlines()
return dict(zip(syms, out)) if len(out) == len(syms) else {s: s for s in syms}
except (OSError, TypeError):
return {s: s for s in syms}
def gen_stubs(gen, meta):
have = implemented() | {"imp_" + h["sym"] for h in meta["hooks"] if nvn_proc_name(h["sym"])}
hooks = [h for h in meta["hooks"] if h["c"] not in have]
dm = demangle([h["sym"] for h in hooks])
lines = ['#include "a64.h"', "void a64_unimplemented(Ctx *A64_NOESC c, const char *sym, int *seen);", ""]
for h in hooks:
lines.append("/* %s */" % dm[h["sym"]].replace("*/", "* /"))
lines.append('void %s(Ctx *A64_NOESC c) { static int seen; a64_unimplemented(c, "%s", &seen); }'
% (h["c"], h["sym"]))
text = "\n".join(lines) + "\n"
path = os.path.join(gen, "a64_stubs.c")
if not os.path.exists(path) or open(path).read() != text:
open(path, "w", newline="\n").write(text)
return len(hooks)
def thread_hooks(img, meta):
"""The SDK thread entry's start and exit hooks (nnmusl's per-thread TLS setup and teardown): the entry starts
`mrs x1, tpidrro_el0; add x1, x1, #0x1f8; str x0, [x1]` and loads each hook through a GOT slot"""
for m in meta["modules"]:
for pc in range(m["base"], m["base"] + m["size"] - 12, 4):
if not img.in_code(pc) or img.word(pc) != 0xD53BD061:
continue
if img.word(pc + 4) != 0x9107E021 or img.word(pc + 8) != 0xF9000020:
continue
hooks, page = [], {}
for q in range(pc + 12, pc + 12 + 4 * 40, 4):
w = img.word(q)
if (w & 0x9F000000) == 0x90000000: # ADRP
imm = ((w >> 3) & 0x1FFFFC) | ((w >> 29) & 3)
imm = imm - (1 << 21) if imm & (1 << 20) else imm
page[w & 31] = (q & ~0xFFF) + (imm << 12)
elif (w & 0xFFC00000) == 0xF9400000 and ((w >> 5) & 31) in page: # LDR Xt, [Xn, #imm]
got = page.pop((w >> 5) & 31) + ((w >> 10) & 0xFFF) * 8
hooks.append(struct.unpack("<Q", img.read(got, 8))[0])
if len(hooks) == 2:
return hooks
return [0, 0]
def write_image_c(gen, img, meta, exefs):
"""Boot constants for hos_main.c: the image span, each module's DT_INIT in rtld's order (rtld itself is not run:
the image is already relocated), the entry points, and the main thread's stack, priority and core (main.npdm)"""
ex = meta["exports"]
mods = [m for m in meta["modules"] if m["init"] and m["name"] != "rtld"]
npdm = open(os.path.join(exefs, "main.npdm"), "rb").read()
if npdm[:4] != b"META":
sys.exit("main.npdm: no META header")
prio, core = npdm[0xE], npdm[0xF]
stack = int.from_bytes(npdm[0x1C:0x20], "little")
aoff, asz = int.from_bytes(npdm[0x70:0x74], "little"), int.from_bytes(npdm[0x74:0x78], "little")
aci = npdm[aoff:aoff + asz]
program_id = int.from_bytes(aci[0x10:0x18], "little") if aci[:4] == b"ACI0" else 0
main_build = next((m["build_id"] for m in meta["modules"] if m["name"] == "main"), "")
lines = ["/* Generated by nx2/lift.py --standalone. */", "#include <stdint.h>",
"const uint64_t hos_main_stack_size = %#xULL;" % stack,
"const int hos_main_priority = %d, hos_main_core = %d;" % (prio, core),
"const uint64_t hos_image_base = %#xULL, hos_image_end = %#xULL;" % (meta["base"], meta["end"]),
"const int hos_module_count = %d;" % len(mods),
"const uint64_t hos_module_init[] = {%s};" % ", ".join("%#xULL" % m["init"] for m in mods),
"const char *const hos_module_name[] = {%s};" % ", ".join('"%s"' % m["name"] for m in mods),
"const uint64_t hos_entry_nninitStartup = %#xULL;" % ex.get("nninitStartup", 0),
"const uint64_t hos_entry_nnMain = %#xULL;" % ex.get("nnMain", 0),
"const uint64_t hos_program_id = %#xULL; /* (main.npdm's ACI0) */" % program_id,
'const char hos_main_build_id[] = "%s";' % main_build]
for sym in ("_init_libc0", "_init_libc1", "_init_libc2", "nnosInitialize", "nninitInitializeSdkModule",
"nvnBootstrapLoaderInternal"):
lines.append("const uint64_t hos_entry_%s = %#xULL;" % (sym.lstrip("_"), ex.get(sym, 0)))
start_hook, exit_hook = thread_hooks(img, meta)
lines.append("const uint64_t hos_thread_start_hook = %#xULL, hos_thread_exit_hook = %#xULL;"
% (start_hook, exit_hook))
text = "\n".join(lines) + "\n"
path = os.path.join(gen, "hos_image.c")
if not os.path.exists(path) or open(path).read() != text:
open(path, "w", newline="\n").write(text)
return start_hook, exit_hook

View File

@@ -0,0 +1,97 @@
"""Create BC/raw views using the actual image flags and make_view; Windows Vulkan device required.
python nx2/test_bc_storage_views.py --headers <Vulkan-Headers/include> [--cxx clang++]
Enable the Khronos validation layer through VK_LAYER_PATH/VK_INSTANCE_LAYERS.
No game data, shader, or image contents are used; this checks creation, not storage-write fidelity.
"""
import argparse
from pathlib import Path
import subprocess,tempfile
RUN_ALL_ARGS = ["--headers", "{vulkan_headers}"]
# (a GPU)
RUN_ALL_SKIP = {"linux": "needs a Windows Vulkan device", "windows": "needs a Windows Vulkan device"}
ROOT=Path(__file__).resolve().parents[1]
def block(text,marker):
start=text.index(marker);end=text.index('{',start)+1;depth=1
while depth:
depth+=(text[end]=='{')-(text[end]=='}');end+=1
return text[start:end]
def main():
p=argparse.ArgumentParser(description=__doc__);p.add_argument('--headers',type=Path,required=True);p.add_argument('--cxx',default='clang++');a=p.parse_args()
text=(ROOT/'gpu/vk/vk_texture.cpp').read_text(encoding='utf-8')
start=text.index(' const VkFormatProperties fp = format_props(x.format);')
end=text.index(' if (x.type == VK_IMAGE_TYPE_2D',start)
flags=text[start:end]
fixture=r'''#define WIN32_LEAN_AND_MEAN
#define VK_NO_PROTOTYPES
#include <windows.h>
#include <vulkan/vulkan.h>
#include <cstdio>
#include <cstdlib>
#include <vector>
#define CHECK(x) do{if(!(x)){std::fprintf(stderr,"FAIL:%d %s\n",__LINE__,#x);std::exit(1);}}while(0)
#define API_LIST(X) X(vkCreateInstance) X(vkDestroyInstance) X(vkEnumeratePhysicalDevices) X(vkGetPhysicalDeviceProperties) X(vkGetPhysicalDeviceFeatures) X(vkGetPhysicalDeviceQueueFamilyProperties) X(vkGetPhysicalDeviceMemoryProperties) X(vkGetPhysicalDeviceFormatProperties) X(vkGetPhysicalDeviceImageFormatProperties) X(vkCreateDevice) X(vkDestroyDevice) X(vkCreateImage) X(vkDestroyImage) X(vkGetImageMemoryRequirements) X(vkAllocateMemory) X(vkFreeMemory) X(vkBindImageMemory) X(vkCreateImageView) X(vkDestroyImageView)
#define DECLARE(n) PFN_##n n;
API_LIST(DECLARE)
struct {VkDevice device;VkPhysicalDevice physical;} dev{};
struct Texture {VkImage image;VkFormat format;VkImageUsageFlags usage;VkSampleCountFlagBits samples;VkImageType type;} ;
VkFormatProperties format_props(VkFormat f){VkFormatProperties p{};vkGetPhysicalDeviceFormatProperties(dev.physical,f,&p);return p;}
''' + block(text,'bool is_bc(VkFormat f) {')+'\n'+block(text,'bool is_astc(VkFormat f) {')+'\n'+block(text,'VkImageView make_view(')+r'''
int main() {
HMODULE loader=LoadLibraryA("vulkan-1.dll");CHECK(loader);
#define LOAD(n) n=(PFN_##n)GetProcAddress(loader,#n);CHECK(n);
API_LIST(LOAD)
VkApplicationInfo app{VK_STRUCTURE_TYPE_APPLICATION_INFO};app.apiVersion=VK_API_VERSION_1_3;
VkInstanceCreateInfo ici{VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO};ici.pApplicationInfo=&app;
VkInstance instance{};CHECK(vkCreateInstance(&ici,nullptr,&instance)==VK_SUCCESS);
uint32_t count=0;CHECK(vkEnumeratePhysicalDevices(instance,&count,nullptr)==VK_SUCCESS && count);
std::vector<VkPhysicalDevice> devices(count);CHECK(vkEnumeratePhysicalDevices(instance,&count,devices.data())==VK_SUCCESS);
dev.physical=devices[0];
for(auto d:devices){VkPhysicalDeviceProperties p{};vkGetPhysicalDeviceProperties(d,&p);if(p.vendorID==0x1002)dev.physical=d;}
VkPhysicalDeviceProperties props{};vkGetPhysicalDeviceProperties(dev.physical,&props);std::printf("device: %s\n",props.deviceName);
vkGetPhysicalDeviceQueueFamilyProperties(dev.physical,&count,nullptr);std::vector<VkQueueFamilyProperties> queues(count);
vkGetPhysicalDeviceQueueFamilyProperties(dev.physical,&count,queues.data());uint32_t family=0;
while(family<count && !(queues[family].queueFlags & VK_QUEUE_GRAPHICS_BIT))family++;CHECK(family<count);
float priority=1;VkDeviceQueueCreateInfo qi{VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO};qi.queueFamilyIndex=family;qi.queueCount=1;qi.pQueuePriorities=&priority;
VkPhysicalDeviceFeatures features{};vkGetPhysicalDeviceFeatures(dev.physical,&features);CHECK(features.textureCompressionBC);
VkDeviceCreateInfo di{VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO};di.queueCreateInfoCount=1;di.pQueueCreateInfos=&qi;di.pEnabledFeatures=&features;
CHECK(vkCreateDevice(dev.physical,&di,nullptr,&dev.device)==VK_SUCCESS);
VkPhysicalDeviceMemoryProperties memory{};vkGetPhysicalDeviceMemoryProperties(dev.physical,&memory);
struct Case{VkFormat compressed,raw;int bytes;} cases[]={{VK_FORMAT_BC1_RGBA_UNORM_BLOCK,VK_FORMAT_R32G32_UINT,8},
{VK_FORMAT_BC3_UNORM_BLOCK,VK_FORMAT_R32G32B32A32_UINT,16},{VK_FORMAT_BC4_UNORM_BLOCK,VK_FORMAT_R16G16B16A16_UINT,8},
{VK_FORMAT_BC5_UNORM_BLOCK,VK_FORMAT_R32G32B32A32_UINT,16},{VK_FORMAT_BC7_UNORM_BLOCK,VK_FORMAT_R32G32B32A32_UINT,16}};
int storage_views=0;
for(auto c:cases){
Texture x{};x.format=c.compressed;x.type=VK_IMAGE_TYPE_2D;x.samples=VK_SAMPLE_COUNT_1_BIT;bool depth=false;int bpb=c.bytes;
''' +flags+r'''
const bool storage=(x.usage & VK_IMAGE_USAGE_STORAGE_BIT)!=0;
VkImageCreateInfo ci{VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO};ci.flags=flags;ci.imageType=VK_IMAGE_TYPE_2D;ci.format=x.format;
ci.extent={32,16,1};ci.mipLevels=3;ci.arrayLayers=2;ci.samples=x.samples;ci.tiling=VK_IMAGE_TILING_OPTIMAL;ci.usage=x.usage;
CHECK(vkCreateImage(dev.device,&ci,nullptr,&x.image)==VK_SUCCESS);
VkMemoryRequirements req{};vkGetImageMemoryRequirements(dev.device,x.image,&req);uint32_t mt=0;
while(mt<memory.memoryTypeCount && !(req.memoryTypeBits & (1u<<mt)))mt++;CHECK(mt<memory.memoryTypeCount);
VkMemoryAllocateInfo ai{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};ai.allocationSize=req.size;ai.memoryTypeIndex=mt;
VkDeviceMemory mem{};CHECK(vkAllocateMemory(dev.device,&ai,nullptr,&mem)==VK_SUCCESS);CHECK(vkBindImageMemory(dev.device,x.image,mem,0)==VK_SUCCESS);
VkComponentMapping cm{};
auto sampled=make_view(&x,VK_IMAGE_VIEW_TYPE_2D_ARRAY,c.compressed,VK_IMAGE_ASPECT_COLOR_BIT,0,3,0,2,cm,VK_IMAGE_USAGE_SAMPLED_BIT);CHECK(sampled);
auto raw=make_view(&x,VK_IMAGE_VIEW_TYPE_2D,c.raw,VK_IMAGE_ASPECT_COLOR_BIT,1,1,1,1,cm,VK_IMAGE_USAGE_STORAGE_BIT);CHECK(bool(raw)==storage);storage_views+=bool(raw);
CHECK(!make_view(&x,VK_IMAGE_VIEW_TYPE_2D,c.raw,VK_IMAGE_ASPECT_COLOR_BIT,0,2,0,1,cm,VK_IMAGE_USAGE_STORAGE_BIT));
CHECK(!make_view(&x,VK_IMAGE_VIEW_TYPE_2D_ARRAY,c.raw,VK_IMAGE_ASPECT_COLOR_BIT,0,1,0,2,cm,VK_IMAGE_USAGE_STORAGE_BIT));
CHECK(!make_view(&x,VK_IMAGE_VIEW_TYPE_2D,c.raw,VK_IMAGE_ASPECT_COLOR_BIT,0,1,0,1,cm,0));
CHECK(!make_view(&x,VK_IMAGE_VIEW_TYPE_2D,c.raw,VK_IMAGE_ASPECT_COLOR_BIT,0,1,0,1,cm,VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT));
if(raw)vkDestroyImageView(dev.device,raw,nullptr);vkDestroyImageView(dev.device,sampled,nullptr);vkDestroyImage(dev.device,x.image,nullptr);vkFreeMemory(dev.device,mem,nullptr);
}
vkDestroyDevice(dev.device,nullptr);vkDestroyInstance(instance,nullptr);FreeLibrary(loader);
CHECK(storage_views>0);std::printf("PASS: five BC formats, %d raw storage views, sampled arrays/mips, rejected empty/unsupported usages\n",storage_views);
}
'''
work=ROOT/'work';work.mkdir(exist_ok=True)
with tempfile.TemporaryDirectory(prefix='bc-views-',dir=work) as temp:
directory=Path(temp).resolve();directory.relative_to(work.resolve());cpp=directory/'fixture.cpp';exe=directory/'fixture.exe';cpp.write_text(fixture,encoding='utf-8')
subprocess.run([a.cxx,'-std=c++20','-O2','-I'+str(a.headers),str(cpp),'-o',str(exe)],check=True)
r=subprocess.run([str(exe)],capture_output=True,text=True);print(r.stdout+r.stderr,end='');r.check_returncode()
assert 'Validation Error' not in r.stderr+r.stdout
if __name__=='__main__':main()

View File

@@ -0,0 +1,69 @@
"""Check actual uniform binding ranges include generated global-buffer table reads.
python nx2/test_cbuf_table_extent.py [--cxx clang++]
No GPU or game files are needed.
"""
import argparse
from pathlib import Path
import subprocess
import tempfile
RUN_ALL_ARGS = ["--cxx", "g++"]
ROOT=Path(__file__).resolve().parents[1]
def main():
p=argparse.ArgumentParser(description=__doc__);p.add_argument('--cxx',default='clang++');a=p.parse_args()
source=(ROOT/'gpu/vk/vk_draw.cpp').read_text(encoding='utf-8')
start=source.index('VkDescriptorBufferInfo uniform_range(')
start=source.index('uint32_t used = index < Shader::Info::MAX_CBUFS',start)
stop=source.index('if (index == 0 || index == 2)',start)
used=source[start:stop]
begin=source.index('const uint32_t n =',stop);end=source.index(';',begin)+1
rounded=source[begin:end]
fixture=r'''
#include <algorithm>
#include <array>
#include <cstdint>
#include <cstdio>
#include <vector>
struct Descriptor { uint32_t cbuf_index,cbuf_offset,count; };
struct Info { static constexpr uint32_t MAX_CBUFS=18;bool uses_global_memory=false;
std::array<uint32_t,MAX_CBUFS> constant_buffer_used_sizes{};std::vector<Descriptor> storage_buffers_descriptors; };
namespace Shader { using Info=::Info; }
struct Stage { Info info; };
uint32_t extent(const Stage& st,uint32_t index) {
''' + used + rounded + r'''
return n;
}
int main() {
unsigned cases=0;
for(uint32_t size : {0x690u,0x6a0u,0x780u,0x790u}) {
Stage st;st.info.uses_global_memory=true;st.info.constant_buffer_used_sizes[0]=size;
st.info.storage_buffers_descriptors={{0,0x880,3}};
if(extent(st,0)!=0x8b0){std::fprintf(stderr,"generated table needs0x8b0; bound0x%x\n",extent(st,0));return 1;}
++cases;
st.info.uses_global_memory=false;
if(extent(st,0)!=size)return 2;
++cases;
}
Stage st;st.info.uses_global_memory=true;st.info.constant_buffer_used_sizes[4]=0x100;
st.info.storage_buffers_descriptors={{0,0x880,3},{4,0xe0,16},{2,0x20,1}};
if(extent(st,4)!=0x1e0 || extent(st,2)!=0x30)return 3;
cases+=2;
st.info.constant_buffer_used_sizes[0]=0xa00;
if(extent(st,0)!=0xa00)return 4;
st.info.storage_buffers_descriptors.clear();st.info.constant_buffer_used_sizes[0]=0;
if(extent(st,0)!=16 || extent(st,99)!=16)return 5;
cases+=3;
std::printf("PASS: %u generated-table, stage-range, non-global, bank and minimum-size cases\n",cases);
}
'''
work=ROOT/'work';work.mkdir(exist_ok=True)
with tempfile.TemporaryDirectory(prefix='cbuf-extent-',dir=work) as t:
folder=Path(t).resolve();folder.relative_to(work.resolve());cpp=folder/'fixture.cpp';exe=folder/'fixture.exe';cpp.write_text(fixture,encoding='utf-8')
subprocess.run([a.cxx,'-std=c++20','-O2',str(cpp),'-o',str(exe)],check=True)
subprocess.run([str(exe)],check=True)
if __name__=='__main__':main()

112
nx2/test_coop_spirv.py Normal file
View File

@@ -0,0 +1,112 @@
"""Validate cooperative-matrix capability setup without a GPU or game files.
python nx2/test_coop_spirv.py --cxx clang++ --headers <SPIRV-Headers/include> --spirv-val <spirv-val>
Uses this repository's Sirit and the actual self-test builder/native-emitter setup.
"""
import argparse
from pathlib import Path
import subprocess
import tempfile
RUN_ALL_ARGS = ["--headers", "{spirv_headers}", "--spirv-val", "{spirv_val}"]
# (spirv-val: SPIRV-Tools comes with the Linux runner's packages, not on Windows)
RUN_ALL_SKIP = {"windows": "needs spirv-val: checked on Linux"}
ROOT = Path(__file__).resolve().parents[1]
def block(text, marker):
start = text.index(marker)
end = text.index('{', start) + 1
depth = 1
while depth:
depth += (text[end] == '{') - (text[end] == '}')
end += 1
return text[start:end]
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--cxx', default='clang++')
parser.add_argument('--headers', type=Path, required=True)
parser.add_argument('--spirv-val', default='spirv-val')
args = parser.parse_args()
vk = ROOT / 'gpu/vk'
source = (vk / 'vk_coopmat.cpp').read_text(encoding='utf-8')
builder = source[source.index('struct TestShader {'):source.index('// one dispatch of a 32-lane shader')]
emitter = (vk / 'eden/shader_recompiler/backend/spirv/emit_spirv.cpp').read_text(encoding='utf-8')
setup = block(emitter, 'if (info.uses_cooperative_matrix && !mma_software)')
fixture = '''#include <cstdio>
#include <cstdlib>
#include <functional>
#include <string>
#include <vector>
#include <sirit/sirit.h>
''' + builder + '''
std::vector<uint32_t> program(bool cooperative, bool mma_software, bool physical = false) {
using namespace Sirit;
Module ctx{0x00010600};
ctx.AddCapability(spv::Capability::Shader);
const auto addressing = physical ? spv::AddressingModel::PhysicalStorageBuffer64 : spv::AddressingModel::Logical;
ctx.SetMemoryModel(addressing, spv::MemoryModel::GLSL450);
if (physical) {
ctx.AddCapability(spv::Capability::PhysicalStorageBufferAddresses);
ctx.AddExtension("SPV_KHR_physical_storage_buffer");
}
struct { bool uses_cooperative_matrix; } info{cooperative};
''' + setup + '''
const Id void_t = ctx.TypeVoid(), u32 = ctx.TypeUInt(32);
const Id main = ctx.OpFunction(void_t, spv::FunctionControlMask::MaskNone, ctx.TypeFunction(void_t));
ctx.AddLabel();
// Native programs can contain device-scope barriers (emit_spirv_barriers.cpp).
const auto semantics = spv::MemorySemanticsMask::AcquireRelease | spv::MemorySemanticsMask::UniformMemory |
spv::MemorySemanticsMask::WorkgroupMemory | spv::MemorySemanticsMask::AtomicCounterMemory |
spv::MemorySemanticsMask::ImageMemory;
ctx.OpMemoryBarrier(ctx.Constant(u32, uint32_t(spv::Scope::Device)), ctx.Constant(u32, uint32_t(semantics)));
ctx.OpReturn(); ctx.OpFunctionEnd();
ctx.AddEntryPoint(spv::ExecutionModel::GLCompute, main, "main");
ctx.AddExecutionMode(main, spv::ExecutionMode::LocalSize, 32U, 1U, 1U);
auto code = ctx.Assemble();
const auto model = cooperative && !mma_software ? spv::MemoryModel::VulkanKHR : spv::MemoryModel::GLSL450;
for (size_t i = 5; i < code.size(); i += code[i] >> 16)
if ((code[i] & 0xffff) == uint32_t(spv::Op::OpMemoryModel) &&
(code[i + 1] != uint32_t(addressing) || code[i + 2] != uint32_t(model))) {
std::fprintf(stderr, "addressing or memory model changed unexpectedly\\n"); std::exit(6);
}
return code;
}
int main(int argc, char** argv) {
if (argc != 2) return 2;
const std::vector<std::vector<uint32_t>> modules{
build_shader([](TestShader&){}), program(true, false), program(true, true), program(false, false),
program(true, false, true), program(true, true, true), program(false, false, true)};
for (unsigned i = 0; i < modules.size(); i++) {
const std::string name = std::string(argv[1]) + "/case" + std::to_string(i) + ".spv";
FILE* f = std::fopen(name.c_str(), "wb"); if (!f) return 3;
const auto n = std::fwrite(modules[i].data(), 4, modules[i].size(), f);
std::fclose(f); if (n != modules[i].size()) return 4;
}
}
'''
work = ROOT / 'work'
work.mkdir(exist_ok=True)
with tempfile.TemporaryDirectory(prefix='coop-spirv-', dir=work) as tmp:
directory = Path(tmp).resolve()
directory.relative_to(work.resolve())
cpp = directory / 'fixture.cpp'
cpp.write_text(fixture, encoding='utf-8')
sirit = vk / 'sirit'
exe = directory / 'fixture.exe'
command = [args.cxx, '-std=c++20', '-O1', '-I' + str(sirit / 'include'), '-I' + str(sirit / 'src'),
'-I' + str(args.headers), str(cpp), *map(str, sorted((sirit / 'src').rglob('*.cpp'))), '-o', str(exe)]
subprocess.run(command, check=True)
subprocess.run([str(exe), str(directory)], check=True)
for index, name in enumerate(('matrix self-test', 'native matrix with device barrier', 'software matrix', 'ordinary shader',
'native matrix with physical addressing', 'software matrix with physical addressing',
'ordinary shader with physical addressing')):
subprocess.run([args.spirv_val, '--target-env', 'vulkan1.3', str(directory / ('case%d.spv' % index))], check=True)
print('PASS:', name)
if __name__ == '__main__':
main()

View File

@@ -0,0 +1,62 @@
"""Check actual vertex runtime setup and Eden's fixed-size emitter; no game/GPU required.
python nx2/test_default_point_size.py --headers <SPIRV-Headers/include> [--cxx clang++] [--spirv-val spirv-val]
"""
import argparse
from pathlib import Path
import subprocess,tempfile
RUN_ALL_ARGS = ["--headers", "{spirv_headers}", "--spirv-val", "{spirv_val}"]
# (spirv-val: SPIRV-Tools comes with the Linux runner's packages, not on Windows)
RUN_ALL_SKIP = {"windows": "needs spirv-val: checked on Linux"}
ROOT=Path(__file__).resolve().parents[1]
def block(text,marker):
start=text.index(marker);end=text.index('{',start)+1;depth=1
while depth:
depth+=(text[end]=='{')-(text[end]=='}');end+=1
return text[start:end]
def main():
p=argparse.ArgumentParser(description=__doc__);p.add_argument('--headers',type=Path,required=True);p.add_argument('--cxx',default='clang++');p.add_argument('--spirv-val',default='spirv-val');a=p.parse_args()
source=(ROOT/'gpu/vk/vk_shader.cpp').read_text(encoding='utf-8')
start=source.index(' Shader::RuntimeInfo vrt{};');end=source.index(' //',start)
setup=source[start:end]
helper=block((ROOT/'gpu/vk/eden/shader_recompiler/backend/spirv/emit_spirv_special.cpp').read_text(encoding='utf-8'),'void SetFixedPipelinePointSize(')
fixture=r'''#include <array>
#include <cstdio>
#include <cstdlib>
#include <optional>
#include <string>
#include <vector>
#include <sirit/sirit.h>
#define CHECK(x) do{if(!(x)){std::fprintf(stderr,"FAIL:%d %s\n",__LINE__,#x);std::exit(1);}}while(0)
using namespace Sirit;
namespace Shader {namespace IR {enum Attribute {PointSize};} struct RuntimeInfo {std::optional<float> fixed_state_point_size;};}
struct EmitContext:Module {Shader::RuntimeInfo runtime_info;Id output_point_size;Id Const(float f){return Constant(TypeFloat(32),f);}};
''' +helper+r'''
std::vector<uint32_t> run(bool writes) {
struct {struct {struct {std::array<bool,1> stores;}info;}program;} vertex{};auto vs=&vertex;vs->program.info.stores[0]=writes;
''' +setup+r'''
CHECK(writes ? !vrt.fixed_state_point_size : vrt.fixed_state_point_size==1.0f);
EmitContext ctx;ctx.runtime_info=vrt;ctx.AddCapability(spv::Capability::Shader);
ctx.SetMemoryModel(spv::AddressingModel::Logical,spv::MemoryModel::GLSL450);
const Id f32=ctx.TypeFloat(32),ptr=ctx.TypePointer(spv::StorageClass::Output,f32);
ctx.output_point_size=ctx.AddGlobalVariable(ptr,spv::StorageClass::Output);ctx.Decorate(ctx.output_point_size,spv::Decoration::BuiltIn,spv::BuiltIn::PointSize);
const Id void_t=ctx.TypeVoid(),main=ctx.OpFunction(void_t,spv::FunctionControlMask::MaskNone,ctx.TypeFunction(void_t));ctx.AddLabel();
if(writes)ctx.OpStore(ctx.output_point_size,ctx.Const(2.0f));
SetFixedPipelinePointSize(ctx);ctx.OpReturn();ctx.OpFunctionEnd();ctx.AddEntryPoint(spv::ExecutionModel::Vertex,main,"main",ctx.output_point_size);
auto words=ctx.Assemble();unsigned stores=0;for(size_t i=5;i<words.size();i+=words[i]>>16)stores+=(words[i]&65535)==62;
CHECK(stores==1);return words;
}
int main(int argc,char**argv){CHECK(argc==2);for(bool writes:{false,true}){auto words=run(writes);std::string path=std::string(argv[1])+"/"+(writes?"shader":"default")+".spv";FILE*f=std::fopen(path.c_str(),"wb");CHECK(f);CHECK(std::fwrite(words.data(),4,words.size(),f)==words.size());std::fclose(f);}puts("PASS: missing PointSize writes default1; explicit shader size retained with no extra store");}
'''
work=ROOT/'work';work.mkdir(exist_ok=True)
with tempfile.TemporaryDirectory(prefix='point-size-',dir=work) as temp:
d=Path(temp).resolve();d.relative_to(work.resolve());cpp=d/'fixture.cpp';exe=d/'fixture.exe';cpp.write_text(fixture,encoding='utf-8')
sirit=ROOT/'gpu/vk/sirit'
sources=sorted((sirit/'src').glob('*.cpp'))+sorted((sirit/'src/instructions').glob('*.cpp'))
subprocess.run([a.cxx,'-std=c++20','-O1','-D_CRT_SECURE_NO_WARNINGS','-I'+str(a.headers),'-I'+str(sirit/'include'),'-I'+str(sirit/'src'),str(cpp),*map(str,sources),'-o',str(exe)],check=True)
subprocess.run([str(exe),str(d)],check=True)
for module in d.glob('*.spv'):subprocess.run([a.spirv_val,'--target-env','vulkan1.3',str(module)],check=True)
if __name__=='__main__':main()

View File

@@ -0,0 +1,54 @@
"""Validate real texture-pass depth sampling without game files or a GPU.
python nx2/test_depth_sample_types.py --headers <SPIRV-Headers/include> --spirv-val spirv-val [--cxx clang++]
Builds the same Eden IR/SPIR-V and Sirit sources as the renderer, with its compatibility headers.
"""
# SPDX-License-Identifier: GPL-3.0-or-later
import argparse
from pathlib import Path
import subprocess
import tempfile
RUN_ALL_ARGS = ["--cxx", "{cxx}", "--headers", "{spirv_headers}", "--spirv-val", "{spirv_val}"]
# (spirv-val: SPIRV-Tools comes with the Linux runner's packages, not on Windows)
RUN_ALL_SKIP = {"windows": "needs spirv-val: checked on Linux"}
ROOT = Path(__file__).resolve().parents[1]
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--cxx', default='clang++')
parser.add_argument('--headers', type=Path, required=True)
parser.add_argument('--spirv-val', default='spirv-val')
args = parser.parse_args()
vk = ROOT / 'gpu/vk'
sources = sorted({p for directory in ('eden/shader_recompiler/backend/spirv',
'eden/shader_recompiler/frontend/ir', 'eden/shader_recompiler/ir_opt', 'sirit/src')
for p in (vk / directory).rglob('*.cpp')})
flags = ['-std=c++20', '-O1', '-D_CRT_SECURE_NO_WARNINGS', '-DVK_NO_PROTOTYPES', '-DMKWVK_INCLUDES=2']
for header in ('algorithm', 'cstdio', 'cstdlib', 'cstring', 'functional', 'string', 'vector'):
flags += ['-include', header]
flags += ['-I' + str(p) for p in (vk / 'compat', ROOT / 'gpu/compat', vk / 'eden',
ROOT / 'gpu/third_party/suyu', vk / 'sirit/include', vk / 'sirit/src', args.headers)]
work = ROOT / 'work'
work.mkdir(exist_ok=True)
with tempfile.TemporaryDirectory(prefix='depth-types-', dir=work) as temp:
folder = Path(temp).resolve()
exe = folder / 'fixture.exe'
subprocess.run([args.cxx, *flags, str(ROOT / 'tests/depth_sample_types_ir.cpp'),
*map(str, sources), '-o', str(exe)], check=True)
emitted = subprocess.run([str(exe), str(folder)], check=True, capture_output=True, text=True)
print(emitted.stdout, end='')
for shape in ('2d', 'array'):
floating, integer = (folder / f'{shape}-{kind}.spv' for kind in ('float', 'integer'))
for module in (floating, integer):
subprocess.run([args.spirv_val, '--target-env', 'vulkan1.3', str(module)], check=True)
assert floating.read_bytes() == integer.read_bytes(), f'{shape}: irrelevant integer answer changed depth SPIR-V'
lines = emitted.stdout.splitlines()
assert len(lines) == 4 and all(line.endswith('integer-queries=0') for line in lines), emitted.stdout
print('PASS: 2D/2D-array depth sampling ignores integer-format answers; all four modules validate')
if __name__ == '__main__':
main()

153
nx2/test_dll.py Normal file
View File

@@ -0,0 +1,153 @@
"""The production build against Unicorn: random functions run from work/build/hookshot_game.dll itself.
test_leaf.py compiles the functions it picks into its own harness. This test picks the same way (self-contained
functions, or with --calls whole call trees) and checks them in the real game library instead: built by
nx2/build.py from the full lift, every chunk at -O1 with the loop-header polls on, reached only through the DLL's
emulator ABI (nx2/rt/mkw_host.h: mkw_recomp_attach, mkw_recomp_lookup), the way Eglantine's ArmRecomp uses it.
The host program maps the image and the scratch area at the DLL's guest window (MkwGameInfo::mem_base), attaches a
host whose poll counts and whose trap raises an exception the case loop catches, sets every case's poll flag (so the
first loop header of each function calls the host), and writes the same results as test_leaf's harness.
Usage: python -I nx2/test_dll.py [--count 2000] [--states 3] [--seed 7] [--calls] (after build.py --dll)
"""
import os
import subprocess
import sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # (python -I leaves the script folder off)
import kit
import test_leaf
import toolchain
# (the game: this runs the lifted code of a real dump against Unicorn)
RUN_ALL_SKIP = {
"linux": "needs a full game build (work/build/hookshot_game.dll) and Unicorn",
"windows": "needs a full game build (work/build/hookshot_game.dll) and Unicorn",
}
DLL = os.path.join(kit.WORK, "build", "hookshot_game.dll")
HOST = r"""
#include "a64.h"
#include "mkw_host.h"
#include <stdio.h>
#include <stdlib.h>
#include <windows.h>
static const uint64_t roots[] = {%(roots)s};
#define IMG_BASE %(img_base)dULL
#define IMG_SIZE %(img_size)dULL
#define SCR_BASE %(scratch_base)dULL
#define SCR_SIZE %(scratch)dULL
static const uint64_t regions[][2] = {%(regions)s};
#define NREG (sizeof regions / sizeof regions[0])
static uint8_t *p_img, *p_scr;
static uint64_t polls, traps;
static void h_svc(void *u, struct GuestContext *c, uint32_t imm) { (void)u; ((Ctx *)c)->halted = 11; RaiseException(0xE0000002, 0, 0, NULL); }
static void h_poll(void *u, struct GuestContext *c) { (void)u; (void)c; polls++; }
static void h_trap(void *u, struct GuestContext *c, uint64_t pc, uint32_t insn, const char *why) {
(void)u; (void)pc; (void)insn; (void)why; ((Ctx *)c)->halted = 9; traps++; RaiseException(0xE0000001, 0, 0, NULL);
}
static int h_miss(void *u, struct GuestContext *c, uint64_t t) { (void)u; (void)c; (void)t; return 0; }
static uint64_t h_cntpct(void *u) { (void)u; return 0; }
static uint8_t *pristine(uint64_t a) { return a >= SCR_BASE ? p_scr + (a - SCR_BASE) : p_img + (a - IMG_BASE); }
static uint64_t page_hash(const uint8_t *p) {
const uint64_t *w = (const uint64_t *)p; uint64_t a = 0, b = 0;
for (uint64_t i = 0; i < 512; i++) { a += w[i] * (2 * i + 1); b += w[i] ^ (i * 0x9E3779B97F4A7C15ULL); }
return a ^ b;
}
static void make_scratch(uint64_t seed, uint64_t *out) {
for (uint64_t i = 0; i < SCR_SIZE / 8; i++) {
uint64_t z = seed + (i + 1) * 0x9E3779B97F4A7C15ULL;
z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ULL; z = (z ^ (z >> 27)) * 0x94D049BB133111EBULL; z ^= z >> 31;
out[i] = (z & 3) == 0 ? ((SCR_BASE + (z >> 12) %% (SCR_SIZE - 256)) & ~7ULL) : z;
}
}
static int run(MkwFn f, Ctx *c) {
__try { f((struct GuestContext *)c); } __except (EXCEPTION_EXECUTE_HANDLER) {
DWORD code = GetExceptionCode(); return code == 0xE0000001 || code == 0xE0000002 ? 0 : (int)code; }
return 0;
}
int main(int argc, char **argv) {
HMODULE dll = LoadLibraryA(argv[4]);
if (!dll) { fprintf(stderr, "cannot load %%s\n", argv[4]); return 6; }
PFN_mkw_recomp_attach attach = (PFN_mkw_recomp_attach)GetProcAddress(dll, "mkw_recomp_attach");
PFN_mkw_recomp_lookup lookup = (PFN_mkw_recomp_lookup)GetProcAddress(dll, "mkw_recomp_lookup");
MkwHost host = {MKW_HOST_VERSION, sizeof(MkwGuestContext), NULL, h_svc, h_poll, h_trap, h_miss, h_cntpct};
MkwGameInfo info;
if (!attach || !lookup || attach(&host, &info)) { fprintf(stderr, "attach failed\n"); return 7; }
fprintf(stderr, "dll: %%u functions, image %%#llx, guest window %%#llx, polls %%u, main %%s\n", info.functions,
(unsigned long long)info.image_base, (unsigned long long)info.mem_base, info.polls, info.main_build_id);
if (info.mem_base != (uint64_t)(uintptr_t)A64_MEM || info.image_base != IMG_BASE) return 8;
FILE *img = fopen(argv[1], "rb"), *in = fopen(argv[2], "rb"), *out = fopen(argv[3], "wb");
if (VirtualAlloc(A64_PTR(IMG_BASE), IMG_SIZE, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE) != A64_PTR(IMG_BASE)) return 3;
if (VirtualAlloc(A64_PTR(SCR_BASE), SCR_SIZE, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE) != A64_PTR(SCR_BASE)) return 4;
if (fread(A64_PTR(IMG_BASE), 1, IMG_SIZE, img) != IMG_SIZE) return 5;
p_img = malloc(IMG_SIZE); p_scr = malloc(SCR_SIZE);
memcpy(p_img, A64_PTR(IMG_BASE), IMG_SIZE);
static uint8_t state[%(state)d];
unsigned idx; uint64_t seed;
while (fread(&idx, 4, 1, in) == 1 && fread(&seed, 8, 1, in) == 1 && fread(state, sizeof state, 1, in) == 1) {
make_scratch(seed, (uint64_t *)p_scr);
memcpy(A64_PTR(SCR_BASE), p_scr, SCR_SIZE);
Ctx c; memset(&c, 0, sizeof c);
memcpy(c.x, state, 256); memcpy(&c.n, state + 256, 4);
memcpy(&c.fpcr, state + 264, 8); memcpy(&c.fpsr, state + 272, 8); memcpy(c.vreg, state + 280, 512);
memcpy(&c.tpidr_el0, state + 792, 8); memcpy(&c.tpidrro_el0, state + 800, 8);
c.poll = 1;
MkwFn f = lookup(roots[idx]);
int fault = f ? run(f, &c) : -1;
memcpy(state, c.x, 256); memcpy(state + 256, &c.n, 4);
memcpy(state + 264, &c.fpcr, 8); memcpy(state + 272, &c.fpsr, 8); memcpy(state + 280, c.vreg, 512);
fwrite(&c.halted, 4, 1, out); fwrite(&fault, 4, 1, out); fwrite(state, sizeof state, 1, out);
uint32_t ndiff = 0; static uint64_t diff[2 * 8192];
for (size_t r = 0; r < NREG; r++)
for (uint64_t a = regions[r][0]; a < regions[r][1]; a += 4096)
if (memcmp(A64_PTR(a), pristine(a), 4096)) {
if (ndiff < 8192) { diff[2 * ndiff] = a; diff[2 * ndiff + 1] = page_hash(A64_PTR(a)); }
ndiff++;
memcpy(A64_PTR(a), pristine(a), 4096);
}
fwrite(&ndiff, 4, 1, out); fwrite(diff, 16, ndiff < 8192 ? ndiff : 8192, out);
}
fclose(out);
fprintf(stderr, "host: polls %%llu, traps %%llu\n", (unsigned long long)polls, (unsigned long long)traps);
return 0;
}
"""
def build_host(tmp, P, roots, funcs, meta, img_size, addr64):
"""(replaces test_leaf.build) a host program for the DLL instead of a harness with the functions compiled in"""
rt64 = os.path.join(kit.WORK, "build", "rt64")
gen = os.path.join(kit.WORK, "gen")
src = HOST % {
"roots": ",".join("%#xULL" % e for _, e, _ in roots),
"regions": ",".join("{%#xULL,%#xULL}" % r for r in test_leaf.writable_regions(meta)),
"scratch": test_leaf.SCRATCH_SIZE, "scratch_base": test_leaf.SCRATCH, "img_base": meta["base"],
"img_size": img_size, "state": test_leaf.STATE.size}
path = os.path.join(tmp, "host.c")
open(path, "w", newline="\n").write(src)
exe = os.path.join(tmp, "host.exe")
cmd = [toolchain.need(toolchain.find("clang"))] + toolchain.native_target() + toolchain.native_link() + \
["-O1", "-g0", "-mcx16", "-Wno-everything", "-I" + rt64, "-I" + gen, "-I" + os.path.join(kit.HERE, "rt"),
path, "-o", exe]
subprocess.run(cmd, check=True)
# the host takes the DLL as its fourth argument: wrap it
bat = os.path.join(tmp, "host.bat")
open(bat, "w").write('@"%s" %%1 %%2 %%3 "%s"\n' % (exe, DLL))
return bat
def main():
if not os.path.exists(DLL):
raise SystemExit("no %s: run nx2/build.py --dll first" % DLL)
if "--seed" not in sys.argv:
sys.argv += ["--seed", "7"] # (other functions than test_leaf's default)
test_leaf.build = build_host
test_leaf.Picker.emit_only = True
return test_leaf.main()
if __name__ == "__main__":
sys.exit(main())

View File

@@ -0,0 +1,65 @@
"""Validate the actual DrawParameters capability block across input/profile combinations; no game or GPU.
python nx2/test_draw_parameters.py --headers <SPIRV-Headers/include> [--cxx clang++] [--spirv-val spirv-val]
"""
import argparse
from pathlib import Path
import subprocess,tempfile
RUN_ALL_ARGS = ["--headers", "{spirv_headers}", "--spirv-val", "{spirv_val}"]
# (spirv-val: SPIRV-Tools comes with the Linux runner's packages, not on Windows)
RUN_ALL_SKIP = {"windows": "needs spirv-val: checked on Linux"}
ROOT=Path(__file__).resolve().parents[1]
def main():
p=argparse.ArgumentParser(description=__doc__);p.add_argument('--headers',type=Path,required=True);p.add_argument('--cxx',default='clang++');p.add_argument('--spirv-val',default='spirv-val');a=p.parse_args()
source=(ROOT/'gpu/vk/eden/shader_recompiler/backend/spirv/emit_spirv.cpp').read_text(encoding='utf-8')
end=source.index('ctx.AddCapability(spv::Capability::DrawParameters);')
start=source.rfind('\n if (',0,end)+1;end=source.index('\n }',end)+6
capability=source[start:end]
fixture='''#include <array>
#include <cstdio>
#include <cstdlib>
#include <string>
#include <vector>
#include <sirit/sirit.h>
namespace IR { enum Attribute { InstanceId, VertexId, BaseInstance, BaseVertex, DrawID }; }
std::vector<uint32_t> program(unsigned mask,bool supported) {
using namespace Sirit;
// (EmitContext's own member the block reads: a task program's DrawIndex; off here, as for every vertex program)
struct Ctx : Module { using Module::Module; bool wants_mesh_first{}; } ctx{0x00010600};ctx.AddCapability(spv::Capability::Shader);
ctx.SetMemoryModel(spv::AddressingModel::Logical,spv::MemoryModel::GLSL450);
struct { bool support_vertex_instance_id; } profile{supported};
struct { std::array<bool,5> loads; } info{};
for(unsigned i=0;i<5;i++)info.loads[i]=(mask>>i)&1;
'''+capability+'''
const Id u32=ctx.TypeUInt(32),ptr=ctx.TypePointer(spv::StorageClass::Input,u32),void_t=ctx.TypeVoid();
std::vector<Id> inputs;
auto input=[&](spv::BuiltIn builtin){
Id v=ctx.AddGlobalVariable(ptr,spv::StorageClass::Input);ctx.Decorate(v,spv::Decoration::BuiltIn,builtin);inputs.push_back(v);
};
// DefineInputs emits these directly, or synthesizes Base* for Vulkan's Instance/VertexIndex path.
if(info.loads[IR::BaseInstance] || (!supported && info.loads[IR::InstanceId]))input(spv::BuiltIn::BaseInstance);
if(info.loads[IR::BaseVertex] || (!supported && info.loads[IR::VertexId]))input(spv::BuiltIn::BaseVertex);
if(info.loads[IR::DrawID])input(spv::BuiltIn::DrawIndex);
const Id main=ctx.OpFunction(void_t,spv::FunctionControlMask::MaskNone,ctx.TypeFunction(void_t));
ctx.AddLabel();ctx.OpReturn();ctx.OpFunctionEnd();ctx.AddEntryPoint(spv::ExecutionModel::Vertex,main,"main",inputs);
return ctx.Assemble();
}
int main(int argc,char**argv) {
if(argc!=2)return 1;
for(unsigned support=0;support<2;support++)for(unsigned mask=0;mask<32;mask++) {
auto code=program(mask,support!=0);std::string name=std::string(argv[1])+"/"+std::to_string(support*32+mask)+".spv";
FILE*f=std::fopen(name.c_str(),"wb");if(!f)return 2;
auto n=std::fwrite(code.data(),4,code.size(),f);std::fclose(f);if(n!=code.size())return 3;
}
}
'''
work=ROOT/'work';work.mkdir(exist_ok=True)
with tempfile.TemporaryDirectory(prefix='draw-parameters-',dir=work) as temp:
directory=Path(temp).resolve();directory.relative_to(work.resolve());cpp=directory/'fixture.cpp';exe=directory/'fixture.exe';cpp.write_text(fixture,encoding='utf-8');sirit=ROOT/'gpu/vk/sirit'
subprocess.run([a.cxx,'-std=c++20','-O1','-I'+str(sirit/'include'),'-I'+str(sirit/'src'),'-I'+str(a.headers),str(cpp),*map(str,sorted((sirit/'src').rglob('*.cpp'))),'-o',str(exe)],check=True)
subprocess.run([str(exe),str(directory)],check=True)
for case in range(64):subprocess.run([a.spirv_val,'--target-env','vulkan1.3',str(directory/f'{case}.spv')],check=True)
print('PASS:64 draw-parameter input/profile combinations')
if __name__=='__main__':main()

View File

@@ -0,0 +1,51 @@
"""Check that optional SSBO diagnostics never read an unbound guest address.
Run: python nx2/test_dump_ssbo_read.py [--cxx clang++]
"""
import argparse
from pathlib import Path
import subprocess
import tempfile
RUN_ALL_ARGS = ["--cxx", "g++"]
ROOT=Path(__file__).resolve().parents[1]
def main():
p=argparse.ArgumentParser(description=__doc__)
p.add_argument('--cxx',default='clang++')
a=p.parse_args()
source=(ROOT/'gpu/vk/vk_draw.cpp').read_text(encoding='utf-8')
start=source.index('const bool read = ',source.index('if (dump_on() && dump_ssbo_wanted(p.key)) {'))
read=source[start:source.index(';',start)+1]
fixture=r'''
#include <cstdint>
#include <cstdio>
#include <vector>
unsigned calls=0;
struct Host { bool gpu_read(uint64_t,void*,uint64_t){++calls;return true;} } instance;
Host* host=&instance;
bool sample(bool b,uint64_t addr,unsigned size) {
std::vector<uint32_t> words(size);
const uint64_t base=addr;
''' + read + r'''
return read;
}
int main() {
if(sample(false,0x1001234000ull,16) || calls){std::puts("FAIL: unbound address read by dump");return 1;}
if(sample(true,0,16) || calls)return 2;
if(sample(true,0x1001234000ull,0) || calls)return 3;
if(!sample(true,0x1001234000ull,16) || calls!=1)return 4;
std::puts("PASS: unbound, null, empty and bound diagnostic reads");
}
'''
work=ROOT/'work';work.mkdir(exist_ok=True)
with tempfile.TemporaryDirectory(prefix='dump-ssbo-',dir=work) as t:
folder=Path(t).resolve();folder.relative_to(work.resolve())
cpp,exe=folder/'fixture.cpp',folder/'fixture.exe'
cpp.write_text(fixture,encoding='utf-8')
subprocess.run([a.cxx,'-std=c++20','-O2',str(cpp),'-o',str(exe)],check=True)
subprocess.run([str(exe)],check=True)
if __name__=='__main__':main()

136
nx2/test_import_windows.py Normal file
View File

@@ -0,0 +1,136 @@
"""Run actual pool lookup/range guards/prefetch queueing with fake GPU/background dependencies.
No Vulkan, background import thread, SDK or game data is used.
python nx2/test_import_windows.py [--cxx clang++]
"""
import argparse,os
from pathlib import Path
import subprocess,tempfile
RUN_ALL_ARGS = ["--cxx", "g++"]
ROOT=Path(__file__).resolve().parents[1]
def block(text,marker):
start=text.index(marker);end=text.index('{',start)+1;depth=1
while depth:
depth+=(text[end]=='{')-(text[end]=='}');end+=1
return text[start:end]
def main():
p=argparse.ArgumentParser(description=__doc__);p.add_argument('--cxx',default='clang++');a=p.parse_args()
source=(ROOT/'gpu/vk/vk_memory.cpp').read_text(encoding='utf-8')
guard=source[source.index('bool import_pool(uint64_t lo, uint64_t hi, Pool& p) {'):source.index(' p.lo = lo;',source.index('bool import_pool('))]
lookup=block(source,'Pool* pool_lookup(uint64_t guest, uint64_t size) {')
chunking=source[source.index('uint64_t import_chunk() {'):source.index('Pool* pool_lookup(uint64_t guest, uint64_t size) {')]
cache=block(source,'Pool* pool_of(uint64_t guest, uint64_t size) {')
prefetch=block(source,'void prefetch_chunks(uint64_t lo, uint64_t hi, uint64_t chunk) {')
fixture='''#include <algorithm>
#include <chrono>
#include <condition_variable>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <deque>
#include <map>
#include <mutex>
#include <vector>
#define CHECK(x) do { if(!(x)){std::fprintf(stderr,"FAIL:%d %s\\n",__LINE__,#x);std::exit(1);} }while(0)
struct Pool { uint64_t lo=0,hi=0,range_lo=0,range_hi=0,address=0; bool prefetched=false; };
struct GuestPool { uint64_t lo,hi; };
std::vector<GuestPool> guest_pools;
std::map<uint64_t,Pool> imported;
Pool *last_pool=nullptr,*pool_cache[4096]{};
uint64_t max_import_size=128ull<<20;
int failures=0,imports=0,retires=0,bg_starts=0;
int64_t st_bg_waits=0,st_prefetch_used=0,st_prefetch_sent=0,st_prefetch_dropped=0;
struct { bool external_memory_host=true; } dev;
std::mutex bg_mu;std::condition_variable bg_done_cv,bg_cv;std::vector<uint64_t> bg_inflight;
std::deque<std::pair<uint64_t,uint64_t>> bg_queue;
void bg_start() { bg_starts++; } // Recording spy: no thread or GPU import is started.
void logf(const char*,...){} void bg_drain(){}
void guest_pages_set(uint64_t,uint64_t,uint64_t){}
void retire_pool(std::map<uint64_t,Pool>::iterator it) { imported.erase(it);last_pool=nullptr;std::fill(std::begin(pool_cache),std::end(pool_cache),nullptr);retires++; }
enum { VK_SUCCESS = 0 };
thread_local bool import_cooling = false;
thread_local int import_result = 0;
bool import_on_hold(uint64_t, uint64_t) { import_cooling = false; return false; }
void import_remember_failure(uint64_t, uint64_t) {}
'''+guard+'''
p.lo=lo;p.hi=hi;imports++;return true;
}
'''+prefetch+'\n'+chunking+'\n'+lookup+'\n'+cache+'''
int check_prefetch(int mode) {
const uint64_t base=0x1000000000ull,chunk=16ull<<20;
guest_pools.push_back({base,base+8*chunk});
dev.external_memory_host=mode!=2;
Pool seed;seed.lo=base+3*chunk;seed.hi=base+4*chunk;seed.prefetched=true;
auto &ready=imported.emplace(seed.lo,seed).first->second;
CHECK(pool_lookup(seed.lo+64,64)==&ready && !ready.prefetched && st_prefetch_used==1 && imports==0);
if(mode==1) {
CHECK(bg_queue.size()==2 && bg_starts==2 && st_prefetch_sent==2);
CHECK(bg_queue[0].first==base+4*chunk && bg_queue[0].second==base+5*chunk);
CHECK(bg_queue[1].first==base+2*chunk && bg_queue[1].second==base+3*chunk);
} else CHECK(bg_queue.empty() && bg_starts==0 && st_prefetch_sent==0);
CHECK(pool_lookup(seed.lo+128,64)==&ready && st_prefetch_used==1 && imports==0);
Pool plain;plain.lo=base+6*chunk;plain.hi=base+7*chunk;
auto &unprefetched=imported.emplace(plain.lo,plain).first->second;
CHECK(pool_lookup(plain.lo+64,64)==&unprefetched && !unprefetched.prefetched && st_prefetch_used==1 && imports==0);
CHECK(bg_starts==(mode==1?2:0) && st_prefetch_sent==(mode==1?2:0));
auto demand=pool_lookup(base+64,64);
CHECK(demand && !demand->prefetched && imports==1 && st_prefetch_used==1);
CHECK(bg_queue.size()==(mode==1?3u:0u) && bg_starts==(mode==1?3:0) && st_prefetch_sent==(mode==1?3:0));
CHECK(bg_inflight.size()==bg_queue.size() && st_prefetch_dropped==0 && st_bg_waits==0);
std::printf("PASS: prefetched first use, reuse, unprefetched lookup, neighbour queueing (mode %d)\\n",mode);
return 0;
}
int main(int argc,char **argv) {
if(argc>1) return check_prefetch(std::atoi(argv[1]));
const uint64_t base=0x1000000000ull,extent=0x99200000ull;
guest_pools.push_back({base,base+extent});
const auto chunk=uint64_t(std::atoi(std::getenv("RX_VK_IMPORT_CHUNK_MB")))<<20;
if(!chunk) {
CHECK(!pool_lookup(base+1234,64) && imports==0);
max_import_size=4ull<<30;
auto p=pool_lookup(base+1234,64);CHECK(p && p->lo==base && p->hi==base+extent);
} else {
const uint64_t address=base+1234,expected=address/chunk*chunk;
auto p=pool_lookup(address,64);CHECK(p && p->lo==expected && p->hi==expected+chunk && imports==1);
CHECK(pool_lookup(address+100,64)==p && imports==1);
p=pool_lookup(expected+4*chunk+123,64);CHECK(p && p->lo==expected+4*chunk && p->hi==expected+5*chunk && imports==2);
p=pool_lookup(expected+chunk-16,64);CHECK(p && p->lo==expected && p->hi==expected+2*chunk && retires==1);
if(chunk==(16ull<<20)) {
// a game pool across a chunk's end: its first binding imports both chunks (span_game_pools)
const uint64_t far=base+0x40000000ull,far2=base+0x50000000ull;
guest_pools.push_back({far+chunk-4096,far+chunk+4096});
guest_pools.push_back({far2+chunk-4096,far2+chunk+4096});
p=pool_lookup(far+chunk-64,32);CHECK(p && p->lo==far && p->hi==far+2*chunk);
CHECK(pool_lookup(far+chunk+64,32)==p); // (the binding across the end: no import again)
// a spanned import that fails (past the device's limit here) is made again without the span
const uint64_t limit=max_import_size;max_import_size=chunk;
p=pool_lookup(far2+chunk-64,32);CHECK(p && p->lo==far2 && p->hi==far2+chunk);
max_import_size=limit;
}
}
const int before=imports;
CHECK(!pool_lookup(base+extent+1,1));
CHECK(!pool_lookup(base+extent-1,2));
CHECK(!pool_lookup(base,UINT64_MAX));
CHECK(!pool_lookup(base,0));
CHECK(!pool_of(base,UINT64_MAX) && !pool_of(base,0) && imports==before);
guest_pools.push_back({UINT64_MAX-1024,UINT64_MAX});
if(chunk) CHECK(!pool_lookup(UINT64_MAX-16,8) && imports==before);
Pool rejected;CHECK(!import_pool(base,base,rejected) && !import_pool(base+1,base,rejected));
CHECK(!import_pool(base,base+max_import_size+1,rejected));
std::printf("PASS: request windows, reuse, overlap retirement, range/overflow guards, import limits (chunk %llu)\\n",(unsigned long long)chunk);
}
'''
work=ROOT/'work';work.mkdir(exist_ok=True)
with tempfile.TemporaryDirectory(prefix='import-windows-',dir=work) as temp:
directory=Path(temp).resolve();directory.relative_to(work.resolve());cpp=directory/'fixture.cpp';exe=directory/'fixture.exe';cpp.write_text(fixture,encoding='utf-8')
subprocess.run([a.cxx,'-std=c++20','-O2','-D_CRT_SECURE_NO_WARNINGS',str(cpp),'-o',str(exe)],check=True)
for chunk in ('16','3','0'):
subprocess.run([str(exe)],env=dict(os.environ,RX_VK_IMPORT_CHUNK_MB=chunk,RX_VK_PREFETCH_CHUNKS='0'),check=True)
for mode in ('1','0','2'):
subprocess.run([str(exe),mode],env=dict(os.environ,RX_VK_IMPORT_CHUNK_MB='16',RX_VK_PREFETCH_CHUNKS='0' if mode=='0' else '1'),check=True)
if __name__=='__main__':main()

106
nx2/test_insn.py Normal file
View File

@@ -0,0 +1,106 @@
"""Every instruction form the game uses, in Unicorn and as lifted C: a64-kit's tests/insn_test.py with the Switch 2
additions.
Differences from running the vendored test as is:
- the Switch 2 forms translate (insn.install()), and RETAx are returns (not sampled, like RET);
- Unicorn runs its "max" CPU model, which has the ARMv8.1+ forms (the default Cortex-A72 rejects LSE, CAS, LDAPR);
- the harness is built with -mcx16 (CASP on 128 bits);
- PAC and MTE forms are left out: here they are the identity by design (see insn.py), not the hardware's result;
- memory operands based on x29 / x30 get their scratch address too: capstone names them fp / lr, which
insn_test.mem_regs does not parse, so those cases ran on a random (often tiny) address that the harness
does not reset between cases (a false LDADD [x29] failure, 2026-10-31).
Usage: python -I nx2/test_insn.py [--modules main,sdk,...] [--per 6] [--states 12] [--only lse]
(--modules: every code module of the exefs but rtld by default)
Needs the module images of nso.py in <work>/modules (made on first use).
"""
import os
import subprocess
import sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # (python -I leaves the script folder off)
import kit
import insn
insn.install()
sys.path.insert(0, os.path.join(kit.KIT, "tests"))
import insn_test # noqa: E402
import toolchain # noqa: E402
import unicorn # noqa: E402
from unicorn import arm64_const as A # noqa: E402
# (the game: this runs the lifted code of a real dump against Unicorn)
RUN_ALL_SKIP = {
"linux": "needs the game's modules (work/dump) and Unicorn",
"windows": "needs the game's modules (work/dump) and Unicorn",
}
class _Unicorn:
"""insn_test's view of the unicorn module: every engine it makes runs the ARMv8.x "max" CPU"""
UcError = unicorn.UcError
UC_ARCH_ARM64 = unicorn.UC_ARCH_ARM64
UC_MODE_ARM = unicorn.UC_MODE_ARM
@staticmethod
def Uc(arch, mode):
uc = unicorn.Uc(arch, mode)
uc.ctl_set_cpu_model(A.UC_CPU_ARM64_MAX)
return uc
ONLY = {"lse": lambda w: (w & 0x3F200C00) == 0x38200000 or (w & 0x3FA07C00) == 0x08A07C00
or (w & 0xBFA07C00) == 0x08207C00,
"nx2": lambda w: insn.emit(w) is not None}
def mem_regs(w):
md = insn_test.capstone.Cs(insn_test.capstone.CS_ARCH_ARM64, insn_test.capstone.CS_MODE_ARM)
md.detail = True
ins = next(md.disasm(insn_test.struct.pack("<I", w), insn_test.CODE), None)
alias = {"fp": 29, "lr": 30, "x29": 29, "x30": 30}
for op in ins.operands if ins else ():
if op.type == insn_test.capstone.arm64.ARM64_OP_MEM:
def num(r):
if r is None:
return None
name = ins.reg_name(r)
if name in ("sp", "wsp"):
return "sp"
if name in alias:
return alias[name]
return int(name[1:]) if name[1:].isdigit() else None
return num(op.mem.base or None), num(op.mem.index or None)
return None
def main():
kit.wait_for_quiet('the instruction test')
only = None
if "--only" in sys.argv:
i = sys.argv.index("--only")
only = ONLY[sys.argv[i + 1]]
del sys.argv[i:i + 2]
modules = os.path.join(kit.WORK, "modules")
if not os.path.exists(os.path.join(modules, "main.json")):
subprocess.run([sys.executable, "-I", os.path.join(kit.KIT, "tools", "nso.py"), kit.need_exefs(), modules],
check=True, stdout=subprocess.DEVNULL)
if "--modules" not in sys.argv:
sys.argv += ["--modules", ",".join(kit.game_modules())]
insn_test.unicorn = _Unicorn
insn_test.mem_regs = mem_regs
base_target = toolchain.native_target
toolchain.native_target = lambda *a, **k: base_target(*a, **k) + ["-mcx16"]
base_sample = insn_test.sample
def sample(image_dir, mods, per, rng):
picked = [(k, w) for k, w in base_sample(image_dir, mods, per, rng) if not insn.is_identity_model(w)]
return [(k, w) for k, w in picked if only(w)] if only else picked
insn_test.sample = sample
sys.argv = [sys.argv[0], modules] + sys.argv[1:]
return insn_test.main()
if __name__ == "__main__":
sys.exit(main())

527
nx2/test_leaf.py Normal file
View File

@@ -0,0 +1,527 @@
"""Whole recompiled functions against Unicorn: random functions of the game's rtld, main, sdk and subsdk*, lifted exactly
as lift.py lifts them, run natively and in Unicorn (ARMv8.x "max" CPU) on the same relocated image.
Two kinds of functions:
- default: self-contained ones, whose lifted C calls nothing: no BL/BLR, SVC, tail call into another entry,
indirect BR, trap or untranslated word, no CNTPCT read (the clock differs), 3..600 instructions;
- --calls: functions with callees. A root qualifies when everything it reaches through BL, tail calls (B or
fallthrough into another entry) and PLT stubs is liftable the same way (no BLR, SVC, indirect BR, trap, CNTPCT),
at most --max-tree instructions and 64 functions in all. The whole call tree is lifted and runs natively, so this
tests calls, returns through LR, tail calls, PLT stubs bound across modules and register state across calls.
The image is laid out as Eglantine loads it (image.py, 0x80000000) and, by default, the scratch area lies above
4 GB (0x10_0000_0000) with the lifted code on the 64-bit guest window (rtgen.py): a 32-bit window would truncate
every scratch pointer. --addr32 runs a64-kit's own 32-bit model instead, with the scratch area below 4 GB.
Each function runs from several random states: X0-X7 a mix of pointers into the scratch area, small integers and
random 64-bit values; the scratch area random qwords of which a quarter are pointers back into it (so pointer
chasing goes a few levels deep); the stack and TPIDR(RO)_EL0 in it too; FPCR 0 or FZ.
Compared after the return: X0-X30, SP, V0-V31, FPSR (cumulative bits, reported apart), and every 4 KB page of the
image's .data/.bss and of the scratch area that ends up different (page set and contents). NZCV is not compared:
flags are dead at a return (the AAPCS64), and the lifter drops flag stores no later instruction reads. With --calls,
X16/X17 are not compared either (counted apart): a call through a PLT stub calls the bound target directly, without
the stub's `adrp x16 / ldr x17` (a64-kit's PLT model); they are the AAPCS64's intra-procedure-call scratch registers,
which any veneer or stub may clobber, so no caller can depend on them after a call. Cases where
Unicorn faults, does not return within 400,000 instructions, or touches an unmapped address are not comparable and
are skipped.
Usage: python -I nx2/test_leaf.py [--count 3000] [--states 4] [--seed 1] [--calls] [--max-tree 4000]
[--addr32] [--list]
"""
import argparse
import collections
import os
import random
import re
import struct
import subprocess
import sys
import tempfile
import time
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # (python -I leaves the script folder off)
import kit
import nso2 # noqa: F401
import insn
import lift_fixes
import poll
import image as eden_image
import rtgen
import a64emit
import a64_lift
import a64_rtgen
import toolchain
import numpy as np
import unicorn
from unicorn import arm64_const as A
# (the game: this runs the lifted code of a real dump against Unicorn)
RUN_ALL_SKIP = {
"linux": "needs the game's modules (work/dump) and Unicorn",
"windows": "needs the game's modules (work/dump) and Unicorn",
}
insn.install()
lift_fixes.install()
poll.install()
SCRATCH_SIZE = 0x100000
SENTINEL = 0x7F000000 # LR of the outermost call: a mapped page with no code, below the image
PAGE = 0x1000
STATE = struct.Struct("<32Q4B4xQQ64QQQ") # x0-x30, sp | nzcv | fpcr fpsr | v0-v31 (lo, hi) | tpidr tpidrro
XREG = [getattr(A, "UC_ARM64_REG_X%d" % i) for i in range(29)] + [A.UC_ARM64_REG_X29, A.UC_ARM64_REG_X30]
QREG = [getattr(A, "UC_ARM64_REG_Q%d" % i) for i in range(32)]
M64 = (1 << 64) - 1
BAD_TEXT = ("a64_trap(", "A64_IJUMP", "A64_ICALL", "a64_icall", "a64_ijump", "recomp_cntpct", "recomp_unhandled",
"a64_svc(")
REF = re.compile(r"\bf_([0-9a-f]{8})\(")
SCRATCH = 0x1000000000
def scratch_words(seed):
"""the scratch area of one case (splitmix64; harness.c makes the same)"""
i = np.arange(SCRATCH_SIZE // 8, dtype=np.uint64)
with np.errstate(over="ignore"):
z = np.uint64(seed) + (i + np.uint64(1)) * np.uint64(0x9E3779B97F4A7C15)
z = (z ^ (z >> np.uint64(30))) * np.uint64(0xBF58476D1CE4E5B9)
z = (z ^ (z >> np.uint64(27))) * np.uint64(0x94D049BB133111EB)
z = z ^ (z >> np.uint64(31))
ptr = np.uint64(SCRATCH) + ((z >> np.uint64(12)) % np.uint64(SCRATCH_SIZE - 256)) & ~np.uint64(7)
return np.where((z & np.uint64(3)) == 0, ptr, z)
def page_hash(b):
"""hash of one 4 KB page (harness.c: the same)"""
w = np.frombuffer(b, dtype=np.uint64)
i = np.arange(len(w), dtype=np.uint64)
with np.errstate(over="ignore"):
return int((w * (i * np.uint64(2) + np.uint64(1))).sum() ^ (w ^ (i * np.uint64(0x9E3779B97F4A7C15))).sum())
class Picker:
"""Walks, translates and emits functions as lift.py does, keeping what it has done for reuse across roots."""
def __init__(self, img):
self.img = img
self.L = a64_lift.Lifter(img)
self.texts = {}
self.emitted = {} # entry -> (text, callee entries) or None (not liftable here)
self.fpx = set()
def liftable_body(self, e, calls):
L = self.L
if e not in L.funcs:
L.funcs[e] = L.walk(e)
t = L.plt_target(e)
if t is not None:
L.plt[e] = t
body = L.funcs[e][0]
kinds = {k[0] for k, _ in body.values()}
bad = {"blr", "svc", "trap"} | (set() if calls else {"bl"})
if not body or kinds & bad:
return False
if e not in L.plt and any(k[0] == "br" and pc not in L.jt for pc, (k, _) in body.items()):
return False
return True
def emit(self, entries):
"""translate the words of these functions (one batch) and emit them"""
L = self.L
keys = {}
for e in entries:
for pc, (k, w) in L.funcs[e][0].items():
if k[0] == "op":
key = (w, pc) if a64_lift.pc_relative(w) else w
if key not in self.texts:
keys.setdefault(key, (pc, w))
if keys:
order = list(keys)
tx, fl = a64emit.translate([keys[x] for x in order])
T = lambda t: a64_lift.stack_stores(a64_lift.pack_flags(a64_lift.widen_vstores(a64_lift.vectorize_fpx(
a64_lift.outline_fpx(t, self.fpx)))))
self.texts.update(zip(order, zip((T(t) for t in tx), fl)))
for e in entries:
text, refs = L.emit_function(e, self.texts)
if any(b in text for b in BAD_TEXT):
self.emitted[e] = None
else:
self.emitted[e] = (text, {int(r[2:], 16) for r in refs})
def tree(self, root, calls, max_insns, max_funcs=64):
"""the root's call tree (entries), or None if some function in it is not liftable here"""
seen, todo = set(), [root]
while todo:
fresh = []
for e in todo:
if e in seen:
continue
seen.add(e)
if len(seen) > max_funcs or not self.liftable_body(e, calls):
return None
if e not in self.emitted:
fresh.append(e)
if sum(len(self.L.funcs[e][0]) for e in seen) > max_insns:
return None
if fresh:
self.emit(fresh)
nxt = []
for e in todo:
got = self.emitted.get(e)
if got is None:
return None
nxt += [r for r in got[1] if r not in seen]
todo = nxt
if not calls and self.emitted[root][1]:
return None
return seen
def module_shares(img):
"""how many of the drawn functions come from each module: main 55 %, sdk 30 %, the subsdk* modules the rest by
code size, rtld 2 % (mkw-recomp's split, for whatever modules this exefs has)"""
size = {n: hi - lo for lo, hi, n in img.code}
share = {"main": 0.55, "sdk": 0.30, "rtld": 0.02}
subs = [n for n in size if n.startswith("subsdk")]
rest = 1.0 - sum(v for k, v in share.items() if k in size)
for n in subs:
share[n] = rest * size[n] / max(1, sum(size[s] for s in subs))
return {k: v for k, v in share.items() if k in size}
def select(P, counts, rng, calls, max_tree):
img = P.img
by_mod = collections.defaultdict(list)
for e in sorted(img.entries):
by_mod[next(n for lo, hi, n in img.code if lo <= e < hi)].append(e)
roots, funcs = [], set()
for mod, want in counts.items():
cands = by_mod[mod]
rng.shuffle(cands)
got = 0
for e in cands[:want * 40]:
if got >= want:
break
if not calls:
if e not in P.L.funcs:
P.L.funcs[e] = P.L.walk(e)
if not 3 <= len(P.L.funcs[e][0]) <= 600:
continue
t = P.tree(e, calls, max_tree if calls else 600)
if t is None or (calls and len(t) < 2):
continue # (--calls: only roots that do call something)
roots.append((mod, e, len(t)))
funcs |= t
got += 1
return roots, funcs
HARNESS = r"""
#include <stdio.h>
#include <stdlib.h>
#include <windows.h>
static const A64Fn T[] = {%(table)s};
void a64_trap(Ctx *c, uint64_t pc, uint32_t insn) { (void)pc; (void)insn; c->halted = 9; }
void a64_ret_mismatch(Ctx *c, uint64_t pc) { (void)pc; c->halted = 10; }
void recomp_unhandled(GuestContext *c, uint32_t i, uint64_t pc) { (void)i; (void)pc; c->halted = 7; }
void recomp_barrier(void) {}
void recomp_ic_ivau(GuestContext *c, uint64_t a, int v) { (void)c; (void)a; (void)v; }
uint64_t recomp_cntpct(GuestContext *c) { (void)c; return 0; }
uint64_t recomp_ldxr(GuestContext *c, uint64_t a, uint32_t s) { uint64_t v = 0; memcpy(&v, A64_PTR(a), s); c->excl_addr = a; c->excl_valid = 1; return v; }
uint32_t recomp_stxr(GuestContext *c, uint64_t a, uint32_t s, uint64_t v) { if (!c->excl_valid || c->excl_addr != a) return 1; c->excl_valid = 0; memcpy(A64_PTR(a), &v, s); return 0; }
void recomp_ldxp(GuestContext *c, uint64_t a, uint32_t s, uint64_t *lo, uint64_t *hi) { *lo = *hi = 0; memcpy(lo, A64_PTR(a), s); memcpy(hi, A64_PTR(a + s), s); c->excl_addr = a; c->excl_valid = 1; }
uint32_t recomp_stxp(GuestContext *c, uint64_t a, uint32_t s, uint64_t lo, uint64_t hi) { if (!c->excl_valid || c->excl_addr != a) return 1; c->excl_valid = 0; memcpy(A64_PTR(a), &lo, s); memcpy(A64_PTR(a + s), &hi, s); return 0; }
void recomp_clrex(GuestContext *c) { c->excl_valid = 0; }
/* loop-header polls (nx2/poll.py) fire on two of every three loop iterations: they must change nothing. The poll is
called with no argument (rtgen.py): the context under test is the one run() was given */
static uint64_t polls;
static Ctx *poll_ctx;
void a64_poll_cur(void) { polls++; poll_ctx->poll = (polls %% 3) != 0; }
void a64_poll(Ctx *c) { c->poll = 0; a64_poll_cur(); }
#define IMG_BASE %(img_base)dULL
#define IMG_SIZE %(img_size)dULL
#define SCR_BASE %(scratch_base)dULL
#define SCR_SIZE %(scratch)dULL
static const uint64_t regions[][2] = {%(regions)s};
#define NREG (sizeof regions / sizeof regions[0])
static uint8_t *p_img, *p_scr;
static uint8_t *pristine(uint64_t a) { return a >= SCR_BASE ? p_scr + (a - SCR_BASE) : p_img + (a - IMG_BASE); }
static uint64_t page_hash(const uint8_t *p) {
const uint64_t *w = (const uint64_t *)p; uint64_t a = 0, b = 0;
for (uint64_t i = 0; i < 512; i++) { a += w[i] * (2 * i + 1); b += w[i] ^ (i * 0x9E3779B97F4A7C15ULL); }
return a ^ b;
}
static void make_scratch(uint64_t seed, uint64_t *out) {
for (uint64_t i = 0; i < SCR_SIZE / 8; i++) {
uint64_t z = seed + (i + 1) * 0x9E3779B97F4A7C15ULL;
z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ULL; z = (z ^ (z >> 27)) * 0x94D049BB133111EBULL; z ^= z >> 31;
out[i] = (z & 3) == 0 ? ((SCR_BASE + (z >> 12) %% (SCR_SIZE - 256)) & ~7ULL) : z;
}
}
static int run(int idx, Ctx *c) {
poll_ctx = c;
__try { T[idx](c); } __except (EXCEPTION_EXECUTE_HANDLER) { return (int)GetExceptionCode(); }
return 0;
}
int main(int argc, char **argv) {
FILE *img = fopen(argv[1], "rb"), *in = fopen(argv[2], "rb"), *out = fopen(argv[3], "wb");
if (VirtualAlloc(A64_PTR(IMG_BASE), IMG_SIZE, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE) != A64_PTR(IMG_BASE)) return 3;
if (VirtualAlloc(A64_PTR(SCR_BASE), SCR_SIZE, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE) != A64_PTR(SCR_BASE)) return 4;
if (fread(A64_PTR(IMG_BASE), 1, IMG_SIZE, img) != IMG_SIZE) return 5;
p_img = malloc(IMG_SIZE); p_scr = malloc(SCR_SIZE);
memcpy(p_img, A64_PTR(IMG_BASE), IMG_SIZE);
static uint8_t state[%(state)d];
unsigned idx; uint64_t seed;
while (fread(&idx, 4, 1, in) == 1 && fread(&seed, 8, 1, in) == 1 && fread(state, sizeof state, 1, in) == 1) {
make_scratch(seed, (uint64_t *)p_scr);
memcpy(A64_PTR(SCR_BASE), p_scr, SCR_SIZE);
Ctx c; memset(&c, 0, sizeof c);
memcpy(c.x, state, 256); memcpy(&c.n, state + 256, 4);
memcpy(&c.fpcr, state + 264, 8); memcpy(&c.fpsr, state + 272, 8); memcpy(c.vreg, state + 280, 512);
memcpy(&c.tpidr_el0, state + 792, 8); memcpy(&c.tpidrro_el0, state + 800, 8);
c.poll = 1;
int fault = run(idx, &c);
memcpy(state, c.x, 256); memcpy(state + 256, &c.n, 4);
memcpy(state + 264, &c.fpcr, 8); memcpy(state + 272, &c.fpsr, 8); memcpy(state + 280, c.vreg, 512);
fwrite(&c.halted, 4, 1, out); fwrite(&fault, 4, 1, out); fwrite(state, sizeof state, 1, out);
uint32_t ndiff = 0; static uint64_t diff[2 * 8192];
for (size_t r = 0; r < NREG; r++)
for (uint64_t a = regions[r][0]; a < regions[r][1]; a += 4096)
if (memcmp(A64_PTR(a), pristine(a), 4096)) {
if (ndiff < 8192) { diff[2 * ndiff] = a; diff[2 * ndiff + 1] = page_hash(A64_PTR(a)); }
ndiff++;
memcpy(A64_PTR(a), pristine(a), 4096);
}
fwrite(&ndiff, 4, 1, out); fwrite(diff, 16, ndiff < 8192 ? ndiff : 8192, out);
}
fclose(out);
fprintf(stderr, "loop-header polls taken: %%llu\n", (unsigned long long)polls);
return 0;
}
"""
def writable_regions(meta):
regs = []
for m in meta["modules"]:
data = next(s for s in m["segments"] if s["name"] == ".data")
lo = data["addr"] & ~(PAGE - 1)
hi = (max(m["bss"][1], data["addr"] + data["size"]) + PAGE - 1) & ~(PAGE - 1)
regs.append((lo, hi))
regs.append((SCRATCH, SCRATCH + SCRATCH_SIZE))
return regs
def build(tmp, P, roots, funcs, meta, img_size, addr64):
a64_rtgen.write(tmp)
with open(os.path.join(tmp, "a64_fpx.h"), "w", newline="\n") as f:
f.write(a64_lift.fpx_header(P.fpx))
rt = os.path.join(kit.KIT, "rt")
if addr64:
rt = rtgen.write(os.path.join(tmp, "rt64")) # (its a64.h shadows the vendored one)
src = ['#include "a64.h"'] + ["void f_%08x(Ctx *A64_NOESC);" % e for e in sorted(funcs)]
body = "\n".join(P.emitted[e][0] for e in sorted(funcs))
src.append(body if os.environ.get("A64_MARK_EACH") else a64_lift.mark_groups(body)) # (nx2/lift.py's last step)
src.append(HARNESS % {
"table": ",".join("f_%08x" % e for _, e, _ in roots),
"regions": ",".join("{%#xULL,%#xULL}" % r for r in writable_regions(meta)),
"scratch": SCRATCH_SIZE, "scratch_base": SCRATCH, "img_base": meta["base"],
"img_size": img_size, "state": STATE.size})
path = os.path.join(tmp, "harness.c")
open(path, "w", newline="\n").write("\n".join(src))
exe = os.path.join(tmp, "harness.exe")
cmd = [toolchain.need(toolchain.find("clang"))] + toolchain.native_target() + toolchain.native_link() + \
["-O1", "-g0", "-mcx16", "-Wno-everything"] + (["-DA64_POLLING"] if addr64 else []) + ["-I" + rt, "-I" + tmp, path,
os.path.join(tmp, "a64_suyu.c"), "-o", exe]
subprocess.run(cmd, check=True)
return exe
def rand_state(rng):
x = []
for i in range(31):
r = rng.random()
if i < 8 and r < 0.45:
x.append(SCRATCH + 0x1000 + 8 * rng.randrange((SCRATCH_SIZE - 0x22000) // 8))
elif r < 0.7:
x.append(rng.choice([0, 1, 2, 3, 4, 7, 8, 16, 31, 32, 64, 100, 255, 0xFFFFFFFF, M64, rng.randrange(1024)]))
else:
x.append(rng.getrandbits(64))
x[30] = SENTINEL
x.append(SCRATCH + SCRATCH_SIZE - 0x10000 - 16 * rng.randrange(256)) # sp
v = []
for _ in range(32):
if rng.random() < 0.5:
v += [rng.getrandbits(64), rng.getrandbits(64)]
else:
lanes = [struct.unpack("<I", struct.pack("<f", rng.uniform(-100, 100)))[0] for _ in range(2)]
v += [lanes[0] | lanes[1] << 32, struct.unpack("<Q", struct.pack("<d", rng.uniform(-1e3, 1e3)))[0]]
fpcr = 0 if rng.random() < 0.75 else 1 << 24
return x, [0, 0, 0, 0], fpcr, 0, v, SCRATCH + 0x800, SCRATCH + 0x400
class Emu:
def __init__(self, image, meta, regions):
uc = unicorn.Uc(unicorn.UC_ARCH_ARM64, unicorn.UC_MODE_ARM)
uc.ctl_set_cpu_model(A.UC_CPU_ARM64_MAX)
base, size = meta["base"], (len(image) + PAGE - 1) & ~(PAGE - 1)
uc.mem_map(base, size)
uc.mem_write(base, bytes(image))
uc.mem_map(SCRATCH, SCRATCH_SIZE)
uc.mem_map(SENTINEL, PAGE)
self.uc, self.dirty, self.regions = uc, set(), regions
self.pristine = {}
for lo, hi in regions[:-1]:
for pg in range(lo, hi, PAGE):
self.pristine[pg] = bytes(uc.mem_read(pg, PAGE))
uc.hook_add(unicorn.UC_HOOK_MEM_WRITE, self._write)
def _write(self, uc, access, addr, size, value, data):
self.dirty.add(addr & ~(PAGE - 1))
self.dirty.add((addr + size - 1) & ~(PAGE - 1))
def run(self, entry, st, scratch):
uc = self.uc
x, nzcv, fpcr, fpsr, v, tpidr, tpidrro = st
uc.mem_write(SCRATCH, scratch)
self.dirty = set()
for i in range(31):
uc.reg_write(XREG[i], x[i])
uc.reg_write(A.UC_ARM64_REG_SP, x[31])
uc.reg_write(A.UC_ARM64_REG_NZCV, 0)
uc.reg_write(A.UC_ARM64_REG_FPCR, fpcr)
uc.reg_write(A.UC_ARM64_REG_FPSR, fpsr)
uc.reg_write(A.UC_ARM64_REG_TPIDR_EL0, tpidr)
uc.reg_write(A.UC_ARM64_REG_TPIDRRO_EL0, tpidrro)
for i in range(32):
uc.reg_write(QREG[i], v[2 * i] | v[2 * i + 1] << 64)
try:
uc.emu_start(entry, SENTINEL, count=400000)
ok = uc.reg_read(A.UC_ARM64_REG_PC) == SENTINEL
except unicorn.UcError:
ok = False
res = None
if ok:
ox = [uc.reg_read(XREG[i]) for i in range(31)] + [uc.reg_read(A.UC_ARM64_REG_SP)]
ov = []
for i in range(32):
q = uc.reg_read(QREG[i])
ov += [q & M64, q >> 64]
pages = {}
for a in self.dirty:
before = scratch[a - SCRATCH:a - SCRATCH + PAGE] if SCRATCH <= a < SCRATCH + SCRATCH_SIZE \
else self.pristine.get(a)
now = bytes(uc.mem_read(a, PAGE))
if before is not None and now != before:
pages[a] = page_hash(now)
res = (ox, ov, uc.reg_read(A.UC_ARM64_REG_FPSR), pages)
for a in self.dirty: # the image pages put back for the next case
if a in self.pristine:
uc.mem_write(a, self.pristine[a])
return res
def main():
kit.wait_for_quiet('the function test')
global SCRATCH
p = argparse.ArgumentParser()
p.add_argument("--count", type=int, default=3000)
p.add_argument("--states", type=int, default=4)
p.add_argument("--seed", type=int, default=1)
p.add_argument("--calls", action="store_true")
p.add_argument("--max-tree", type=int, default=4000)
p.add_argument("--addr32", action="store_true")
p.add_argument("--list", action="store_true")
a = p.parse_args()
if a.addr32:
SCRATCH = 0x0E000000
rng = random.Random(a.seed)
t0 = time.time()
img = eden_image.EdenImage(kit.need_exefs())
meta = img.write(os.path.join(kit.WORK, "image_test"))
P = Picker(img)
share = module_shares(img)
roots, funcs = select(P, {m: max(1, int(a.count * s)) for m, s in share.items()}, rng, a.calls, a.max_tree)
print("%d %s, %d functions lifted in all, %d instructions (%.0fs)" % (
len(roots), "call trees" if a.calls else "self-contained functions", len(funcs),
sum(len(P.L.funcs[e][0]) for e in funcs), time.time() - t0), flush=True)
tmp = tempfile.mkdtemp(prefix="oot_leaf_")
exe = build(tmp, P, roots, funcs, meta, len(img.mem), not a.addr32)
print("harness built, %s guest window, scratch at %#x (%.0fs)" % ("32-bit" if a.addr32 else "64-bit", SCRATCH,
time.time() - t0), flush=True)
emu = Emu(img.mem, meta, writable_regions(meta))
cases, expect, skipped = [], [], 0
for i, (mod, e, _) in enumerate(roots):
for _ in range(a.states):
st = rand_state(rng)
seed = rng.getrandbits(64)
res = emu.run(e, st, scratch_words(seed).tobytes())
if res is None:
skipped += 1
continue
cases.append((i, seed, st))
expect.append(res)
print("%d comparable cases, %d skipped (Unicorn fault or no return) (%.0fs)" % (
len(cases), skipped, time.time() - t0), flush=True)
img_path = os.path.join(kit.WORK, "image_test", "image.bin")
inp, outp = os.path.join(tmp, "in.bin"), os.path.join(tmp, "out.bin")
with open(inp, "wb") as f:
for i, seed, st in cases:
x, nzcv, fpcr, fpsr, v, tpidr, tpidrro = st
f.write(struct.pack("<IQ", i, seed) + STATE.pack(*(x + nzcv + [fpcr, fpsr] + v + [tpidr, tpidrro])))
r = subprocess.run([exe, img_path, inp, outp], timeout=900)
if r.returncode:
raise SystemExit("harness exited with %d" % r.returncode)
fn_fail, fn_pass, fpsr_only = collections.defaultdict(list), collections.Counter(), collections.Counter()
ip_only = collections.Counter()
tested = set()
with open(outp, "rb") as f:
for (i, seed, st), (ex_x, ex_v, ex_fpsr, ex_pages) in zip(cases, expect):
halted, fault = struct.unpack("<ii", f.read(8))
vals = STATE.unpack(f.read(STATE.size))
ndiff = struct.unpack("<I", f.read(4))[0]
raw = f.read(16 * min(ndiff, 8192))
pages = {struct.unpack_from("<Q", raw, 16 * k)[0]: struct.unpack_from("<Q", raw, 16 * k + 8)[0]
for k in range(min(ndiff, 8192))}
mod, e, _ = roots[i]
tested.add(i)
d = []
if halted or fault:
d.append("halted %d fault %#x" % (halted, fault & 0xFFFFFFFF))
else:
ip = a.calls and (vals[16], vals[17]) != (ex_x[16], ex_x[17])
ip_only[i] += ip
d += ["%s %#x != %#x" % ("sp" if r == 31 else "x%d" % r, vals[r], ex_x[r]) for r in range(32)
if vals[r] != ex_x[r] and not (a.calls and r in (16, 17))]
d += ["v%d.%s %#x != %#x" % (r // 2, "hi" if r & 1 else "lo", vals[38 + r], ex_v[r]) for r in range(64)
if vals[38 + r] != ex_v[r]]
if pages != ex_pages:
d.append("memory: pages %s" % sorted("%x" % p for p in set(pages) ^ set(ex_pages)
or [p for p in pages if pages[p] != ex_pages.get(p)])[:4])
if d:
fn_fail[i].append(d[:4])
else:
fn_pass[i] += 1
if (vals[37] & 0x9F) != (ex_fpsr & 0x9F):
fpsr_only[i] += 1
bad = sorted(fn_fail)
per = collections.Counter(roots[i][0] for i in tested)
sizes = [roots[i][2] for i in tested]
print("%d %s run (%s)%s, %d cases: %d pass every case, %d with differences; FPSR cumulative bits differ in %d" % (
len(tested), "call trees" if a.calls else "functions", ", ".join("%s %d" % kv for kv in per.most_common()),
" (%d functions per tree on average, up to %d)" % (sum(sizes) // max(1, len(sizes)), max(sizes or [0]))
if a.calls else "", len(cases), len(tested) - len(bad), len(bad), len(fpsr_only)))
if a.calls:
print("X16/X17 (PLT scratch, not compared) differed in %d cases of %d trees" % (
sum(ip_only.values()), sum(1 for v in ip_only.values() if v)))
for i in bad if a.list else bad[:30]:
mod, e, _ = roots[i]
print(" FAIL f_%08x %-40s %d/%d: %s" % (e, (img.names.get(e) or "")[:40], len(fn_fail[i]),
len(fn_fail[i]) + fn_pass[i], "; ".join(fn_fail[i][0])))
print("PASS" if not bad else "FAIL", "(%.0fs)" % (time.time() - t0))
return 0 if not bad else 1
if __name__ == "__main__":
sys.exit(main())

View File

@@ -0,0 +1,53 @@
"""Run the actual topology/restart cache fragment with command spies; no game, Vulkan or SDK required."""
import argparse
from pathlib import Path
import subprocess,tempfile
RUN_ALL_ARGS = ["--cxx", "g++"]
ROOT=Path(__file__).resolve().parents[1]
def main():
p=argparse.ArgumentParser(description=__doc__);p.add_argument('--cxx',default='clang++');a=p.parse_args()
text=(ROOT/'gpu/vk/vk_draw.cpp').read_text(encoding='utf-8')
macro=next(l for l in text.splitlines() if l.startswith('#define DYN_CHANGED'))
start=text.index(' if (!mesh) {',text.index('void dynamic_state('));end=text.index('\n }',start)+6
commands=text[start:end]
start=text.index(' d.topo = ',text.index('void dynamic_state('))
end=text.index(';',text.index(' d.restart = ',start))+1
state=text[start:end]
start=text.index(' last_dyn = d;',text.index('void dynamic_state('));end=text.index('\n',text.index('last_dyn_valid = true;',start))
snapshot=text[start:end]
fixture='''#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#define CHECK(x) do{if(!(x)){std::fprintf(stderr,"FAIL:%d %s\\n",__LINE__,#x);std::exit(1);}}while(0)
struct DynState { uint32_t topo,restart; } last_dyn{};
enum {VK_PRIMITIVE_TOPOLOGY_MAX_ENUM=0x7fffffff, VK_PRIMITIVE_TOPOLOGY_POINT_LIST=0, VK_PRIMITIVE_TOPOLOGY_LINE_LIST=1, VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST=3, VK_PRIMITIVE_TOPOLOGY_PATCH_LIST=10};
bool last_dyn_valid=false;int topology_calls=0,restart_calls=0;
uint32_t actual_topology=999,actual_restart=999;
void vkCmdSetPrimitiveTopology(int,uint32_t v){topology_calls++;actual_topology=v;}
void vkCmdSetPrimitiveRestartEnable(int,uint32_t v){restart_calls++;actual_restart=v;}
enum {FC_DYN_CALLS};
#define FINE_X(c, n) ((void)0)
'''+macro+'''
void run(bool mesh,uint32_t topo,uint32_t restart) {
int cmd=1;DynState d{};struct {uint32_t restart;}s{restart};
'''+state+'\n'+commands+'\n'+snapshot+'''
}
int main() {
run(true,3,0);CHECK(topology_calls==0 && restart_calls==0);
run(false,3,0);CHECK(topology_calls==1 && restart_calls==1 && actual_topology==3 && actual_restart==0);
run(false,3,0);CHECK(topology_calls==1 && restart_calls==1);
run(true,0,0);run(true,0,0);run(false,0,0);
CHECK(topology_calls==2 && restart_calls==2 && actual_topology==0);
run(false,4,1);CHECK(topology_calls==3 && restart_calls==3 && actual_restart==1);
last_dyn_valid=false;run(false,4,1);CHECK(topology_calls==4 && restart_calls==4);
puts("PASS: mesh/vertex transitions, repeated states, restart changes and command-buffer invalidation");
}
'''
work=ROOT/'work';work.mkdir(exist_ok=True)
with tempfile.TemporaryDirectory(prefix='mesh-dynamic-',dir=work) as temp:
directory=Path(temp).resolve();directory.relative_to(work.resolve());cpp=directory/'fixture.cpp';exe=directory/'fixture.exe';cpp.write_text(fixture,encoding='utf-8')
subprocess.run([a.cxx,'-std=c++20','-O2',str(cpp),'-o',str(exe)],check=True);subprocess.run([str(exe)],check=True)
if __name__=='__main__':main()

View File

@@ -0,0 +1,89 @@
"""Check the actual first-task setup creates its required 64-bit address type.
python nx2/test_mesh_first_types.py --headers <SPIRV-Headers/include> [--cxx clang++]
No GPU or game files needed; tests setup/type prerequisites, not full task execution.
"""
import argparse
import os
from pathlib import Path
import subprocess
import tempfile
RUN_ALL_ARGS = ["--headers", "{spirv_headers}"]
ROOT=Path(__file__).resolve().parents[1]
def block(text,marker):
start=text.index(marker);end=text.index('{',start)+1;depth=1
while depth:
depth+=(text[end]=='{')-(text[end]=='}');end+=1
return text[start:end]
def main():
p=argparse.ArgumentParser(description=__doc__);p.add_argument('--headers',type=Path,required=True);p.add_argument('--cxx',default='clang++');a=p.parse_args()
vk=ROOT/'gpu/vk';source=(vk/'eden/shader_recompiler/backend/spirv/spirv_emit_context.cpp').read_text(encoding='utf-8')
setup=block(source,'if (stage == Stage::Task && !runtime_info.mesh_fallback && getenv("RX_VK_MESH_FIRST")')
fixture=r'''
#include <sirit/sirit.h>
#include <array>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <stdexcept>
using Sirit::Id;enum class Stage { Task,Compute };
struct Program { struct { bool uses_global_memory=false; } info; };
struct EmitContext;
Id DefineInput(EmitContext&,Id,bool,spv::BuiltIn);
struct EmitContext:Sirit::Module {
struct { bool unified_descriptor_binding=true,support_int64=true; } profile;
struct { bool mesh_fallback=false; } runtime_info;
Stage stage=Stage::Task;std::array<Id,2> U32{};Id U64{},draw_index{},mesh_first_block_pointer{},mesh_first_member_pointer{};
bool wants_mesh_first=false,wants_guest_pages=false;unsigned pushes=0;
EmitContext(bool had64):Module(0x00010600) {
AddCapability(spv::Capability::Shader);U32[1]=TypeUInt(32);
if(had64){AddCapability(spv::Capability::Int64);U64=TypeUInt(64);}
}
void DefineRescalingInputPushConstant() {
if(!Sirit::ValidId(U64))throw std::runtime_error("first-task push block requires an initialized Int64 type");
TypeStruct(U64);++pushes;
}
void Enable(const Program& program) {
''' + setup + r'''
}
};
// Input declaration is irrelevant to type initialization; use a valid ID without introducing a shader interface.
Id DefineInput(EmitContext& ctx,Id type,bool,spv::BuiltIn){return ctx.Constant(type,0U);}
int main() {
unsigned cases=0;const bool requested=std::getenv("RX_VK_MESH_FIRST")!=nullptr;
for(unsigned i=0;i<6;i++) {
EmitContext ctx{i==1};const Id before=ctx.U64;
if(i==2)ctx.stage=Stage::Compute;
if(i==3)ctx.profile.unified_descriptor_binding=false;
if(i==4)ctx.profile.support_int64=false;
if(i==5)ctx.runtime_info.mesh_fallback=true;
try{ctx.Enable(Program{});}catch(const std::exception& e){std::fprintf(stderr,"%s\n",e.what());return 1;}
const bool enabled=requested && i<2;
if(ctx.wants_mesh_first!=enabled || ctx.pushes!=(enabled?1:0))return 2;
if(enabled && !Sirit::ValidId(ctx.U64))return 3;
if(i==1 && ctx.U64.value!=before.value)return 4;
if(!enabled && i!=1 && Sirit::ValidId(ctx.U64))return 5;
auto words=ctx.Assemble();bool capability=false;
for(size_t at=5;at<words.size();at+=words[at]>>16)
if((words[at]&65535)==unsigned(spv::Op::OpCapability) && words[at+1]==unsigned(spv::Capability::Int64))capability=true;
if(capability!=(enabled || i==1))return 6;
++cases;
}
std::printf("PASS: %u setup/type/capability cases, first-task option %s\n",cases,requested?"on":"off");
}
'''
work=ROOT/'work';work.mkdir(exist_ok=True)
with tempfile.TemporaryDirectory(prefix='mesh-first-types-',dir=work) as t:
folder=Path(t).resolve();folder.relative_to(work.resolve());cpp=folder/'fixture.cpp';exe=folder/'fixture.exe';cpp.write_text(fixture,encoding='utf-8');sirit=vk/'sirit'
subprocess.run([a.cxx,'-std=c++20','-O1','-I'+str(sirit/'include'),'-I'+str(sirit/'src'),'-I'+str(a.headers),str(cpp),*map(str,sorted((sirit/'src').rglob('*.cpp'))),'-o',str(exe)],check=True)
env=dict(os.environ);env.pop('RX_VK_MESH_FIRST',None)
subprocess.run([str(exe)],env=env,check=True)
env['RX_VK_MESH_FIRST']='1';subprocess.run([str(exe)],env=env,check=True)
if __name__=='__main__':main()

108
nx2/test_nup_codec.py Normal file
View File

@@ -0,0 +1,108 @@
"""Game-free actual Nup constructor/copy_to regression; requires zstandard.
python -I -B nx2/test_nup_codec.py
"""
import argparse
from contextlib import redirect_stdout
import importlib.util
import io
from pathlib import Path
import struct
import tempfile
from unittest.mock import patch
import zstandard
RUN_ALL_MODULES = ["zstandard"]
p = argparse.ArgumentParser(description=__doc__)
p.add_argument('--source', type=Path, default=Path(__file__).resolve().parents[1])
a = p.parse_args()
spec = importlib.util.spec_from_file_location('tested_nup', a.source/'nx2/nup.py')
nup = importlib.util.module_from_spec(spec)
spec.loader.exec_module(nup)
def package(name, logical, stored, codec, extent=None):
"""One title, one root directory, one file; no private content or other modules."""
name = name.encode()
file_at = 0x80 + 0x30 + 0x40 + 24
strings_at = file_at + 80 + 4
data_at = (strings_at + len(name) + 15) & ~15
# The tail deliberately contains the rest of a truncated declared extent.
blob = bytearray(data_at + len(stored) + len(logical) + 64)
blob[:4] = b'NUP\x01'
struct.pack_into('<Q', blob, 0x40, len(blob))
blob[0x80:0x84] = b'TOC\x01'
struct.pack_into('<IQQQQ', blob, 0x84, 1, 1, 1, strings_at-0x80, len(name))
struct.pack_into('<QIIII', blob, 0xb0, 1, 0, 0, 0, 1)
struct.pack_into('<4I', blob, file_at-24, 0, 0, 0, 1)
struct.pack_into('<IIQQQ', blob, file_at, 0, len(name), len(logical), data_at,
len(stored) if extent is None else extent)
blob[file_at+64] = codec
struct.pack_into('<I', blob, file_at+80, 1)
blob[strings_at:strings_at+len(name)] = name
blob[data_at:data_at+len(stored)] = stored
return blob
with tempfile.TemporaryDirectory(prefix='nup-codec-') as temporary:
path = Path(temporary)/'fixture.nup'
def copy(name, logical, stored, codec, extent=None):
path.write_bytes(package(name, logical, stored, codec, extent))
reader = nup.Nup(str(path))
try:
relative, record = next(reader.walk(0))
assert relative == name
output = io.BytesIO()
reader.copy_to(record, output, relative)
return output.getvalue()
finally:
reader.f.close()
compressor = zstandard.ZstdCompressor(write_checksum=True)
plain = bytes(32) + b'ordinary payload, not another Zstd frame'*100
packed = compressor.compress(plain)
assert copy('member.zs', plain, packed, 1) == plain, 'explicit outer codec ignored for .zs plaintext'
game_zstd = compressor.compress(b'game asset content'*200)
assert copy('asset.zs', game_zstd, game_zstd, 0) == game_zstd, 'raw game Zstd was decompressed'
assert copy('legacy.bin', plain, packed, 0) == plain, 'legacy outer-frame detection changed'
try:
copy('truncated.zs', plain, packed, 1, len(packed)-1)
except SystemExit:
pass
else:
raise AssertionError('explicit extent was overread or incomplete frame accepted')
oversized = compressor.compress(bytes(65536))
path.write_bytes(package('oversized.bin', bytes(8), oversized, 1))
reader = nup.Nup(str(path))
with reader.f:
relative, record = next(reader.walk(0))
output = io.BytesIO()
try:
reader.copy_to(record, output, relative)
except SystemExit:
assert output.tell() == 0, f'wrote {output.tell()} bytes past the declared size'
else:
raise AssertionError('oversized compressed member accepted')
def extract_member(folder, logical, stored, codec, ticks, quiet=False):
path.write_bytes(package('member.bin', logical, stored, codec))
reader = nup.Nup(str(path))
output = io.StringIO()
with reader.f, redirect_stdout(output), patch.object(nup, 'monotonic', side_effect=ticks, create=True):
assert nup.extract(reader, reader.titles[0], str(folder), only='exefs', quiet=quiet) == (1, len(logical))
assert (folder / 'exefs' / 'member.bin').read_bytes() == logical
return output.getvalue().splitlines()
root = Path(temporary)
raw = b'x' * (24 << 20)
assert extract_member(root / 'raw', raw, raw, 0, [0, 2.25, 2.50, 4.50]) == [
' 8.0 MiB written (0 files completed)',
' 24.0 MiB written (0 files completed)',
], 'one large member must report bytes before it finishes, without printing for every chunk'
logical = b'z' * (1 << 20)
stored = compressor.compress(logical)
assert extract_member(root / 'packed', logical, stored, 1, [0, 3]) == [' 1.0 MiB written (0 files completed)']
assert extract_member(root / 'quiet', logical, stored, 1, AssertionError('quiet mode read the clock'), quiet=True) == []
print('PASS: NUP codecs, extent and output limits, and streamed extraction progress')

View File

@@ -0,0 +1,73 @@
"""Game-free: a NUP extraction that runs out of space names the member and its drive (issue 1388):
python -I -B nx2/test_nup_out_of_space.py"""
import errno
import os
from pathlib import Path
import sys
import tempfile
from types import SimpleNamespace
import unittest
from unittest.mock import patch
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import nup
class Package:
"""One authored member whose write fails."""
def __init__(self, failure):
self.failure = failure
def walk(self, directory):
yield "Model/authored.zs", (0, 0, 12345, 0, 12345, 0)
def copy_to(self, record, output, name):
output.write(b"some bytes")
raise self.failure
class OutOfSpace(unittest.TestCase):
def extract(self, failure, out):
return nup.extract(Package(failure), {"romfs": 0}, out, only="romfs", quiet=True)
def test_full_drive_names_the_member_and_the_drive(self):
with tempfile.TemporaryDirectory() as out:
member = Path(os.path.realpath(out)) / "romfs" / "Model" / "authored.zs"
partial = member.with_name("authored.zs.part")
seen = []
def usage(path):
seen.append((path, partial.exists()))
return SimpleNamespace(free=196 * 10 ** 9)
original = OSError(errno.ENOSPC, "authored disk-full failure")
with patch.object(nup.shutil, "disk_usage", side_effect=usage), \
patch.object(nup, "file_system", return_value="NTFS"), \
self.assertRaises(OSError) as error:
self.extract(original, out)
self.assertEqual(error.exception.errno, errno.ENOSPC)
self.assertEqual(error.exception.strerror,
"No space left on device writing romfs/Model/authored.zs (12345 bytes): 196.0 GB free there, NTFS")
self.assertIs(error.exception.__cause__, original)
# (measured while the failed member still holds its space)
self.assertEqual(seen, [(str(member.parent), True)])
self.assertFalse(partial.exists())
def test_other_errors_pass_through(self):
with tempfile.TemporaryDirectory() as out:
original = PermissionError(errno.EACCES, "authored access failure")
with self.assertRaises(PermissionError) as error:
self.extract(original, out)
self.assertIs(error.exception, original)
def test_file_system_of_a_real_folder(self):
with tempfile.TemporaryDirectory() as out:
if os.name == "nt":
self.assertTrue(nup.file_system(out))
else:
self.assertIsNone(nup.file_system(out))
if __name__ == "__main__":
unittest.main()

424
nx2/test_nup_paths.py Normal file
View File

@@ -0,0 +1,424 @@
"""Game-free extraction boundary regressions: python -I nx2/test_nup_paths.py."""
from contextlib import contextmanager, ExitStack, redirect_stdout
import errno
import importlib.util
import io
import os
from pathlib import Path
import re
import stat
import struct
import subprocess
import sys
import tempfile
import unittest
from unittest.mock import patch
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import nup
class WriteAttempt(AssertionError):
pass
@contextmanager
def guard_io(root):
"""Keep unsafe probes inside this fixture, without faking successful metadata reads."""
def normalized(path):
path = os.path.normcase(os.path.abspath(path))
# Compare equivalent filesystem paths; keep the original spelling for the real metadata call.
if os.name == "nt":
if path.startswith("\\\\?\\unc\\"):
return "\\\\" + path[8:]
if path.startswith("\\\\?\\") and path[5:7] == ":\\":
return path[4:]
return path
roots = {normalized(root), normalized(os.path.realpath(root))}
attempts = []
def metadata(function):
def checked(path, *args, **kwargs):
candidate = normalized(path)
for owned in roots:
try:
common = os.path.commonpath((candidate, owned))
except ValueError:
continue
if common in (candidate, owned): # realpath may inspect the fixture's ancestors
return function(path, *args, **kwargs)
raise AssertionError("metadata access outside fixture: %r" % path)
return checked
def writing(name):
def blocked(*args, **kwargs):
attempts.append(name)
raise WriteAttempt(name)
return blocked
with ExitStack() as stack:
for name in ("stat", "lstat"):
stack.enter_context(patch.object(os, name, side_effect=metadata(getattr(os, name))))
for name in ("makedirs", "mkdir", "open", "replace", "rename", "renames", "link", "symlink",
"unlink", "remove", "rmdir", "removedirs"):
stack.enter_context(patch.object(os, name, side_effect=writing("os." + name)))
stack.enter_context(patch("builtins.open", side_effect=writing("builtins.open")))
stack.enter_context(patch.object(io, "open", side_effect=writing("io.open")))
yield attempts
def snapshot(root):
"""Record directories and link targets too; never descend into a link or junction."""
result = {}
pending = [root]
while pending:
for path in pending.pop().iterdir():
info = path.lstat()
relative = path.relative_to(root)
if stat.S_ISLNK(info.st_mode) or getattr(info, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0):
result[relative] = ("link", os.readlink(path))
elif stat.S_ISDIR(info.st_mode):
result[relative] = ("directory",)
pending.append(path)
else:
result[relative] = ("file", path.read_bytes())
return result
class Package:
def __init__(self, names, data=b"authored fixture"):
self.names, self.data = names, data
def walk(self, directory):
for name in self.names:
yield name, (0, 0, len(self.data), 0, len(self.data))
def copy_to(self, record, output, name):
output.write(self.data)
class HealingPackage(Package):
"""The selected update is missing main; the base has an authored module."""
titles = [dict(index=0, id=0, version=0, exefs=0, romfs=1),
dict(index=1, id=0, version=1, exefs=2, romfs=3)]
file_size, nd, nf = 0, 4, 1
def __init__(self, data=b"NSO0authored module"):
super().__init__(["main"], data)
def walk(self, directory):
if directory == 0:
yield from super().walk(directory)
def read(self, record, n=None):
return self.data if n is None else self.data[:n]
class ExtractionPaths(unittest.TestCase):
def setUp(self):
self.temporary = tempfile.TemporaryDirectory()
self.addCleanup(self.temporary.cleanup)
self.root = Path(self.temporary.name)
self.out = self.root / "out"
self.out.mkdir()
self.sentinel = self.root / "outside.bin"
self.sentinel.write_bytes(b"keep outside data")
def extract(self, names):
return nup.extract(Package(names), {"romfs": 0}, str(self.out), only="romfs", quiet=True)
def test_cli_measures_space_at_the_extraction_destination(self):
package = Package(["authored.bin"])
package.titles = [dict(index=0, id=0, version=1, exefs=0, romfs=1)]
package.file_size, package.nd, package.nf = 0, 2, 1
for destination in (self.out, self.out / "new" / "nested"):
with self.subTest(destination=destination), \
patch.object(nup, "Nup", return_value=package), \
patch.object(nup.shutil, "disk_usage", wraps=nup.shutil.disk_usage) as usage, \
patch.object(sys, "argv", ["nup.py", "authored.nup", "--extract", str(destination), "--only", "romfs"]), \
redirect_stdout(io.StringIO()):
nup.main()
usage.assert_called_once_with(str(self.out.resolve()))
self.assertEqual((destination / "romfs" / "authored.bin").read_bytes(), package.data)
def heal(self):
package = HealingPackage()
with redirect_stdout(io.StringIO()):
nup.heal(package, package.titles[-1], str(self.out), ["main"])
def unpack(self):
# This authored empty-segment NSO exercises the real unpack writer, without compressed game data.
raw = bytearray(0x100)
raw[:4] = b"NSO0"
struct.pack_into("<I", raw, 0xC, 1 << 7)
for i in range(3):
struct.pack_into("<I", raw, 0x10 + 0x10 * i, 0x100)
(self.out / "exefs" / "main").write_bytes(raw)
spec = importlib.util.spec_from_file_location("unpack_fixture", Path(nup.__file__).with_name("unpack.py"))
unpacker = importlib.util.module_from_spec(spec)
# No compressed segment may call the project's native decompressor in this fixture.
with patch.dict(sys.modules, {"zbic": object()}):
spec.loader.exec_module(unpacker)
unpacker.NAMES = ["main"]
with patch.object(nup, "Nup", return_value=HealingPackage(bytes(raw))), \
patch.object(sys, "argv", ["nup.py", "authored.nup", "--extract", str(self.out)]), \
patch.dict(sys.modules, {"unpack": unpacker}), redirect_stdout(io.StringIO()):
return nup.main()
def reject(self, names, message):
before = snapshot(self.root)
with self.assertRaisesRegex(SystemExit, "^" + re.escape(message)):
self.extract(names)
self.assertEqual(before, snapshot(self.root))
def directory_link(self, link, target):
try:
os.symlink(target, link, target_is_directory=True)
except OSError as error:
if os.name != "nt":
raise
# Junctions need no Developer Mode or symbolic-link privilege on Windows.
result = subprocess.run(["cmd", "/c", "mklink", "/J", str(link), str(target)],
capture_output=True, text=True)
if result.returncode:
self.fail("Cannot create the link fixture: %s; %s" % (error, result.stderr))
def test_nested_names_and_resume(self):
names = ["Map/Forest/data.bin", "Text/Français.bin", "com10.txt", ".hidden", "conifer.txt"]
self.assertEqual(self.extract(names), (len(names), len(names) * len(b"authored fixture")))
for name in names:
self.assertEqual((self.out / "romfs" / name).read_bytes(), b"authored fixture")
self.assertEqual(self.extract(names), (0, 0))
def test_paths_cannot_leave_the_output(self):
names = ["", ".", "..", "../../outside.bin", "nested/../../outside.bin", "nested/../file.bin",
"/rooted.bin", "//server/share/file.bin", r"\rooted.bin", r"\\server\share\file.bin",
r"C:\outside.bin", "C:/outside.bin", "C:outside.bin", r"nested\..\outside.bin",
"nested//file.bin", "nested/.. /outside.bin", "file:stream", "trailing.", "trailing ",
"control\x00.bin", "CON", "aux.txt", "NUL.bin", "COM1", "LPT9.txt", "COM¹.txt",
"CONIN$", "conout$", "nested/CONIN$", "nested/ConOut$"]
for name in names:
with self.subTest(name=name):
# Even a broken validator must not let the fixture touch an absolute path or UNC share.
before = snapshot(self.root)
with guard_io(self.root) as attempts:
with self.assertRaisesRegex(SystemExit, "^" + re.escape("unsafe path in the package: %r" % name) + "$"):
self.extract([name])
self.assertEqual(attempts, [])
# In this exact context, a valid name must pass validation and reach a write.
with self.assertRaisesRegex(WriteAttempt, "^os\\.makedirs$"):
self.extract(["valid.bin"])
self.assertEqual(attempts, ["os.makedirs"])
self.assertEqual(before, snapshot(self.root))
def test_write_guards_block_alternative_io(self):
before = snapshot(self.root)
operations = (("builtins.open", lambda: open(self.sentinel, "wb")),
("io.open", lambda: io.open(self.sentinel, "wb")),
("os.open", lambda: os.open(self.sentinel, os.O_WRONLY | os.O_TRUNC)),
("os.mkdir", lambda: os.mkdir(self.root / "new")),
("os.rename", lambda: os.rename(self.sentinel, self.root / "renamed")),
("os.link", lambda: os.link(self.sentinel, self.root / "linked")))
with guard_io(self.root) as attempts:
for name, operation in operations:
with self.subTest(operation=name), self.assertRaisesRegex(WriteAttempt, "^" + re.escape(name) + "$"):
result = operation()
if name == "os.open":
os.close(result)
elif hasattr(result, "close"):
result.close()
self.assertEqual(attempts, [name for name, _ in operations])
self.assertEqual(before, snapshot(self.root))
def test_metadata_guards_accept_owned_paths_and_reject_siblings(self):
outside = self.root.with_name(self.root.name + "-sibling") / "outside.bin"
with guard_io(self.root) as attempts:
for path in dict.fromkeys((str(self.sentinel), nup.long_path(str(self.sentinel)))):
with self.subTest(owned=path):
self.assertEqual(os.lstat(path).st_size, len(b"keep outside data"))
for path in dict.fromkeys((str(outside), nup.long_path(str(outside)))):
with self.subTest(outside=path), self.assertRaisesRegex(AssertionError, "^metadata access outside fixture:"):
os.lstat(path)
self.assertEqual(attempts, [])
def test_parent_path_cannot_overwrite_a_sibling(self):
# This also safely demonstrates the original failure: both files belong to this fixture.
self.reject(["../../outside.bin"], "unsafe path in the package: '../../outside.bin'")
def test_ordinary_partial_file_is_replaced(self):
(self.out / "romfs").mkdir()
partial = self.out / "romfs" / "file.bin.part"
partial.write_bytes(b"an interrupted extraction")
self.extract(["file.bin"])
self.assertEqual((self.out / "romfs" / "file.bin").read_bytes(), b"authored fixture")
self.assertFalse(partial.exists())
def test_failed_member_keeps_previous_file_and_can_be_retried(self):
target = self.out / "romfs" / "file.bin"
target.parent.mkdir()
partial = target.with_name("file.bin.part")
package = Package(["file.bin"])
title = {"romfs": 0}
def out_of_space(record, output, name):
output.write(b"some bytes")
raise OSError(errno.ENOSPC, "authored disk-full failure")
failures = (
(patch.object(package, "copy_to", side_effect=out_of_space), errno.ENOSPC),
(patch.object(nup.os, "replace", side_effect=PermissionError(errno.EACCES, "authored replacement failure")),
errno.EACCES),
)
for failure, code in failures:
with self.subTest(error=code):
target.write_bytes(b"previous file")
with failure, self.assertRaises(OSError) as error:
nup.extract(package, title, str(self.out), only="romfs", quiet=True)
self.assertEqual(error.exception.errno, code)
self.assertEqual(target.read_bytes(), b"previous file")
self.assertFalse(partial.exists())
self.assertEqual(nup.extract(package, title, str(self.out), only="romfs", quiet=True),
(1, len(package.data)))
self.assertEqual(target.read_bytes(), package.data)
def test_part_directory_link_cannot_redirect_extraction(self):
outside = self.root / "outside"
outside.mkdir()
self.directory_link(self.out / "romfs", outside)
self.reject(["new.bin"], "extraction path is a link or reparse point: ")
def test_nested_directory_link_cannot_redirect_extraction(self):
(self.out / "romfs").mkdir()
outside = self.root / "outside"
outside.mkdir()
self.directory_link(self.out / "romfs" / "nested", outside)
self.reject(["nested/new.bin"], "extraction path is a link or reparse point: ")
def test_directory_reparse_point_cannot_redirect_extraction(self):
outside = self.root / "outside"
outside.mkdir()
link = self.out / "romfs"
if os.name == "nt":
result = subprocess.run(["cmd", "/c", "mklink", "/J", str(link), str(outside)],
capture_output=True, text=True)
self.assertEqual(result.returncode, 0, result.stderr)
else:
os.symlink(outside, link, target_is_directory=True)
self.reject(["new.bin"], "extraction path is a link or reparse point: ")
def test_destination_link_is_rejected_before_resume(self):
(self.out / "romfs").mkdir()
self.sentinel.write_bytes(b"authored fixture")
try:
os.symlink(self.sentinel, self.out / "romfs" / "file.bin")
except OSError as error:
self.fail("Cannot create the file-link fixture: %s" % error)
self.reject(["file.bin"], "extraction path is a link or reparse point: ")
def test_temporary_link_cannot_truncate_an_outside_file(self):
(self.out / "romfs").mkdir()
try:
os.symlink(self.sentinel, self.out / "romfs" / "file.bin.part")
except OSError as error:
self.fail("Cannot create the file-link fixture: %s" % error)
self.reject(["file.bin"], "extraction path is a link or reparse point: ")
def test_temporary_hard_link_cannot_truncate_an_outside_file(self):
(self.out / "romfs").mkdir()
os.link(self.sentinel, self.out / "romfs" / "file.bin.part")
self.reject(["file.bin"], "extraction temporary file has other hard links: ")
def test_heal_missing_module_and_resume(self):
(self.out / "exefs").mkdir()
self.heal()
self.assertEqual((self.out / "exefs" / "main").read_bytes(), b"NSO0authored module")
before = snapshot(self.root)
self.heal()
self.assertEqual(before, snapshot(self.root))
def test_heal_directory_link_cannot_redirect_module(self):
outside = self.root / "outside"
outside.mkdir()
self.directory_link(self.out / "exefs", outside)
before = snapshot(self.root)
with self.assertRaisesRegex(SystemExit, "^extraction path is a link or reparse point: "):
self.heal()
self.assertEqual(before, snapshot(self.root))
def test_heal_directory_reparse_point_cannot_redirect_module(self):
outside = self.root / "outside"
outside.mkdir()
link = self.out / "exefs"
if os.name == "nt":
result = subprocess.run(["cmd", "/c", "mklink", "/J", str(link), str(outside)],
capture_output=True, text=True)
self.assertEqual(result.returncode, 0, result.stderr)
else:
os.symlink(outside, link, target_is_directory=True)
before = snapshot(self.root)
with self.assertRaisesRegex(SystemExit, "^extraction path is a link or reparse point: "):
self.heal()
self.assertEqual(before, snapshot(self.root))
def test_heal_destination_link_cannot_truncate_an_outside_file(self):
(self.out / "exefs").mkdir()
os.symlink(self.sentinel, self.out / "exefs" / "main")
before = snapshot(self.root)
with self.assertRaisesRegex(SystemExit, "^extraction path is a link or reparse point: "):
self.heal()
self.assertEqual(before, snapshot(self.root))
def test_heal_destination_hard_link_is_replaced_without_truncation(self):
(self.out / "exefs").mkdir()
module = self.out / "exefs" / "main"
os.link(self.sentinel, module)
self.heal()
self.assertEqual(self.sentinel.read_bytes(), b"keep outside data")
self.assertEqual(module.read_bytes(), b"NSO0authored module")
def test_heal_temporary_link_cannot_truncate_an_outside_file(self):
(self.out / "exefs").mkdir()
os.symlink(self.sentinel, self.out / "exefs" / "main.part")
before = snapshot(self.root)
with self.assertRaisesRegex(SystemExit, "^extraction path is a link or reparse point: "):
self.heal()
self.assertEqual(before, snapshot(self.root))
def test_heal_temporary_hard_link_cannot_truncate_an_outside_file(self):
(self.out / "exefs").mkdir()
os.link(self.sentinel, self.out / "exefs" / "main.part")
before = snapshot(self.root)
with self.assertRaisesRegex(SystemExit, "^extraction temporary file has other hard links: "):
self.heal()
self.assertEqual(before, snapshot(self.root))
def test_unpack_temporary_link_cannot_truncate_an_outside_file(self):
(self.out / "exefs").mkdir()
os.symlink(self.sentinel, self.out / "exefs" / "main.plain")
# unpack() creates only the authored module before entering nup.main's production path.
with self.assertRaisesRegex(SystemExit, r"^extraction path is a link or reparse point: .*main\.plain$"):
self.unpack()
self.assertEqual(self.sentinel.read_bytes(), b"keep outside data")
self.assertTrue((self.out / "exefs" / "main.plain").is_symlink())
def test_unpack_temporary_hard_link_cannot_truncate_an_outside_file(self):
(self.out / "exefs").mkdir()
os.link(self.sentinel, self.out / "exefs" / "main.plain")
with self.assertRaisesRegex(SystemExit, r"^extraction temporary file has other hard links: .*main\.plain$"):
self.unpack()
self.assertEqual(self.sentinel.read_bytes(), b"keep outside data")
def test_unpack_ordinary_temporary_file_is_replaced(self):
(self.out / "exefs").mkdir()
(self.out / "exefs" / "main.plain").write_bytes(b"interrupted unpack")
self.unpack()
module = (self.out / "exefs" / "main").read_bytes()
self.assertEqual(module[:4], b"NSO0")
self.assertEqual(struct.unpack_from("<I", module, 0xC)[0], 0)
self.assertFalse((self.out / "exefs" / "main.plain").exists())
if __name__ == "__main__":
unittest.main()

57
nx2/test_nup_read_name.py Normal file
View File

@@ -0,0 +1,57 @@
"""Actual Nup read/copy classification with synthetic bytes; requires existing zstandard.
python -I -B nx2/test_nup_read_name.py
"""
# SPDX-License-Identifier: GPL-3.0-or-later
import argparse
import importlib.util
import io
from pathlib import Path
import zstandard
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--source', type=Path, default=Path(__file__).resolve().parents[1])
args = parser.parse_args()
spec = importlib.util.spec_from_file_location('nup_read_name', args.source / 'nx2/nup.py')
nup = importlib.util.module_from_spec(spec)
spec.loader.exec_module(nup)
# Synthetic frame, not a game asset. Its compressed and decoded lengths are both 49.
FRAME = bytes.fromhex('28b52ffd20314501001402000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f00010041c013')
PAYLOAD = bytes(range(32)) + bytes(17)
assert len(FRAME) == len(PAYLOAD) == 49
assert zstandard.ZstdDecompressor().decompress(FRAME) == PAYLOAD
def member(data, name, size=None, codec=0):
reader = nup.Nup.__new__(nup.Nup)
reader.strings = name.encode('utf-8')
reader.f = io.BytesIO(bytes(32) + data)
record = (0, len(reader.strings), len(data) if size is None else size, 32, len(data), codec)
reader.files = [record]
return reader, record
def check(label, data, name, expected, size=None, codec=0):
reader, record = member(data, name, size, codec)
assert reader.read(record, 18) == expected[:18], label + ': read lost filename context'
assert reader.read(record) == expected, label + ': full read'
for supplied in (None, name):
output = io.BytesIO()
if supplied is None:
reader.copy_to(record, output)
else:
reader.copy_to(record, output, supplied)
assert output.getvalue() == expected, label + ': copy/name parity'
print('PASS:', label)
return reader, record
reader, record = check('raw equal-length .zs', FRAME, 'x.bcul.zs', FRAME)
assert not reader.is_packed(record)
assert reader.is_packed(record, 'main'), 'provided filename must remain authoritative'
plain = b'not another Zstd frame' * 3
compress = zstandard.ZstdCompressor().compress
check('explicit outer codec1', compress(plain), 'x.bcul.zs', plain, len(plain), 1)
check('legacy exefs outer frame', FRAME, 'main', PAYLOAD, len(PAYLOAD))
check('ordinary raw exefs', b'NSO0' + bytes(40), 'main', b'NSO0' + bytes(40))
check('legacy nested outer .zs', compress(FRAME), 'x.bcul.zs', FRAME, len(FRAME))

91
nx2/test_nup_stand_in.py Normal file
View File

@@ -0,0 +1,91 @@
"""Damaged .zs files in an unpacked romfs and their stand-ins: python -I -B nx2/test_nup_stand_in.py
No game bytes: small made-up files in a temporary folder. Standard library only.
"""
import importlib.util
import os
import sys
import tempfile
HERE = os.path.dirname(os.path.abspath(__file__))
spec = importlib.util.spec_from_file_location("tested_nup", os.path.join(HERE, "nup.py"))
nup = importlib.util.module_from_spec(spec)
spec.loader.exec_module(nup)
ZSTD = b"\x28\xb5\x2f\xfd"
failures = []
def check(what, got, want):
if got != want:
failures.append("%s: %.200r, %.200r expected" % (what, got, want))
def write(path, data):
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "wb") as f:
f.write(data)
with tempfile.TemporaryDirectory() as romfs:
tex = os.path.join(romfs, "Tex")
# (as in the 23.3 GB dump: the start of the file is zeros, the rest is there)
write(os.path.join(tex, "Rock_Temple_Gate_M_01_Nrm.bntx.zs"), bytes(400) + b"rest" * 30)
write(os.path.join(tex, "Rock_Temple_Gate_P_01_Nrm.bntx.zs"), ZSTD + b"P" * 500) # the nearest name: chosen
write(os.path.join(tex, "Rock_Temple_Gate_M_01_Alb.bntx.zs"), ZSTD + b"A" * 4000) # another kind of texture
write(os.path.join(tex, "Rock_Temple_Gate_A_Nrm.bntx.zs"), ZSTD + b"A" * 4000) # as close a name, a size further off
write(os.path.join(tex, "Rock_Temple_Gate_M_02_Nrm.bntx.zs"), bytes(64)) # damaged too: never a stand-in
write(os.path.join(tex, "Bake_Stage_G_2_1_d0_t0.bntx.zs"), bytes(64))
write(os.path.join(tex, "Bake_Stage_G_2_2_d0_t0.bntx.zs"), ZSTD + b"t" * 60)
write(os.path.join(tex, "Bake_Stage_G_2_2_d1_un3.bntx.zs"), ZSTD + b"u" * 60)
write(os.path.join(romfs, "Lonely", "Only_One_Nrm.bntx.zs"), bytes(16)) # nothing of its kind beside it
write(os.path.join(romfs, "Other", "plain.txt"), bytes(16)) # not a .zs: not looked at
write(os.path.join(romfs, "Other", "Skip_A.bntx.zs"), b"\x5a\x2a\x4d\x18" + bytes(8) + ZSTD) # a skippable frame first
damaged = sorted(os.path.relpath(p, romfs) for p in nup.damaged_game_files(romfs))
check("damaged", damaged, sorted(os.path.join(*p) for p in (
("Lonely", "Only_One_Nrm.bntx.zs"), ("Tex", "Bake_Stage_G_2_1_d0_t0.bntx.zs"),
("Tex", "Rock_Temple_Gate_M_01_Nrm.bntx.zs"), ("Tex", "Rock_Temple_Gate_M_02_Nrm.bntx.zs"))))
done = {os.path.relpath(p, romfs): o and os.path.basename(o) for p, o in nup.stand_in_damaged(romfs)}
check("normal map", done.get(os.path.join("Tex", "Rock_Temple_Gate_M_01_Nrm.bntx.zs")), "Rock_Temple_Gate_P_01_Nrm.bntx.zs")
check("second normal map", done.get(os.path.join("Tex", "Rock_Temple_Gate_M_02_Nrm.bntx.zs")), "Rock_Temple_Gate_P_01_Nrm.bntx.zs")
check("bake", done.get(os.path.join("Tex", "Bake_Stage_G_2_1_d0_t0.bntx.zs")), "Bake_Stage_G_2_2_d0_t0.bntx.zs")
check("alone", done.get(os.path.join("Lonely", "Only_One_Nrm.bntx.zs"), "missing"), None)
with open(os.path.join(tex, "Rock_Temple_Gate_M_01_Nrm.bntx.zs"), "rb") as f:
check("replaced bytes", f.read(), ZSTD + b"P" * 500)
check("no .part left", [n for _, _, fs in os.walk(romfs) for n in fs if n.endswith(".part")], [])
check("a second pass", [os.path.relpath(p, romfs) for p, _ in nup.stand_in_damaged(romfs)],
[os.path.join("Lonely", "Only_One_Nrm.bntx.zs")])
check("zstd starts", [nup.zstd_start(h) for h in (ZSTD, b"\x50\x2a\x4d\x18", b"\x5f\x2a\x4d\x18", b"\x60\x2a\x4d\x18",
bytes(4), ZSTD[:3])], [True, True, True, False, False, False])
# make_game's step: only in a romfs the builder unpacked (its stamp), once per unpack
sys.path[:0] = [HERE, os.path.join(os.path.dirname(HERE), "third_party", "a64-kit", "tools")]
mg_spec = importlib.util.spec_from_file_location("tested_make_game", os.path.join(HERE, "make_game.py"))
make_game = importlib.util.module_from_spec(mg_spec)
mg_spec.loader.exec_module(make_game)
calls = []
reported = [] # (whether each call was given a progress(done, total): the check says how far it is, #1764)
sys.modules["nup"] = type(sys)("nup")
sys.modules["nup"].stand_in_damaged = lambda romfs, progress=None: calls.append(romfs) or reported.append(callable(progress)) or []
with tempfile.TemporaryDirectory() as out:
romfs = os.path.join(out, "romfs")
os.makedirs(romfs)
make_game.stand_in_damaged_files(out, romfs)
check("no stamp: not the builder's romfs", calls, [])
with open(os.path.join(out, ".hookshot_romfs"), "w") as f:
f.write("unpacked by this builder: game.nup 1\n")
make_game.stand_in_damaged_files(out, romfs)
make_game.stand_in_damaged_files(out, romfs)
check("checked once", calls, [romfs])
with open(os.path.join(out, ".hookshot_romfs"), "w") as f:
f.write("unpacked by this builder: other.nup 2\n")
make_game.stand_in_damaged_files(out, romfs)
check("checked again for another unpack", calls, [romfs, romfs])
check("each check reports its progress", reported, [True, True])
if failures:
print("\n".join(failures))
sys.exit(1)
print("stand-ins: ok")

169
nx2/test_nup_verify.py Normal file
View File

@@ -0,0 +1,169 @@
"""Synthetic actual-reader verification: python -I -B nx2/test_nup_verify.py.
No game bytes, archive extraction or compiler. Requires existing zstandard dependency.
"""
import argparse
from contextlib import redirect_stdout
import hashlib
import importlib.util
import io
from pathlib import Path
import struct
import subprocess
import sys
import tempfile
import zstandard as zs
RUN_ALL_MODULES = ["zstandard"]
p = argparse.ArgumentParser(description=__doc__)
p.add_argument('--source', type=Path, default=Path(__file__).resolve().parents[1])
a = p.parse_args()
spec = importlib.util.spec_from_file_location('tested_nup', a.source / 'nx2/nup.py')
nup = importlib.util.module_from_spec(spec)
spec.loader.exec_module(nup)
compressor = zs.ZstdCompressor(write_checksum=True)
def package(rows, titles=1):
"""(title index, name, logical bytes, codec); identical contents share extents."""
files_at = 0xb0 + 64 * titles + 24 * titles
strings = b''.join(name.encode() for _, name, _, _ in rows)
strings_at = files_at + 80 * len(rows) + 4 * len(rows)
blob_at = (strings_at + len(strings) + 15) & ~15
blob = bytearray(blob_at)
blob[:4], blob[0x80:0x84] = b'NUP\x01', b'TOC\x01'
struct.pack_into('<HHQQ', blob, 4, 0x80, 1, 0x80, strings_at + len(strings) - 0x80)
struct.pack_into('<IQQQQ', blob, 0x84, titles, titles, len(rows), strings_at - 0x80, len(strings))
struct.pack_into('<Q', blob, 0x38, blob_at)
blob[strings_at:strings_at + len(strings)] = strings
slot = 0
for t in range(titles):
struct.pack_into('<QIIII', blob, 0xb0 + 64 * t, 0x1000 + t, t, 0xffffffff, t, t + 1)
members = [i for i, row in enumerate(rows) if row[0] == t]
struct.pack_into('<4I', blob, 0xb0 + 64 * titles + 24 * t, 0, 0, slot, len(members))
for i in members:
struct.pack_into('<I', blob, files_at + 80 * len(rows) + 4 * slot, titles + i)
slot += 1
name_at, seen = 0, {}
for i, (_, name, logical, codec) in enumerate(rows):
stored = compressor.compress(logical) if codec else logical
key = (stored, codec)
if key not in seen:
blob.extend(bytes((-len(blob)) % 16))
seen[key] = len(blob)
blob.extend(stored)
at = files_at + 80 * i
struct.pack_into('<IIQQQ', blob, at, name_at, len(name.encode()), len(logical), seen[key], len(stored))
blob[at + 32:at + 64] = hashlib.sha256(logical).digest()
blob[at + 64] = codec
name_at += len(name.encode())
struct.pack_into('<Q', blob, 0x40, len(blob))
return blob, files_at
def seal(blob):
offset, size = struct.unpack_from('<QQ', blob, 8)
blob[24:56] = hashlib.sha256(blob[offset:offset + size]).digest()
with tempfile.TemporaryDirectory(prefix='nup-verify-') as temp:
path = Path(temp) / 'synthetic.nup'
def check(blob, bad, titles=None):
seal(blob)
path.write_bytes(blob)
reader = nup.Nup(str(path))
calls, copy = [], reader.copy_to
def counted(*args, **kwargs):
calls.append(args[0])
return copy(*args, **kwargs)
reader.copy_to = counted
try:
with redirect_stdout(io.StringIO()) as output:
result = nup.verify(reader, titles)
assert result['bad'] == bad, output.getvalue()
return result, len(calls), output.getvalue()
finally:
reader.f.close()
game = compressor.compress(b'self-authored game payload' * 500)
good, at = package([(0, 'raw.bin', b'raw bytes', 0), (0, 'raw.zs', game, 0), (0, 'outer.zs', game, 1)])
assert check(good, 0)[0]['checked_records'] == 3
empty, at = package([(0, 'empty.bin', b'', 0)])
struct.pack_into('<QQ', empty, at + 16, 0, 0) # canonical empty extent
check(empty, 0)
# Hash-valid zero prefix: container integrity alone must not bless the game asset.
damaged, at = package([(0, 'bad.zs', bytes(12288) + b'not a Zstd frame', 1)])
assert 'expected game Zstd header' in check(damaged, 1)[2]
dedup, at = package([(0, 'one.bin', b'shared bytes', 0), (0, 'two.bin', b'shared bytes', 0)])
dedup[at + 80 + 32] ^= 1
result, calls, output = check(dedup, 1)
assert calls == result['decoded_extents'] == 1 and 'SHA-256 mismatch' in output
for problem in ('checksum', 'truncated', 'trailing', 'bounds', 'codec', 'flags', 'reserved'):
blob, at = package([(0, 'test.zs', game, 1)])
off, size = struct.unpack_from('<QQ', blob, at + 16)
if problem == 'checksum': blob[off + size - 1] ^= 1
elif problem == 'truncated': struct.pack_into('<Q', blob, at + 24, size - 1)
elif problem == 'trailing':
blob.extend(b'x')
struct.pack_into('<Q', blob, 0x40, len(blob))
struct.pack_into('<Q', blob, at + 24, size + 1)
elif problem == 'bounds': struct.pack_into('<Q', blob, at + 16, (len(blob) + 15) & ~15)
else: blob[at + {'codec': 64, 'flags': 65, 'reserved': 66}[problem]] = 9
check(blob, 1)
blob, at = package([(0, 'bad.zs', bytes(32), 0), (1, 'good.zs', game, 0)], titles=2)
check(blob, 1)
selected = dict(index=1, id=0x1001, version=2, root=1, exefs=0xffffffff, romfs=1)
check(blob, 0, [selected])
before = path.read_bytes()
cli = [sys.executable, '-I', '-B', str(a.source / 'nx2/nup.py'), str(path), '--verify', '--only', 'romfs']
assert subprocess.run(cli, capture_output=True).returncode == 1
assert subprocess.run(cli + ['--title', '0000000000001001'], capture_output=True).returncode == 0
assert path.read_bytes() == before
# Rehashed short TOC must not leave parsed tables outside the hashed region.
def cli_bad(blob, reason, rehash=True):
if rehash:
seal(blob)
path.write_bytes(blob)
before = path.read_bytes()
result = subprocess.run(cli, capture_output=True, text=True, encoding='utf-8', timeout=5)
assert result.returncode == 1 and 'BAD package:' in result.stdout, (result.stdout, result.stderr)
assert reason in result.stdout and 'Traceback' not in result.stderr, (reason, result.stdout, result.stderr)
assert path.read_bytes() == before
for problem, reason in (
('short-toc', 'exceed declared TOC'), ('strings', 'exceed declared TOC'),
('child', 'child index'), ('directory-range', 'child range'),
('title', 'invalid directory index'), ('cycle', 'directory cycle'),
('orphan', 'orphan'), ('utf8', 'utf-8')):
blob, at = package([(0, 'test.zs', game, 0)])
dirs_at, kids_at = 0xb0 + 64, at + 80
if problem == 'short-toc': struct.pack_into('<Q', blob, 0x10, 0x30)
elif problem == 'strings': struct.pack_into('<Q', blob, 0xa0, 1 << 20)
elif problem == 'child': struct.pack_into('<I', blob, kids_at, 99)
elif problem == 'directory-range': struct.pack_into('<I', blob, dirs_at + 8, 1)
elif problem == 'title': struct.pack_into('<I', blob, 0xb0 + 8, 1)
elif problem == 'cycle': struct.pack_into('<I', blob, kids_at, 0)
elif problem == 'orphan': struct.pack_into('<I', blob, dirs_at + 12, 0)
elif problem == 'utf8': blob[kids_at + 4] = 0xff
cli_bad(blob, reason)
blob, at = package([(0, 'one.bin', b'one', 0), (1, 'two.bin', b'two', 0)], titles=2)
struct.pack_into('<I', blob, 0xb0 + 16, 1)
cli_bad(blob, 'subtree is not reachable')
struct.pack_into('<I', blob, 0xb0 + 16, 0)
struct.pack_into('<I', blob, 0xb0 + 64 + 8, 0)
struct.pack_into('<I', blob, 0xb0 + 64 + 16, 0)
cli_bad(blob, 'orphan')
blob, at = package([(0, 'test.zs', game, 0)])
seal(blob)
struct.pack_into('<I', blob, at + 80, 2) # invalid child AND unhashed TOC
cli_bad(blob, 'TOC SHA-256 mismatch', rehash=False)
assert 'COM\u00b9' in nup.WINDOWS_DEVICES and 'LPT\u00b3' in nup.WINDOWS_DEVICES
try:
nup.extraction_path(temp, 'romfs', 'COM\u00b9/file')
except SystemExit:
pass
else:
raise AssertionError('base superscript Windows device protection changed')
print('PASS: raw/codec1, dedup hashes, hash-valid damaged .zs, checksum/extent/metadata failures, title scope, strict TOC/tree rejection and read-only CLI')

View File

@@ -0,0 +1,69 @@
"""Check Sirit keeps independently decorated structures separate.
python nx2/test_spirv_struct_identity.py --headers <SPIRV-Headers/include> --spirv-val spirv-val
No GPU or game files needed. Requires a configured C++ compiler.
"""
import argparse
from pathlib import Path
import subprocess
import tempfile
RUN_ALL_ARGS = ["--headers", "{spirv_headers}", "--spirv-val", "{spirv_val}"]
# (spirv-val: SPIRV-Tools comes with the Linux runner's packages, not on Windows)
RUN_ALL_SKIP = {"windows": "needs spirv-val: checked on Linux"}
ROOT=Path(__file__).resolve().parents[1]
def main():
p=argparse.ArgumentParser(description=__doc__)
p.add_argument('--headers',type=Path,required=True);p.add_argument('--spirv-val',required=True);p.add_argument('--cxx',default='clang++');a=p.parse_args()
fixture=r'''
#include <sirit/sirit.h>
#include <cstdio>
#include <string>
#include <vector>
using namespace Sirit;
int main(int argc,char** argv) {
if(argc!=2)return 1;
for(unsigned local=0;local<2;local++) {
Module m{0x00010600};m.AddCapability(spv::Capability::Shader);m.AddCapability(spv::Capability::Int64);
m.AddCapability(spv::Capability::PhysicalStorageBufferAddresses);
m.SetMemoryModel(spv::AddressingModel::PhysicalStorageBuffer64,spv::MemoryModel::GLSL450);
const Id u32=m.TypeUInt(32),u64=m.TypeUInt(64),void_t=m.TypeVoid(),zero=m.Constant(u32,0U);
std::vector<Id> interfaces;
// Mirrors first-task, Private local memory, and global-memory wrapper declarations.
auto block=[&]{Id t=m.TypeStruct(u32);m.Decorate(t,spv::Decoration::Block);m.MemberDecorate(t,0U,spv::Decoration::Offset,0U);return t;};
const Id first=block();
if(local) {
const Id element=m.TypeStruct(u32),array=m.TypeArray(element,m.Constant(u32,4U));
interfaces.push_back(m.AddGlobalVariable(m.TypePointer(spv::StorageClass::Private,array),spv::StorageClass::Private));
}
const Id global=block(),member=m.TypePointer(spv::StorageClass::PhysicalStorageBuffer,u32);
const Id first_ptr=m.TypePointer(spv::StorageClass::PhysicalStorageBuffer,first);
const Id global_ptr=m.TypePointer(spv::StorageClass::PhysicalStorageBuffer,global);
const Id fn=m.OpFunction(void_t,spv::FunctionControlMask::MaskNone,m.TypeFunction(void_t));m.AddLabel();
const Id address=m.Constant(u64,uint64_t{16});
m.OpLoad(u32,m.OpAccessChain(member,m.OpConvertUToPtr(first_ptr,address),zero),spv::MemoryAccessMask::Aligned,4U);
const Id ptr=m.OpAccessChain(member,m.OpConvertUToPtr(global_ptr,address),zero);
m.OpStore(ptr,zero,spv::MemoryAccessMask::Aligned,4U);
m.OpAtomicIAdd(u32,ptr,m.Constant(u32,1U),zero,zero);
m.OpReturn();m.OpFunctionEnd();m.AddEntryPoint(spv::ExecutionModel::GLCompute,fn,"main",interfaces);
m.AddExecutionMode(fn,spv::ExecutionMode::LocalSize,1U,1U,1U);
auto words=m.Assemble();std::string path=std::string(argv[1])+"/"+std::to_string(local)+".spv";
FILE* f=std::fopen(path.c_str(),"wb");if(!f)return 2;
const auto written=std::fwrite(words.data(),4,words.size(),f);std::fclose(f);if(written!=words.size())return 3;
// Scalar and pointer declarations must retain their existing reuse.
if(m.TypeUInt(32).value!=u32.value || m.TypePointer(spv::StorageClass::PhysicalStorageBuffer,u32).value!=member.value)return 4;
}
}
'''
work=ROOT/'work';work.mkdir(exist_ok=True)
with tempfile.TemporaryDirectory(prefix='struct-identity-',dir=work) as temp:
folder=Path(temp).resolve();cpp=folder/'fixture.cpp';exe=folder/'fixture.exe';cpp.write_text(fixture,encoding='utf-8');sirit=ROOT/'gpu/vk/sirit'
subprocess.run([a.cxx,'-std=c++20','-O1','-I'+str(sirit/'include'),'-I'+str(sirit/'src'),'-I'+str(a.headers),str(cpp),*map(str,sorted((sirit/'src').rglob('*.cpp'))),'-o',str(exe)],check=True)
subprocess.run([str(exe),str(folder)],check=True)
for case in range(2):subprocess.run([a.spirv_val,'--target-env','vulkan1.3',str(folder/f'{case}.spv')],check=True)
print('PASS: two physical-buffer modules, with/without Private local memory; scalar/pointer reuse preserved')
if __name__=='__main__':main()

View File

@@ -0,0 +1,82 @@
"""Execute create()'s image initialization with Vulkan command spies; no GPU/loader.
Covers Segmentree's initialization dependency fix in PR #228.
Run in a compiler environment: python -I -B nx2/test_texture_init_barrier.py --cxx clang++ --headers path/to/include
"""
import argparse
from pathlib import Path
import subprocess
import tempfile
RUN_ALL_ARGS = ["--headers", "{vulkan_headers}"]
ROOT = Path(__file__).resolve().parents[1]
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument('--cxx', default='clang++')
p.add_argument('--headers', required=True, type=Path)
p.add_argument('--source', type=Path, default=ROOT / 'gpu/vk/vk_texture.cpp')
a = p.parse_args()
text = a.source.read_text(encoding='utf-8')
start = text.index(' // into GENERAL, and cleared', text.index('Texture* create('))
fragment = text[start:text.index(' Texture* p = tex.get();', start)]
fixture = r'''
#define VK_NO_PROTOTYPES
#include <vulkan/vulkan_core.h>
#include <cstdio>
#include <cstdlib>
#define CHECK(v) do {if (!(v)) {std::fprintf(stderr,"FAIL:%d %s\n",__LINE__,#v);std::exit(1);}} while(0)
int commands, clears, dependencies;
VkImage image = reinterpret_cast<VkImage>(1);
VkCommandBuffer upload_cmd() {return reinterpret_cast<VkCommandBuffer>(1);}
bool is_bc(VkFormat f) {return f == VK_FORMAT_BC3_UNORM_BLOCK;}
bool is_astc(VkFormat f) {return f == VK_FORMAT_ASTC_4x4_UNORM_BLOCK;}
void vkCmdPipelineBarrier2(VkCommandBuffer, const VkDependencyInfo* di) {
CHECK(di->imageMemoryBarrierCount == 1);
const auto& b = *di->pImageMemoryBarriers;
CHECK(b.image == image && b.newLayout == VK_IMAGE_LAYOUT_GENERAL);
CHECK(b.subresourceRange.baseMipLevel == 0 && b.subresourceRange.levelCount == VK_REMAINING_MIP_LEVELS);
CHECK(b.subresourceRange.baseArrayLayer == 0 && b.subresourceRange.layerCount == VK_REMAINING_ARRAY_LAYERS);
CHECK(b.srcQueueFamilyIndex == VK_QUEUE_FAMILY_IGNORED && b.dstQueueFamilyIndex == VK_QUEUE_FAMILY_IGNORED);
if (b.oldLayout == VK_IMAGE_LAYOUT_UNDEFINED) {CHECK(commands == 0);}
else {
CHECK(commands == 2 && clears == 1 && b.oldLayout == VK_IMAGE_LAYOUT_GENERAL);
CHECK(b.srcStageMask & VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT);
CHECK(b.srcAccessMask & VK_ACCESS_2_TRANSFER_WRITE_BIT);
CHECK(b.dstStageMask & VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT);
CHECK((b.dstAccessMask & (VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT)) ==
(VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT));
dependencies++;
}
commands++;
}
void clear(VkImage i, VkImageLayout layout, uint32_t n, const VkImageSubresourceRange* range) {
CHECK(commands == 1 && i == image && layout == VK_IMAGE_LAYOUT_GENERAL && n == 1);
CHECK(range->levelCount == VK_REMAINING_MIP_LEVELS && range->layerCount == VK_REMAINING_ARRAY_LAYERS);
clears++;commands++;
}
void vkCmdClearDepthStencilImage(VkCommandBuffer, VkImage i, VkImageLayout l, const VkClearDepthStencilValue*, uint32_t n, const VkImageSubresourceRange* r) {clear(i,l,n,r);}
void vkCmdClearColorImage(VkCommandBuffer, VkImage i, VkImageLayout l, const VkClearColorValue*, uint32_t n, const VkImageSubresourceRange* r) {clear(i,l,n,r);}
void run(VkFormat format, bool depth) {
commands = clears = dependencies = 0;
struct {VkImage image;VkFormat format;VkImageAspectFlags aspect;} x{image,format,VkImageAspectFlags(depth ? VK_IMAGE_ASPECT_DEPTH_BIT : VK_IMAGE_ASPECT_COLOR_BIT)};
''' + fragment + r'''
const bool compressed = is_bc(format) || is_astc(format);
CHECK(clears == !compressed && dependencies == !compressed);
CHECK(commands == (compressed ? 1 : 3));
}
int main() {
run(VK_FORMAT_R8_UNORM,false);run(VK_FORMAT_R8G8B8A8_UNORM,false);
run(VK_FORMAT_D32_SFLOAT,true);run(VK_FORMAT_BC3_UNORM_BLOCK,false);run(VK_FORMAT_ASTC_4x4_UNORM_BLOCK,false);
puts("PASS: actual create initialization orders colour/depth clears before dependent commands; BC/ASTC excluded");
}
'''
with tempfile.TemporaryDirectory(prefix='hookshot-init-spy-') as temp:
cpp = Path(temp) / 'spy.cpp'
exe = Path(temp) / 'spy.exe'
cpp.write_text(fixture, encoding='utf-8')
subprocess.run([a.cxx, '-std=c++20', '-I' + str(a.headers), str(cpp), '-o', str(exe)], check=True)
subprocess.run([str(exe)], check=True)
if __name__ == '__main__':
main()

View File

@@ -0,0 +1,58 @@
"""Real TexturePass/Collect/Emit conditional descriptor-index regression; synthetic inputs only."""
# SPDX-License-Identifier: GPL-3.0-or-later
# Build pattern retained from test_depth_sample_types.py.
import argparse
import hashlib
from pathlib import Path
import subprocess
import tempfile
RUN_ALL_ARGS = ["--headers", "{spirv_headers}", "--spirv-val", "{spirv_val}"]
# (spirv-val: SPIRV-Tools comes with the Linux runner's packages, not on Windows)
RUN_ALL_SKIP = {"windows": "needs spirv-val: checked on Linux"}
ROOT = Path(__file__).resolve().parents[1]
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--cxx', default='clang++')
parser.add_argument('--headers', type=Path, required=True)
parser.add_argument('--spirv-val', default='spirv-val')
parser.add_argument('--source', type=Path, default=ROOT)
args = parser.parse_args()
root = args.source.resolve()
vk = root / 'gpu/vk'
sources = sorted({p for directory in ('eden/shader_recompiler/backend/spirv',
'eden/shader_recompiler/frontend/ir', 'eden/shader_recompiler/ir_opt', 'sirit/src')
for p in (vk / directory).rglob('*.cpp')})
flags = ['-std=c++20', '-O1', '-D_CRT_SECURE_NO_WARNINGS', '-DVK_NO_PROTOTYPES', '-DMKWVK_INCLUDES=2']
for header in ('algorithm', 'cstdio', 'cstdlib', 'cstring', 'functional', 'string', 'vector'):
flags += ['-include', header]
flags += ['-I' + str(p) for p in (vk / 'compat', root / 'gpu/compat', vk / 'eden',
root / 'gpu/third_party/suyu', vk / 'sirit/include', vk / 'sirit/src', args.headers)]
with tempfile.TemporaryDirectory(prefix='texture-phi-') as temp:
folder = Path(temp).resolve()
exe = folder / 'fixture.exe'
subprocess.run([args.cxx, *flags, str(ROOT / 'tests/texture_phi_dominance_ir.cpp'),
*map(str, sources), '-o', str(exe)], check=True)
emitted = subprocess.run([str(exe), str(folder)], capture_output=True, text=True)
print(emitted.stdout, end='', flush=True)
print(emitted.stderr, end='', flush=True)
emitted.check_returncode()
failures = []
for name in ('same-block', 'diamond', 'offset-phi', 'repeated-condition'):
print(name, 'sha256', hashlib.sha256((folder / (name + '.spv')).read_bytes()).hexdigest(), flush=True)
result = subprocess.run([args.spirv_val, '--target-env', 'vulkan1.3', str(folder / (name + '.spv'))], capture_output=True, text=True)
print(name, 'validator-exit', result.returncode, result.stdout, result.stderr, flush=True)
if result.returncode:
failures.append(name + ': SPIR-V validation')
if 'IR_DOMINANCE=FAIL' in emitted.stdout:
failures.append('rewritten IR directly consumes a branch-local index')
if 'SELECTION=FAIL' in emitted.stdout:
failures.append('rewritten index selects the wrong incoming texture')
if failures:
raise AssertionError('; '.join(failures))
print('PASS: conditional handle indices preserve SSA dominance; positive control validates')
if __name__ == '__main__':
main()

View File

@@ -0,0 +1,81 @@
"""Exercise the actual fence allocators across ordered, dump/legacy and queued paths; no GPU/game.
python nx2/test_timeline_modes.py [--cxx clang++]
"""
import argparse
from pathlib import Path
import subprocess,tempfile
RUN_ALL_ARGS = ["--cxx", "g++"]
ROOT=Path(__file__).resolve().parents[1]
def block(text,marker):
start=text.index(marker);end=text.index('{',start)+1;depth=1
while depth:
depth+=(text[end]=='{')-(text[end]=='}');end+=1
return text[start:end]
def main():
p=argparse.ArgumentParser(description=__doc__);p.add_argument('--cxx',default='clang++');a=p.parse_args()
s=(ROOT/'gpu/vk/vk_replay.cpp').read_text(encoding='utf-8')
direct=block(s,'void do_submit(')
start=s.index(' const uint64_t value = ',s.index('void do_present('));end=s.index('\n',s.index(' present_value = 0;',start))
present=s[start:end]
source_submit=block(s,'void submit_q(');source_present=block(s,'void present(')
# (8c28ed7 takes the value first, for the queue it belongs to, then submits it: both lines, as the source has them)
ordered_submit='\n'.join(next(l for l in source_submit.splitlines() if text in l)
for text in ('const uint64_t value = ++q_values;','submit_job(h, value, queue);'))
ordered_present=next(l for l in source_present.splitlines() if 'present_value = ++q_values;' in l)
enqueue=next(l for l in block(s,'void enqueue(').splitlines() if 'j.value = ++q_values;' in l)
fixture=r'''#include <algorithm>
#include <atomic>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <vector>
#define CHECK(x) do{if(!(x)){std::fprintf(stderr,"FAIL:%d %s\n",__LINE__,#x);std::exit(1);}}while(0)
std::atomic<uint64_t> pushed{0},q_values{0};uint64_t present_value=0,last=0;unsigned allocations=0;
void signal(uint64_t v){CHECK(v>last);last=v;pushed=v;allocations++;}
bool run_submit(const uint64_t*,int,uint64_t value,uint64_t,bool){signal(value);return true;}
void submit_job(std::vector<uint64_t>&,uint64_t value,uint64_t){signal(value);}
''' +direct+r'''
void direct_present(){
''' +present+r'''
signal(value);
}
void ordered_submit(){std::vector<uint64_t>h;uint64_t queue=0;
''' +ordered_submit+r'''
}
void ordered_present(){
''' +ordered_present+r'''
const auto reserved=q_values.load();direct_present();CHECK(q_values.load()==reserved);
}
enum Kind{JOB_SUBMIT,JOB_PRESENT};struct Job{Kind kind;uint64_t value=0;};
void queued(Kind kind){Job j{kind};
''' +enqueue+r'''
if(kind==JOB_PRESENT){present_value=j.value;const auto reserved=q_values.load();direct_present();CHECK(q_values.load()==reserved);}
else signal(j.value);
}
int main(){
// Normal replay -> frame dump (legacy replay) -> normal replay, repeated.
for(unsigned frame=0;frame<100;frame++){
ordered_submit();ordered_present();do_submit(nullptr,0);direct_present();ordered_submit();ordered_present();
}
CHECK(last==600 && q_values==600 && allocations==600);
// Non-dump legacy-only setting; queued paths retain their preassigned values.
for(unsigned i=0;i<100;i++){do_submit(nullptr,0);direct_present();queued(JOB_SUBMIT);queued(JOB_PRESENT);}
CHECK(last==1000 && q_values==1000 && allocations==1000);
// Allocated ordered work can be ahead of the last GPU submission.
q_values=1100;do_submit(nullptr,0);CHECK(last==1101 && q_values==1101);
puts("PASS: 100 dump transitions, legacy/queued paths, no present double-allocation, reserved work ahead of pushed");
}
'''
work=ROOT/'work';work.mkdir(exist_ok=True)
with tempfile.TemporaryDirectory(prefix='timeline-modes-',dir=work) as temp:
d=Path(temp).resolve();d.relative_to(work.resolve());cpp=d/'fixture.cpp';exe=d/'fixture.exe';cpp.write_text(fixture,encoding='utf-8')
subprocess.run([a.cxx,'-std=c++20','-O2',str(cpp),'-o',str(exe)],check=True);subprocess.run([str(exe)],check=True)
# Both synchronous branches must retain ordering while allocating/recording, including dump mode.
for body in (source_submit,source_present):
branch=body[body.index(' if (!async_on()) {'):]
assert branch.index('sync_order_mu')<branch.index('if (sync_waits_on())')
if __name__=='__main__':main()

View File

@@ -0,0 +1,63 @@
"""Validate real texture-pass unsigned image types without game files or a GPU.
python nx2/test_unsigned_image_types.py --headers <SPIRV-Headers/include> --spirv-val spirv-val [--cxx clang++]
Builds the same Eden IR/SPIR-V and Sirit sources as the renderer, with its compatibility headers.
"""
# SPDX-License-Identifier: GPL-3.0-or-later
import argparse
from pathlib import Path
import subprocess
import tempfile
RUN_ALL_ARGS = ["--cxx", "{cxx}", "--headers", "{spirv_headers}", "--spirv-val", "{spirv_val}"]
# (spirv-val: SPIRV-Tools comes with the Linux runner's packages, not on Windows)
RUN_ALL_SKIP = {"windows": "needs spirv-val: checked on Linux"}
ROOT = Path(__file__).resolve().parents[1]
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--cxx', default='clang++')
parser.add_argument('--headers', type=Path, required=True)
parser.add_argument('--spirv-val', default='spirv-val')
args = parser.parse_args()
vk = ROOT / 'gpu/vk'
sources = sorted({p for directory in ('eden/shader_recompiler/backend/spirv',
'eden/shader_recompiler/frontend/ir', 'eden/shader_recompiler/ir_opt', 'sirit/src')
for p in (vk / directory).rglob('*.cpp')})
flags = ['-std=c++20', '-O1', '-D_CRT_SECURE_NO_WARNINGS', '-DVK_NO_PROTOTYPES', '-DMKWVK_INCLUDES=2']
for header in ('algorithm', 'cstdio', 'cstdlib', 'cstring', 'functional', 'string', 'vector'):
flags += ['-include', header]
flags += ['-I' + str(p) for p in (vk / 'compat', ROOT / 'gpu/compat', vk / 'eden',
ROOT / 'gpu/third_party/suyu', vk / 'sirit/include', vk / 'sirit/src', args.headers)]
work = ROOT / 'work'
work.mkdir(exist_ok=True)
with tempfile.TemporaryDirectory(prefix='unsigned-image-types-', dir=work) as temp:
folder = Path(temp).resolve()
exe = folder / 'fixture.exe'
subprocess.run([args.cxx, *flags, str(ROOT / 'tests/unsigned_image_types_ir.cpp'),
*map(str, sources), '-o', str(exe)], check=True)
emitted = subprocess.run([str(exe), str(folder)], check=True, capture_output=True, text=True)
print(emitted.stderr, end='')
cases = ('r32-read', 'r32-write', 'buffer-r32-write', 'atomic-r32-add',
'atomic-r32-smin', 'typeless-read', 'typeless-write')
for case in cases:
floating, integer = (folder / f'{case}-{kind}.spv' for kind in ('float', 'int'))
for module in (floating, integer):
subprocess.run([args.spirv_val, '--target-env', 'vulkan1.3', str(module)], check=True)
if not case.startswith('typeless-'):
assert floating.read_bytes() == integer.read_bytes(), f'{case}: unsigned format depends on Env answer'
counts = dict(line.split(' questions=') for line in emitted.stderr.splitlines())
assert len(counts) == 14, emitted.stderr
for case in cases:
for kind in ('float', 'int'):
expected = '1' if case.startswith('typeless-') else '0'
assert counts[f'{case}-{kind}'] == expected, emitted.stderr
print('PASS: explicit unsigned image types ignore Env answers; Typeless queries retained; 14 modules validate')
if __name__ == '__main__':
main()

47
nx2/test_version.py Normal file
View File

@@ -0,0 +1,47 @@
"""nx2/version.py, game-free: where the release name comes from, and a fingerprint that ignores line endings.
python -I -B nx2/test_version.py
"""
import os
import sys
import tempfile
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import version # noqa: E402
def tree(root, eol):
for rel in ("host/a.c", "gpu/vk/b.cpp", "nx2/c.py"):
p = os.path.join(root, rel)
os.makedirs(os.path.dirname(p), exist_ok=True)
with open(p, "wb") as f:
f.write(("line one%sline two%s" % (eol, eol)).encode())
def main():
os.environ.pop("HOOKSHOT_VERSION", None)
with tempfile.TemporaryDirectory() as lf, tempfile.TemporaryDirectory() as crlf:
tree(lf, "\n")
tree(crlf, "\r\n")
assert version.release(lf) == "unknown", version.release(lf)
assert version.fingerprint(lf) == version.fingerprint(crlf), "line endings changed the fingerprint"
with open(os.path.join(lf, "nx2", "c.py"), "a") as f:
f.write("changed\n")
assert version.fingerprint(lf) != version.fingerprint(crlf), "a changed source kept the fingerprint"
with open(os.path.join(lf, "VERSION"), "w") as f:
f.write("movie-sync\nignored second line\n")
assert version.release(lf) == "movie-sync"
os.environ["HOOKSHOT_VERSION"] = "from-env"
assert version.release(lf) == "from-env", "HOOKSHOT_VERSION comes first"
os.environ["HOOKSHOT_VERSION"] = 'v1 "quoted" \\ & | %PATH%'
d = version.describe(lf)
assert not set(d) & set('"\\&|%'), d
assert version.define(lf).startswith('-DHOOKSHOT_VERSION="') and version.define(lf).endswith(')"')
os.environ.pop("HOOKSHOT_VERSION")
print("version: ok (%s)" % version.describe())
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,61 @@
// Covers the initialization dependency fixed by Segmentree in PR #228.
// Build: clang++ -std=c++20 -O2 -I"%VULKAN_SDK%/Include" nx2/texture_init_dependency_probe.cpp -o texture_init_dependency_probe.exe
// Set VK_LAYER_PATH to the Khronos validation layer directory; run the executable with old, then fixed.
// API dependency probe only: no game data and no claim about coloured-noise causality.
// Run with Khronos validation installed. old: expect SYNC-HAZARD-WRITE-AFTER-WRITE;
// fixed: expect no validation errors. Uses the exact PR228 dependency scopes.
#define WIN32_LEAN_AND_MEAN
#define VK_NO_PROTOTYPES
#include <windows.h>
#include <vulkan/vulkan.h>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <vector>
#define CHECK(v) do {if(!(v)){std::fprintf(stderr,"FAIL:%d %s\n",__LINE__,#v);std::exit(2);}}while(0)
#define APIS(X) X(vkCreateInstance) X(vkDestroyInstance) X(vkEnumeratePhysicalDevices) X(vkGetPhysicalDeviceProperties) X(vkGetPhysicalDeviceQueueFamilyProperties) X(vkGetPhysicalDeviceMemoryProperties) X(vkCreateDevice) X(vkDestroyDevice) X(vkGetDeviceQueue) X(vkCreateImage) X(vkDestroyImage) X(vkGetImageMemoryRequirements) X(vkAllocateMemory) X(vkFreeMemory) X(vkBindImageMemory) X(vkCreateBuffer) X(vkDestroyBuffer) X(vkGetBufferMemoryRequirements) X(vkBindBufferMemory) X(vkMapMemory) X(vkUnmapMemory) X(vkCreateCommandPool) X(vkDestroyCommandPool) X(vkAllocateCommandBuffers) X(vkBeginCommandBuffer) X(vkEndCommandBuffer) X(vkCmdPipelineBarrier2) X(vkCmdClearColorImage) X(vkCmdCopyBufferToImage) X(vkQueueSubmit) X(vkQueueWaitIdle)
#define DECL(n) PFN_##n n;
APIS(DECL)
int errors=0,hazards=0;
VKAPI_ATTR VkBool32 VKAPI_CALL debug(VkDebugUtilsMessageSeverityFlagBitsEXT severity,VkDebugUtilsMessageTypeFlagsEXT,const VkDebugUtilsMessengerCallbackDataEXT* data,void*) {
if(severity & VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT){errors++;std::fprintf(stderr,"%s: %s\n",data->pMessageIdName ? data->pMessageIdName : "(no message ID)",data->pMessage);}
if((data->pMessageIdName && std::strstr(data->pMessageIdName,"SYNC-HAZARD-WRITE-AFTER-WRITE")) || std::strstr(data->pMessage,"WRITE_AFTER_WRITE"))hazards++;
return VK_FALSE;
}
int main(int argc,char**argv) {
const bool fixed=argc==2 && !std::strcmp(argv[1],"fixed");CHECK(fixed || (argc==2 && !std::strcmp(argv[1],"old")));
auto loader=LoadLibraryA("vulkan-1.dll");CHECK(loader);
#define LOAD(n) n=(PFN_##n)GetProcAddress(loader,#n);CHECK(n);
APIS(LOAD)
auto gip=(PFN_vkGetInstanceProcAddr)GetProcAddress(loader,"vkGetInstanceProcAddr");CHECK(gip);
VkValidationFeatureEnableEXT feature=VK_VALIDATION_FEATURE_ENABLE_SYNCHRONIZATION_VALIDATION_EXT;
VkValidationFeaturesEXT vf{VK_STRUCTURE_TYPE_VALIDATION_FEATURES_EXT};vf.enabledValidationFeatureCount=1;vf.pEnabledValidationFeatures=&feature;
VkDebugUtilsMessengerCreateInfoEXT db{VK_STRUCTURE_TYPE_DEBUG_UTILS_MESSENGER_CREATE_INFO_EXT};
db.messageSeverity=VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT;db.messageType=VK_DEBUG_UTILS_MESSAGE_TYPE_VALIDATION_BIT_EXT;db.pfnUserCallback=debug;db.pNext=&vf;
const char* layers[]={"VK_LAYER_KHRONOS_validation"};const char* extensions[]={VK_EXT_DEBUG_UTILS_EXTENSION_NAME,VK_EXT_VALIDATION_FEATURES_EXTENSION_NAME};
VkApplicationInfo app{VK_STRUCTURE_TYPE_APPLICATION_INFO};app.apiVersion=VK_API_VERSION_1_3;
VkInstanceCreateInfo ici{VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO};ici.pNext=&db;ici.pApplicationInfo=&app;ici.enabledLayerCount=1;ici.ppEnabledLayerNames=layers;ici.enabledExtensionCount=2;ici.ppEnabledExtensionNames=extensions;
VkInstance instance{};CHECK(vkCreateInstance(&ici,nullptr,&instance)==VK_SUCCESS);
auto createDebug=(PFN_vkCreateDebugUtilsMessengerEXT)gip(instance,"vkCreateDebugUtilsMessengerEXT");auto destroyDebug=(PFN_vkDestroyDebugUtilsMessengerEXT)gip(instance,"vkDestroyDebugUtilsMessengerEXT");CHECK(createDebug && destroyDebug);
db.pNext=nullptr;VkDebugUtilsMessengerEXT messenger{};CHECK(createDebug(instance,&db,nullptr,&messenger)==VK_SUCCESS);
uint32_t n=0;CHECK(vkEnumeratePhysicalDevices(instance,&n,nullptr)==VK_SUCCESS && n);std::vector<VkPhysicalDevice> devices(n);CHECK(vkEnumeratePhysicalDevices(instance,&n,devices.data())==VK_SUCCESS);
auto physical=devices[0];for(auto d:devices){VkPhysicalDeviceProperties props{};vkGetPhysicalDeviceProperties(d,&props);if(props.vendorID==0x1002)physical=d;}
VkPhysicalDeviceProperties props{};vkGetPhysicalDeviceProperties(physical,&props);std::printf("device: %s; mode: %s\n",props.deviceName,argv[1]);
vkGetPhysicalDeviceQueueFamilyProperties(physical,&n,nullptr);std::vector<VkQueueFamilyProperties> families(n);vkGetPhysicalDeviceQueueFamilyProperties(physical,&n,families.data());uint32_t family=0;while(family<n && !(families[family].queueFlags & VK_QUEUE_GRAPHICS_BIT))family++;CHECK(family<n);
float priority=1;VkDeviceQueueCreateInfo qi{VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO};qi.queueFamilyIndex=family;qi.queueCount=1;qi.pQueuePriorities=&priority;
VkPhysicalDeviceVulkan13Features f13{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_3_FEATURES};f13.synchronization2=VK_TRUE;
VkDeviceCreateInfo dci{VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO};dci.pNext=&f13;dci.queueCreateInfoCount=1;dci.pQueueCreateInfos=&qi;VkDevice device{};CHECK(vkCreateDevice(physical,&dci,nullptr,&device)==VK_SUCCESS);VkQueue queue{};vkGetDeviceQueue(device,family,0,&queue);
VkPhysicalDeviceMemoryProperties memory{};vkGetPhysicalDeviceMemoryProperties(physical,&memory);
auto allocate=[&](VkMemoryRequirements r,VkMemoryPropertyFlags flags){uint32_t i=0;while(i<memory.memoryTypeCount && (!(r.memoryTypeBits & (1u<<i)) || (memory.memoryTypes[i].propertyFlags & flags)!=flags))i++;CHECK(i<memory.memoryTypeCount);VkMemoryAllocateInfo ai{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};ai.allocationSize=r.size;ai.memoryTypeIndex=i;VkDeviceMemory m{};CHECK(vkAllocateMemory(device,&ai,nullptr,&m)==VK_SUCCESS);return m;};
VkImageCreateInfo ii{VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO};ii.imageType=VK_IMAGE_TYPE_2D;ii.format=VK_FORMAT_R8_UNORM;ii.extent={16,16,1};ii.mipLevels=ii.arrayLayers=1;ii.samples=VK_SAMPLE_COUNT_1_BIT;ii.tiling=VK_IMAGE_TILING_OPTIMAL;ii.usage=VK_IMAGE_USAGE_TRANSFER_DST_BIT;VkImage image{};CHECK(vkCreateImage(device,&ii,nullptr,&image)==VK_SUCCESS);
VkMemoryRequirements r{};vkGetImageMemoryRequirements(device,image,&r);auto im=allocate(r,VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);CHECK(vkBindImageMemory(device,image,im,0)==VK_SUCCESS);
VkBufferCreateInfo bi{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO};bi.size=256;bi.usage=VK_BUFFER_USAGE_TRANSFER_SRC_BIT;VkBuffer buffer{};CHECK(vkCreateBuffer(device,&bi,nullptr,&buffer)==VK_SUCCESS);vkGetBufferMemoryRequirements(device,buffer,&r);auto bm=allocate(r,VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT|VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);CHECK(vkBindBufferMemory(device,buffer,bm,0)==VK_SUCCESS);void* map{};CHECK(vkMapMemory(device,bm,0,VK_WHOLE_SIZE,0,&map)==VK_SUCCESS);std::memset(map,127,256);vkUnmapMemory(device,bm);
VkCommandPoolCreateInfo pci{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO};pci.queueFamilyIndex=family;VkCommandPool pool{};CHECK(vkCreateCommandPool(device,&pci,nullptr,&pool)==VK_SUCCESS);VkCommandBufferAllocateInfo cai{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO};cai.commandPool=pool;cai.level=VK_COMMAND_BUFFER_LEVEL_PRIMARY;cai.commandBufferCount=1;VkCommandBuffer cmd{};CHECK(vkAllocateCommandBuffers(device,&cai,&cmd)==VK_SUCCESS);VkCommandBufferBeginInfo begin{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO};CHECK(vkBeginCommandBuffer(cmd,&begin)==VK_SUCCESS);
VkImageMemoryBarrier2 b{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2};b.srcStageMask=b.dstStageMask=VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT;b.dstAccessMask=VK_ACCESS_2_MEMORY_READ_BIT|VK_ACCESS_2_MEMORY_WRITE_BIT;b.oldLayout=VK_IMAGE_LAYOUT_UNDEFINED;b.newLayout=VK_IMAGE_LAYOUT_GENERAL;b.srcQueueFamilyIndex=b.dstQueueFamilyIndex=VK_QUEUE_FAMILY_IGNORED;b.image=image;b.subresourceRange={VK_IMAGE_ASPECT_COLOR_BIT,0,VK_REMAINING_MIP_LEVELS,0,VK_REMAINING_ARRAY_LAYERS};VkDependencyInfo di{VK_STRUCTURE_TYPE_DEPENDENCY_INFO};di.imageMemoryBarrierCount=1;di.pImageMemoryBarriers=&b;vkCmdPipelineBarrier2(cmd,&di);
VkClearColorValue zero{};vkCmdClearColorImage(cmd,image,VK_IMAGE_LAYOUT_GENERAL,&zero,1,&b.subresourceRange);
if(fixed){b.srcStageMask=VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT;b.srcAccessMask=VK_ACCESS_2_TRANSFER_WRITE_BIT;b.dstStageMask=VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT;b.dstAccessMask=VK_ACCESS_2_MEMORY_READ_BIT|VK_ACCESS_2_MEMORY_WRITE_BIT;b.oldLayout=b.newLayout=VK_IMAGE_LAYOUT_GENERAL;vkCmdPipelineBarrier2(cmd,&di);}
VkBufferImageCopy copy{};copy.imageSubresource={VK_IMAGE_ASPECT_COLOR_BIT,0,0,1};copy.imageExtent={16,16,1};vkCmdCopyBufferToImage(cmd,buffer,image,VK_IMAGE_LAYOUT_GENERAL,1,&copy);
CHECK(vkEndCommandBuffer(cmd)==VK_SUCCESS);VkSubmitInfo submit{VK_STRUCTURE_TYPE_SUBMIT_INFO};submit.commandBufferCount=1;submit.pCommandBuffers=&cmd;CHECK(vkQueueSubmit(queue,1,&submit,VK_NULL_HANDLE)==VK_SUCCESS);CHECK(vkQueueWaitIdle(queue)==VK_SUCCESS);
vkDestroyCommandPool(device,pool,nullptr);vkDestroyBuffer(device,buffer,nullptr);vkFreeMemory(device,bm,nullptr);vkDestroyImage(device,image,nullptr);vkFreeMemory(device,im,nullptr);vkDestroyDevice(device,nullptr);destroyDebug(instance,messenger,nullptr);vkDestroyInstance(instance,nullptr);FreeLibrary(loader);
std::printf("validation_errors=%d WAW_hazards=%d\n",errors,hazards);CHECK(fixed ? errors==0 : hazards>0);puts("PASS: API dependency probe expectation met");
}

44
nx2/ui_fonts.py Normal file
View File

@@ -0,0 +1,44 @@
"""The overlays' bundled fonts (gpu/vk/vk_imgui.cpp): third_party/fonts/libertinus's TTF files as byte arrays in a C++ file of the
build's own, compiled into the Vulkan back end. They stand in when the player's game files do not have the game's own menu
font (which is read from them at run time and never copied anywhere).
python nx2/ui_fonts.py [out.cpp]
"""
import os
import sys
ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
FONTS = os.path.join(ROOT, "third_party", "fonts", "libertinus")
# (the symbol, the file: vk_imgui.cpp declares the same names)
FILES = (("hookshot_font_body", "LibertinusSans-Regular.ttf"), ("hookshot_font_strong", "LibertinusSans-Bold.ttf"),
("hookshot_font_heading", "LibertinusSerif-Semibold.ttf"))
def source():
lines = ["// generated by nx2/ui_fonts.py from third_party/fonts/libertinus (SIL Open Font License 1.1): do not edit"]
for name, file in FILES:
data = open(os.path.join(FONTS, file), "rb").read()
lines.append("extern const unsigned %s_size = %d;" % (name, len(data)))
lines.append("extern const unsigned char %s[%d] = {" % (name, len(data)))
for i in range(0, len(data), 48):
lines.append(",".join(str(b) for b in data[i:i + 48]) + ",")
lines.append("};")
return "\n".join(lines) + "\n"
def write(out_path):
"""Writes the file when it differs (an unchanged one keeps its timestamp); returns the fonts' bytes, for the object's key"""
text = source()
os.makedirs(os.path.dirname(out_path), exist_ok=True)
try:
same = open(out_path, encoding="utf-8").read() == text
except OSError:
same = False
if not same:
with open(out_path, "w", encoding="utf-8", newline="\n") as f:
f.write(text)
return b"".join(open(os.path.join(FONTS, file), "rb").read() for _, file in FILES)
if __name__ == "__main__":
write(sys.argv[1] if len(sys.argv) > 1 else os.path.join(ROOT, "ui_fonts.cpp"))

69
nx2/unpack.py Normal file
View File

@@ -0,0 +1,69 @@
"""Rewrite an exefs's ZBIC-compressed NSOs (flags bit 7, Switch firmware 22.0.0+) as plain uncompressed NSOs, in place,
so a64-kit's nso.py (LZ4 or none) and everything after it reads them unchanged.
python -I nx2/unpack.py <exefs dir>
The segment hashes at 0xa0 are of the decompressed data, so they stay valid; they are checked here.
NSOs with LZ4 or no compression are left as they are.
"""
import hashlib
import os
import struct
import sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # (python -I leaves the script folder off)
import zbic # noqa: E402
ZBIC_FLAG = 1 << 7
NAMES = ["rtld", "main"] + ["subsdk%d" % i for i in range(10)] + ["sdk"]
def unpack_nso(path):
raw = open(path, "rb").read()
if raw[:4] != b"NSO0":
return None
flags = struct.unpack_from("<I", raw, 0xC)[0]
if not flags & ZBIC_FLAG:
return False
segs = [list(struct.unpack_from("<4I", raw, 0x10 + 0x10 * i)) for i in range(3)]
csize = struct.unpack_from("<3I", raw, 0x60)
datas = []
for i, ((foff, moff, size, _), cs) in enumerate(zip(segs, csize)):
d = raw[foff:foff + cs]
if flags & (1 << i):
d = zbic.decompress(d, size)
if len(d) != size:
raise SystemExit("%s segment %d: %d bytes, %d expected" % (path, i, len(d), size))
if flags & (8 << i):
want = raw[0xA0 + 0x20 * i:0xC0 + 0x20 * i]
if hashlib.sha256(d).digest() != want:
raise SystemExit("%s segment %d: hash mismatch after ZBIC" % (path, i))
datas.append(d)
hdr = bytearray(raw[:0x100])
body = bytearray(raw[0x100:min(s[0] for s in segs)]) # (the module name, which the header points at)
for i, d in enumerate(datas):
struct.pack_into("<I", hdr, 0x10 + 0x10 * i, 0x100 + len(body))
struct.pack_into("<I", hdr, 0x60 + 4 * i, len(d))
body += d
struct.pack_into("<I", hdr, 0xC, flags & ~(ZBIC_FLAG | 7)) # (uncompressed; the hash bits stay)
tmp = path + ".plain"
with open(tmp, "wb") as g:
g.write(hdr)
g.write(body)
os.replace(tmp, path)
return True
def main():
exefs = sys.argv[1] if len(sys.argv) > 1 else None
if not exefs:
raise SystemExit(__doc__)
for n in NAMES:
p = os.path.join(exefs, n)
if os.path.exists(p):
r = unpack_nso(p)
print("%-8s %s" % (n, {None: "not an NSO", False: "not ZBIC, kept", True: "ZBIC -> plain"}[r]))
if __name__ == "__main__":
main()

72
nx2/version.py Normal file
View File

@@ -0,0 +1,72 @@
"""Which Hookshot a build is: the release it came from and a fingerprint of its sources.
The game prints it as the first line of triforce_log.txt and in the Insert menu, so a report says what was run.
python -I nx2/version.py prints it for this folder
The release name comes from, in order: HOOKSHOT_VERSION; a VERSION file at the root (written when a release is packaged,
since the builder's tools folder has no git history); `git describe --tags --always --dirty` in a checkout, matching
only version tags (`v0.1.0`, not the branch-like tags from before the scheme). The
fingerprint is a hash of the sources in this folder (host/, gpu/, nx2/), line endings aside: every build from the
same release zip gives the same one, so even a build that says "unknown" can be traced to the zip it came from. (A
checkout and a release's tools folder hold different sets of files, so their fingerprints differ.)
"""
import hashlib
import os
import re
import subprocess
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
SOURCES = ("host", "gpu", "nx2")
SOURCE_EXT = (".c", ".cc", ".cpp", ".h", ".hpp", ".inc", ".py", ".glsl", ".comp", ".vert", ".frag", ".spv")
def release(root=ROOT):
name = os.environ.get("HOOKSHOT_VERSION", "").strip()
if not name:
for file in ("VERSION", "version.txt"): # (version.txt: the release's tag, written into the builder's tools when it is packed)
try:
with open(os.path.join(root, file), encoding="utf-8") as f:
name = f.readline().strip()
except OSError:
pass
if name:
break
if not name and os.path.exists(os.path.join(root, ".git")):
try:
r = subprocess.run(["git", "-C", root, "describe", "--tags", "--always", "--dirty", "--match", "v[0-9]*"],
capture_output=True, text=True, timeout=20)
name = r.stdout.strip() if r.returncode == 0 else ""
except (OSError, subprocess.SubprocessError):
pass
return name or "unknown"
def fingerprint(root=ROOT):
h = hashlib.sha1(usedforsecurity=False)
for top in SOURCES:
base = os.path.join(root, top)
for d, dirs, files in os.walk(base):
dirs[:] = sorted(x for x in dirs if x != "__pycache__")
for f in sorted(files):
if not f.endswith(SOURCE_EXT):
continue
p = os.path.join(d, f)
with open(p, "rb") as src:
data = src.read().replace(b"\r\n", b"\n")
h.update(os.path.relpath(p, root).replace("\\", "/").encode() + b"\0" + data + b"\0")
return h.hexdigest()[:8]
def describe(root=ROOT):
"""'movie-sync (source 1a2b3c4d)': safe to pass as a C string literal on any command line."""
text = "%s (source %s)" % (release(root), fingerprint(root))
return re.sub(r"[^A-Za-z0-9 ._+()/-]", "_", text)[:80]
def define(root=ROOT):
return '-DHOOKSHOT_VERSION="%s"' % describe(root)
if __name__ == "__main__":
print(describe())

67
nx2/vkcheck.py Normal file
View File

@@ -0,0 +1,67 @@
"""Syntax check of the Vulkan backend's sources (gpu/vk): Eden's shader recompiler, sirit and the backend.
Usage: python -I nx2/vkcheck.py [substring ...] (only the sources whose path contains one of them)
"""
import concurrent.futures
import os
import subprocess
import sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import kit
import toolchain
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
VK = os.path.join(ROOT, "gpu", "vk")
CPM = os.path.join(kit.WORK, "eg", ".cache", "cpm")
def pkg(name):
d = os.path.join(CPM, name)
return os.path.join(d, sorted(os.listdir(d))[-1])
def sources():
out = []
for dp, _, fn in os.walk(os.path.join(VK, "eden")):
out += [os.path.join(dp, f) for f in fn if f.endswith(".cpp")]
for dp, _, fn in os.walk(os.path.join(VK, "sirit", "src")):
out += [os.path.join(dp, f) for f in fn if f.endswith(".cpp")]
out += [os.path.join(VK, f) for f in os.listdir(VK) if f.endswith(".cpp")]
return sorted(out)
def flags():
return toolchain.native_target() + [
"-std=c++20", "-O2", "-g0", "-DNDEBUG", "-Wno-everything", "-DVK_NO_PROTOTYPES",
"-I" + os.path.join(VK, "compat"), "-I" + os.path.join(ROOT, "gpu", "compat"),
"-I" + os.path.join(VK, "eden"), "-I" + os.path.join(VK, "sirit", "include"),
"-I" + os.path.join(VK, "sirit", "src"),
"-I" + os.path.join(pkg("spirv-headers"), "include"),
"-I" + os.path.join(pkg("spirv-headers"), "include", "spirv", "unified1"),
"-I" + os.path.join(pkg("vulkanheaders"), "include"),
"-I" + os.path.join(pkg("vulkanmemoryallocator"), "include"),
"-I" + os.path.join(ROOT, "gpu", "render")]
def main():
cxx = toolchain.need(toolchain.find("clang++"))
srcs = [s for s in sources() if len(sys.argv) < 2 or any(a in s for a in sys.argv[1:])]
fl = flags()
def one(s):
r = subprocess.run([cxx] + fl + ["-fsyntax-only", s], capture_output=True, text=True)
return s, r.returncode, r.stderr
bad = 0
with concurrent.futures.ThreadPoolExecutor(4) as ex:
for s, rc, err in ex.map(one, srcs):
if rc:
bad += 1
print("== %s" % os.path.relpath(s, ROOT))
print("\n".join(l for l in err.splitlines() if "error" in l)[:1500])
print("%d of %d sources failed" % (bad, len(srcs)))
if __name__ == "__main__":
main()

19
nx2/whole_write.py Normal file
View File

@@ -0,0 +1,19 @@
"""(issue 1197) A lifted C file rewritten in place (the patches, the chunk split) is written beside it and moved over it: a run
stopped mid-write (the builder closed, a second build stopping the first, a full disk) leaves the old file whole, never the first
megabytes of the new one, which the compile then stopped on at every try ("g_module_b" where g_module_base was)."""
import os
import time
def write_whole(path, text, newline="\n"):
tmp = path + ".part"
with open(tmp, "w", encoding="utf-8", newline=newline) as f:
f.write(text)
for attempt in range(20): # (an antivirus reading the file holds it a moment: Windows refuses the move until it lets go)
try:
os.replace(tmp, path)
return
except PermissionError:
if attempt == 19:
raise
time.sleep(0.25)

60
nx2/win_manifest.py Normal file
View File

@@ -0,0 +1,60 @@
"""The game's Windows manifest, as a .res the exe links in (nx2/build.py): UTF-8 as the process's code page.
host/ and gpu/vk keep paths as UTF-8 char strings and call Windows' A functions (GetModuleFileNameA, CreateFileA, fopen ...)
with them. Without a manifest those take and give the system's ANSI code page (1252 on a Western Windows), and a game folder
whose name it cannot spell (Japanese on a Western Windows, an emoji anywhere) came back as "?": the game found neither
image.bin nor its log's folder and closed with nothing on screen. activeCodePage UTF-8 (Windows 10 1903 and newer) makes
every A function and the C runtime's narrow file functions UTF-8 for the whole process, its DLLs too (triforce_vk.dll,
SDL2, FFmpeg). host/hos_main.c says so in a message box when the Windows is too old for it and the folder's name needs it.
python nx2/win_manifest.py [out.res]
"""
import os
import struct
import sys
MANIFEST = b"""<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<assembly xmlns="urn:schemas-microsoft-com:asm.v1" manifestVersion="1.0">
<assemblyIdentity type="win32" name="Hookshot.Game" version="1.0.0.0"/>
<application xmlns="urn:schemas-microsoft-com:asm.v3">
<windowsSettings>
<activeCodePage xmlns="http://schemas.microsoft.com/SMI/2019/WindowsSettings">UTF-8</activeCodePage>
</windowsSettings>
</application>
<trustInfo xmlns="urn:schemas-microsoft-com:asm.v3">
<security><requestedPrivileges><requestedExecutionLevel level="asInvoker" uiAccess="false"/></requestedPrivileges></security>
</trustInfo>
</assembly>
"""
RT_MANIFEST = 24
CREATEPROCESS_MANIFEST_RESOURCE_ID = 1 # (the one an exe's loader reads)
def res_entry(kind, name, data, flags=0x1030, language=0x0409):
"""one resource of a .res file (the format nx2/make_icon.py writes too): sizes, numeric type and name, flags, language"""
header = struct.pack("<II", len(data), 32) + struct.pack("<HHHH", 0xFFFF, kind, 0xFFFF, name)
header += struct.pack("<IHHII", 0, flags, language, 0, 0)
return header + data + b"\0" * (-len(data) & 3)
def res_bytes(manifest=MANIFEST):
return res_entry(0, 0, b"", 0, 0) + res_entry(RT_MANIFEST, CREATEPROCESS_MANIFEST_RESOURCE_ID, manifest)
def write(path):
"""hookshot_manifest.res at path, rewritten only when its bytes change (the link reads it each time)"""
data = res_bytes()
try:
with open(path, "rb") as f:
if f.read() == data:
return path
except OSError:
pass
with open(path, "wb") as f:
f.write(data)
return path
if __name__ == "__main__":
print(write(sys.argv[1] if len(sys.argv) > 1 else os.path.join(os.getcwd(), "hookshot_manifest.res")))

36
nx2/xref.py Normal file
View File

@@ -0,0 +1,36 @@
"""Cross references in the guest image (Eden layout): direct B/BL to an address, and 8-byte words equal to it
(vtables, function pointer tables, GOT entries).
python -I nx2/xref.py 0x800bae10 [0x800bae20 ...]
"""
import sys, os
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import kit # noqa: F401
import image as eden_image
import nx_image
def main():
targets = {int(x, 0) for x in sys.argv[1:]}
img = eden_image.EdenImage(kit.need_exefs())
mem, base = img.mem, nx_image.MAIN_BASE
for lo, hi, name in img.code:
for pc in range(lo, hi, 4):
w = int.from_bytes(mem[pc - base:pc - base + 4], "little")
if (w & 0x7C000000) == 0x14000000: # B / BL
off = (w & 0x3FFFFFF) << 2
if off & (1 << 27):
off -= 1 << 28
if pc + off in targets:
print(f"{'bl' if w >> 31 else 'b '} at {pc:#x} -> {pc + off:#x}")
for t in targets:
needle = t.to_bytes(8, "little")
i = mem.find(needle)
while i >= 0:
if i % 8 == 0:
print(f"word {t:#x} at {base + i:#x}")
i = mem.find(needle, i + 1)
if __name__ == "__main__":
main()

58
nx2/zbic.py Normal file
View File

@@ -0,0 +1,58 @@
"""ZBIC decompression (Switch firmware 22.0.0+'s zstd variant) through third_party/zbic, built to zbic.dll on first use.
import zbic; plain = zbic.decompress(data)
"""
import ctypes
import os
import subprocess
import sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # (python -I leaves the script folder off)
import kit # noqa: F401 (a64-kit on sys.path)
import toolchain
HERE = os.path.dirname(os.path.abspath(__file__))
SRC = os.path.join(os.path.dirname(HERE), "third_party", "zbic")
DLL = os.path.join(SRC, "zbic.dll" if os.name == "nt" else "zbic.so")
_lib = None
def _build():
# (the one that came with the tools is used as it is. This compared file dates, which unpacking a zip can change: the source then
# looked newer, and reading a .nup needed Microsoft's C library, which a player need not have at that point)
if os.path.exists(DLL):
return
srcs = [os.path.join(SRC, "zstd.c"), os.path.join(SRC, "zbic_api.c")]
cmd = [toolchain.need(toolchain.find("clang", host=True)), *toolchain.native_target(host=True), *toolchain.native_link(host=True),
"-O2", "-shared", *([] if os.name == "nt" else ["-fPIC"]), "-DZSTD_ZBIC_SUPPORT=1", "-I", SRC, "-o", DLL, *srcs]
print("building", os.path.basename(DLL), file=sys.stderr)
subprocess.run(cmd, check=True)
def lib():
global _lib
if _lib is None:
_build()
_lib = ctypes.CDLL(DLL)
_lib.zbic_content_size.restype = ctypes.c_longlong
_lib.zbic_content_size.argtypes = [ctypes.c_char_p, ctypes.c_size_t]
_lib.zbic_decompress.restype = ctypes.c_longlong
_lib.zbic_decompress.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_char_p, ctypes.c_size_t]
return _lib
def is_zbic(data):
return data[:4] == b"ZBIC"
def decompress(data, size=None):
L = lib()
if size is None:
size = L.zbic_content_size(data, len(data))
if size < 0:
raise ValueError("ZBIC: no content size in the frame header")
out = ctypes.create_string_buffer(size)
n = L.zbic_decompress(out, size, data, len(data))
if n != size:
raise ValueError("ZBIC: decompression failed (%d of %d bytes)" % (n, size))
return out.raw