Triforce v0.2.0

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
ruckus7382sfgsgdf
2026-11-10 19:32:48 -06:00
commit 34b3683441
1730 changed files with 554853 additions and 0 deletions

177
third_party/a64-kit/tools/nso.py vendored Normal file
View File

@@ -0,0 +1,177 @@
"""NSO (Switch executable) loader: segments, MOD0, dynamic section, symbols, relocations, unwind table.
NSO (switchbrew.org "NSO"):
- header 0x100: three segments (.text, .rodata, .data), each {file offset, memory offset, size}, LZ4-block
compressed when flags bit 0/1/2 is set; bss size at +0x3C; build ID at +0x40.
- .text + 4 holds the offset of the MOD0 header. MOD0 points (relative to itself) at the ELF dynamic section,
bss, .eh_frame_hdr and the module object.
- The dynamic section is ordinary AArch64 ELF: DT_SYMTAB/DT_STRTAB, DT_RELA (R_AARCH64_RELATIVE, GLOB_DAT,
ABS64), DT_JMPREL (JUMP_SLOT), DT_INIT_ARRAY. rtld binds the modules of one ExeFS to each other by symbol name,
so the sdk's exports are named (nn::*, nvn*) even in a stripped game.
- .eh_frame_hdr has a sorted table of (function start, FDE): one entry per function with unwind info.
Usage: python nso.py <exefs_dir | file.nso> [out_dir]
Prints a summary per module; with out_dir writes <module>.bin (the loaded image, bss zeroed) and <module>.json
(segments, symbols, relocations, eh_frame function starts) for the lifter.
"""
import json
import os
import struct
import sys
import lz4.block
DT = {1: "NEEDED", 2: "PLTRELSZ", 3: "PLTGOT", 4: "HASH", 5: "STRTAB", 6: "SYMTAB", 7: "RELA", 8: "RELASZ",
9: "RELAENT", 10: "STRSZ", 11: "SYMENT", 12: "INIT", 13: "FINI", 20: "PLTREL", 23: "JMPREL",
25: "INIT_ARRAY", 26: "FINI_ARRAY", 27: "INIT_ARRAYSZ", 28: "FINI_ARRAYSZ", 30: "FLAGS",
0x6FFFFEF5: "GNU_HASH", 0x6FFFFFF9: "RELACOUNT"}
R_AARCH64 = {257: "ABS64", 1025: "GLOB_DAT", 1026: "JUMP_SLOT", 1027: "RELATIVE", 1030: "TLSDESC"}
STT = {0: "notype", 1: "object", 2: "func", 3: "section", 4: "file", 6: "tls"}
STB = {0: "local", 1: "global", 2: "weak"}
MODULE_ORDER = ["rtld", "main"] + [f"subsdk{i}" for i in range(10)] + ["sdk"]
class Nso:
def __init__(self, path):
raw = open(path, "rb").read()
if raw[:4] != b"NSO0":
raise ValueError(f"{path}: not an NSO")
self.name = os.path.basename(path)
flags = struct.unpack_from("<I", raw, 0xC)[0]
segs = [struct.unpack_from("<III", raw, 0x10 + 0x10 * i) for i in range(3)]
csize = struct.unpack_from("<III", raw, 0x60)
self.bss_size = struct.unpack_from("<I", raw, 0x3C)[0]
self.build_id = raw[0x40:0x60].rstrip(b"\0").hex()
self.segments = []
end = 0
for i, ((foff, moff, size), cs) in enumerate(zip(segs, csize)):
data = raw[foff:foff + cs]
if flags & (1 << i):
data = lz4.block.decompress(data, uncompressed_size=size)
assert len(data) == size, f"{self.name} segment {i}: {len(data)} != {size}"
self.segments.append((moff, size, (".text", ".rodata", ".data")[i]))
end = max(end, moff + size)
self.image = bytearray(end)
for (moff, size, _), (foff, _, _), cs, i in zip(self.segments, segs, csize, range(3)):
data = raw[foff:foff + cs]
if flags & (1 << i):
data = lz4.block.decompress(data, uncompressed_size=size)
self.image[moff:moff + size] = data
self._mod0()
def u32(self, off):
return struct.unpack_from("<I", self.image, off)[0]
def u64(self, off):
return struct.unpack_from("<Q", self.image, off)[0]
def _mod0(self):
m = self.u32(4)
if self.image[m:m + 4] != b"MOD0":
raise ValueError(f"{self.name}: no MOD0")
dyn, bss_s, bss_e, eh_s, eh_e, modobj = struct.unpack_from("<6i", self.image, m + 4)
self.mod0 = m
self.bss = (m + bss_s, m + bss_e)
self.eh_frame_hdr = (m + eh_s, m + eh_e)
self.module_object = m + modobj
if len(self.image) < self.bss[1]:
self.image.extend(bytes(self.bss[1] - len(self.image)))
self.dynamic = {}
off = m + dyn
while True:
tag, val = struct.unpack_from("<qQ", self.image, off)
off += 16
if tag == 0:
break
self.dynamic.setdefault(DT.get(tag, tag), []).append(val)
d = lambda k: self.dynamic.get(k, [0])[0]
strtab, symtab, syment = d("STRTAB"), d("SYMTAB"), d("SYMENT") or 24
nsyms = (strtab - symtab) // syment if strtab > symtab else 0
self.symbols = []
for i in range(nsyms):
name, info, other, shndx, value, size = struct.unpack_from("<IBBHQQ", self.image, symtab + i * syment)
self.symbols.append({"name": _cstr(self.image, strtab + name), "type": STT.get(info & 0xF, info & 0xF),
"bind": STB.get(info >> 4, info >> 4), "defined": shndx != 0,
"value": value, "size": size})
self.relocs = []
for start_key, size_key in (("RELA", "RELASZ"), ("JMPREL", "PLTRELSZ")):
start, size = d(start_key), d(size_key)
for off in range(start, start + size, 24):
r_off, r_info, addend = struct.unpack_from("<QQq", self.image, off)
sym = r_info >> 32
self.relocs.append({"offset": r_off, "type": R_AARCH64.get(r_info & 0xFFFFFFFF, r_info & 0xFFFFFFFF),
"sym": self.symbols[sym]["name"] if sym else None, "addend": addend})
self.init_array = [self.u64(d("INIT_ARRAY") + 8 * i) for i in range(d("INIT_ARRAYSZ") // 8)]
self.eh_funcs = self._eh_frame_hdr_table()
def _eh_frame_hdr_table(self):
"""Function starts from .eh_frame_hdr's binary-search table (datarel sdata4 entries)."""
s, e = self.eh_frame_hdr
if e <= s:
return []
ver, ptr_enc, cnt_enc, tbl_enc = self.image[s:s + 4]
if ver != 1 or tbl_enc != 0x3B or cnt_enc != 0x03:
return []
p = s + 4 + 4 # eh_frame_ptr (sdata4, pcrel) assumed 4 bytes
if ptr_enc & 0x0F not in (0x0B, 0x03):
return []
count = self.u32(p)
p += 4
return [s + struct.unpack_from("<i", self.image, p + 8 * i)[0] for i in range(count)]
def code_range(self):
moff, size, _ = self.segments[0]
return moff, moff + size
def _cstr(buf, off):
end = buf.find(b"\0", off)
return bytes(buf[off:end]).decode("utf-8", "replace")
def load_exefs(path):
if os.path.isfile(path):
return [Nso(path)]
return [Nso(os.path.join(path, n)) for n in MODULE_ORDER if os.path.exists(os.path.join(path, n))]
def main():
mods = load_exefs(sys.argv[1])
exports = {}
for m in mods:
for s in m.symbols:
if s["defined"] and s["name"]:
exports.setdefault(s["name"], m.name)
for m in mods:
print(f"\n== {m.name} build {m.build_id[:16]}... image {len(m.image):,} bytes")
for moff, size, n in m.segments:
print(f" {n:<8} {moff:#010x} {size:>12,}")
print(f" .bss {m.bss[0]:#010x} {m.bss[1] - m.bss[0]:>12,}")
defined = [s for s in m.symbols if s["defined"] and s["name"]]
undef = [s for s in m.symbols if not s["defined"] and s["name"]]
resolved = {}
for s in undef:
resolved[exports.get(s["name"], "?")] = resolved.get(exports.get(s["name"], "?"), 0) + 1
print(f" symbols: {len(defined):,} exported, {len(undef):,} imported "
f"({', '.join(f'{k}: {v:,}' for k, v in sorted(resolved.items()))})")
rc = {}
for r in m.relocs:
rc[r["type"]] = rc.get(r["type"], 0) + 1
print(f" relocations: {', '.join(f'{k} {v:,}' for k, v in sorted(rc.items(), key=str))}")
print(f" init_array: {len(m.init_array)} eh_frame_hdr function starts: {len(m.eh_funcs):,}")
if len(sys.argv) > 2:
out = sys.argv[2]
os.makedirs(out, exist_ok=True)
for m in mods:
open(os.path.join(out, m.name + ".bin"), "wb").write(m.image)
json.dump({"name": m.name, "build_id": m.build_id,
"segments": [{"name": n, "offset": o, "size": s} for o, s, n in m.segments],
"bss": m.bss, "mod0": m.mod0, "init_array": m.init_array, "eh_funcs": m.eh_funcs,
"symbols": m.symbols, "relocs": m.relocs},
open(os.path.join(out, m.name + ".json"), "w"), indent=0)
print(f"\nwrote images + json to {out}")
if __name__ == "__main__":
main()

128
third_party/a64-kit/tools/survey.py vendored Normal file
View File

@@ -0,0 +1,128 @@
"""Instruction survey of NSO code: what an AArch64 emitter has to handle, by count.
Disassembles every .text word with Capstone (skipdata on, so data in code shows up as .byte/.word) and reports:
- mnemonics by count, split into integer, scalar FP, vector (NEON), atomics/exclusives, system, branches;
- system registers read and written (mrs/msr), svc numbers, hint/barrier forms;
- indirect branches (br/blr) and words Capstone could not decode.
Usage: python survey.py <image_dir> [module ...] [--top N] [--json out.json]
image_dir is what nso.py wrote (<module>.bin + <module>.json). Default modules: main.
"""
import collections
import json
import os
import re
import sys
from concurrent.futures import ProcessPoolExecutor
import capstone
VEC = re.compile(r"\bv\d+\.|\bv\d+\b")
FPREG = re.compile(r"\b[sdhbq]\d+\b")
EXCL = {"ldxr", "ldxrb", "ldxrh", "ldxp", "ldaxr", "ldaxrb", "ldaxrh", "ldaxp", "stxr", "stxrb", "stxrh", "stxp",
"stlxr", "stlxrb", "stlxrh", "stlxp", "ldar", "ldarb", "ldarh", "stlr", "stlrb", "stlrh", "clrex"}
SYSTEM = {"mrs", "msr", "svc", "hint", "nop", "yield", "wfe", "wfi", "sev", "sevl", "dmb", "dsb", "isb", "dc", "ic",
"sys", "brk", "hlt", "udf", "prfm", "prfum"}
BRANCH = {"b", "bl", "br", "blr", "ret", "cbz", "cbnz", "tbz", "tbnz"}
def classify(mn, ops):
base = mn.split(".")[0]
if base in EXCL or base.startswith(("cas", "ldadd", "swp", "ldclr", "ldset", "ldeor", "ldsmax", "ldumax")):
return "atomic"
if base in SYSTEM:
return "system"
if base in BRANCH or base.startswith("b."):
return "branch"
if base in (".byte", ".word", ".hword", ".quad"):
return "data"
if VEC.search(ops):
return "vector"
if base.startswith("f") or base in ("scvtf", "ucvtf") or FPREG.search(ops):
return "fp"
return "int"
def scan(args):
path, start, end = args
with open(path, "rb") as f:
f.seek(start)
code = f.read(end - start)
md = capstone.Cs(capstone.CS_ARCH_ARM64, capstone.CS_MODE_ARM)
md.skipdata = True
mnem = collections.Counter()
sysreg = collections.Counter()
svc = collections.Counter()
cls = collections.Counter()
forms = collections.Counter()
for addr, size, mn, ops in md.disasm_lite(code, start):
c = classify(mn, ops)
cls[c] += 1
if c == "vector":
arr = sorted(set(re.findall(r"\.(\d*[bhsdq])\b", ops)))
mnem[f"{mn} ({','.join(arr) or 'v'})"] += 1
forms[mn] += 1
else:
mnem[mn] += 1
if mn in ("mrs", "msr"):
sysreg[f"{mn} {ops.split(',')[1 if mn == 'mrs' else 0].strip()}"] += 1
elif mn == "svc":
svc[ops] += 1
elif mn in ("dc", "ic", "sys"):
sysreg[f"{mn} {ops.split(',')[0]}"] += 1
return mnem, sysreg, svc, cls, forms
def survey(image_dir, module, workers=None):
meta = json.load(open(os.path.join(image_dir, module + ".json")))
text = next(s for s in meta["segments"] if s["name"] == ".text")
path = os.path.join(image_dir, module + ".bin")
n = (workers or os.cpu_count() or 4) * 4
step = ((text["size"] // n) + 3) & ~3
jobs = [(path, text["offset"] + i, min(text["offset"] + i + step, text["offset"] + text["size"]))
for i in range(0, text["size"], step)]
tot = [collections.Counter() for _ in range(5)]
with ProcessPoolExecutor(workers) as ex:
for part in ex.map(scan, jobs):
for t, p in zip(tot, part):
t.update(p)
return tot
def main():
args = sys.argv[1:]
top = 60
out_json = None
if "--top" in args:
i = args.index("--top")
top = int(args[i + 1])
del args[i:i + 2]
if "--json" in args:
i = args.index("--json")
out_json = args[i + 1]
del args[i:i + 2]
image_dir, modules = args[0], args[1:] or ["main"]
report = {}
for mod in modules:
mnem, sysreg, svc, cls, forms = survey(image_dir, mod)
total = sum(cls.values())
print(f"\n== {mod}: {total:,} instructions, {len(mnem):,} distinct mnemonic forms")
for k, v in cls.most_common():
print(f" {k:<8} {v:>12,} {100 * v / total:5.1f}%")
print(f" vector mnemonics (ignoring arrangement): {len(forms)}")
print(f"\n top {top} mnemonics:")
for k, v in mnem.most_common(top):
print(f" {v:>10,} {k}")
print("\n system registers / cache ops:")
for k, v in sysreg.most_common():
print(f" {v:>8,} {k}")
print(f" svc: {dict(svc) or 'none'}")
for k in ("br", "blr", "ret"):
print(f" {k}: {mnem.get(k, 0):,}")
report[mod] = {"classes": cls, "mnemonics": mnem, "sysregs": sysreg, "svc": svc, "vector_forms": forms}
if out_json:
json.dump(report, open(out_json, "w"), indent=1)
if __name__ == "__main__":
main()

105
third_party/a64-kit/tools/toolchain.py vendored Normal file
View File

@@ -0,0 +1,105 @@
import os, shutil
HERE = os.path.dirname(os.path.abspath(__file__))
def emsdk():
return os.environ.get("EMSDK") or (r"C:\emsdk" if os.name == "nt" else os.path.expanduser("~/emsdk"))
def windows():
"""(hookshot) the game is built for Windows: on Windows, or with RX_TARGET=windows on another system (a cross build, with
llvm-mingw's release for that system in HOOKSHOT_MINGW)"""
return os.name == "nt" or os.environ.get("RX_TARGET", "").lower() == "windows"
def mingw(host=False):
"""(hookshot) HOOKSHOT_MINGW=<llvm-mingw folder>: Windows builds with that toolchain's clang, headers and libraries (the GNU ABI
on the UCRT), as TotK Recomp's installer does: nothing of Visual Studio or the Windows SDK is needed. Empty: the MSVC target.
host: for a tool the build itself runs (zbic, a64emit), which a cross build makes for this PC, not for Windows"""
return os.environ.get("HOOKSHOT_MINGW", "") if windows() and not (host and os.name != "nt") else ""
def find(tool, host=False):
if mingw(host) and (tool in ("clang", "clang++", "lld") or tool.startswith("llvm-")):
path = os.path.join(mingw(host), "bin", ("ld.lld" if tool == "lld" else tool) + (".exe" if os.name == "nt" else ""))
if os.path.isfile(path):
return path
root = emsdk()
exts = (".exe", ".bat") if os.name == "nt" else ("",)
native = tool in ("clang", "clang++", "lld") or tool.startswith("llvm-")
folder = os.path.join(root, "upstream", "bin") if native else os.path.join(root, "upstream", "emscripten")
for ext in exts:
path = os.path.join(folder, tool + ext)
if os.path.isfile(path):
return path
return shutil.which(tool) or os.path.join(folder, tool + exts[0])
def need(path):
if not os.path.isfile(path):
raise SystemExit("%s not found: set EMSDK to an emsdk checkout (now %s), or put %s on PATH"
% (path, emsdk(), os.path.splitext(os.path.basename(path))[0]))
return path
def lld_link():
"""emsdk's lld under the name lld-link (lld picks its COFF linker from its own file name), made on first use"""
path = os.path.join(HERE, "lld-link-mingw.exe" if mingw() else "lld-link.exe")
if not os.path.exists(path):
shutil.copyfile(need(find("lld")), path)
return path
def native_target(host=False):
if mingw(host): # (-fms-extensions: __try/__except and __declspec, which clang takes on this target too)
return ["--target=x86_64-w64-mingw32", "-fms-extensions", "-D_CRT_SECURE_NO_WARNINGS"]
return ["--target=x86_64-pc-windows-msvc", "-fms-extensions", "-D_CRT_SECURE_NO_WARNINGS"] if os.name == "nt" else []
def native_link(host=False):
# (-fuse-ld=lld, found through -B: clang ignores --ld-path for the MSVC target and ran Microsoft's link.exe, which
# cannot read the renderer's ThinLTO bitcode)
if mingw(host): # (its own lld; the C++ runtime and winpthread linked in, so no DLLs of the toolchain go beside the game)
# (/nodefaultlib: libraries built with Microsoft's compiler, the CPM cache's FFmpeg, ask in their objects for its C runtime;
# the MinGW one stands in, with host/mingw_msvc_shims.c for what it lacks)
return ["-fuse-ld=lld", "-static"] + ["-Wl,-Xlink=/nodefaultlib:" + l for l in ("libcmt", "oldnames", "libvcruntime", "msvcrt")]
# (issue 439: clang names oldnames.lib as a default library for the MSVC target, and recent Windows SDKs no longer
# ship it (an older SDK has it in ucrt\x64; Visual Studio has its own copy in its lib folder). The game and host use
# no legacy non-underscore name; the ones the prebuilt static FFmpeg libraries ask for are host/msvc_oldnames.c)
return (["-fuse-ld=lld", "-B" + os.path.dirname(lld_link()), "-Wl,/nodefaultlib:oldnames"]
if os.name == "nt" else [])
def link_opt(opt):
"""A link.exe-style option ("/map:x"): as it is for the MSVC target, through lld's MinGW driver (-Xlink) for llvm-mingw"""
return "-Wl,-Xlink=" + opt if mingw() else "-Wl," + opt
def default_jobs(gb_per_job=3.5):
"""Parallel compiles that fit in memory: a -O3 compile of one of the game's big lifted chunks takes 3-8 GB, and one
job per logical core ran 16 GB machines out of memory (22 threads on an Ultra 7 155H). 80% of the physical memory
at gb_per_job each, at most one a core. RX_JOBS=<n> sets it. (After SkyboundTotk PR 14 by leonelalej.)"""
if os.environ.get("RX_JOBS"):
return max(1, int(os.environ["RX_JOBS"]))
total = 0
try:
if os.name == "nt":
import ctypes
class MemoryStatus(ctypes.Structure):
_fields_ = [("length", ctypes.c_ulong), ("load", ctypes.c_ulong)] + \
[(f, ctypes.c_ulonglong) for f in ("phys", "avail_phys", "page", "avail_page", "virt",
"avail_virt", "avail_ext")]
m = MemoryStatus()
m.length = ctypes.sizeof(m)
if ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(m)):
total = m.phys
else:
total = os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES")
except (OSError, ValueError, AttributeError):
pass
cores = os.cpu_count() or 1
if not total:
return cores
return max(1, min(cores, int(total / 2 ** 30 * 0.8 / gb_per_job)))