Files
Triforce-Source/nx2/lift.py
ruckus7382sfgsgdf 6f338c8f56 Triforce v0.5.0
In-game mods (GameBanana browser, search and installs, with the Mask Salesman), code-patch and cheat mods, the devs' test18 to test45 work (full performance overlay, frame pacing on, steady frame rate, save export and import, bug capture, new options rows, Setup's Mods page and Add to Steam), occlusion queries and new defaults, Auto resolution that scales, replay helper threads and robust buffers off by default, the options in sections.
2026-11-17 22:40:50 -06:00

209 lines
11 KiB
Python

"""The lift driver: image (we are rtld) -> function discovery -> C, on the vendored a64-kit plus the Switch 2
additions (nso2: DT_RELR, insn: ARMv8.1-8.5 forms, lift_fixes: jump tables).
Usage: python -I nx2/lift.py [--exefs <dir>] [--work <dir>] [--write] [--max-func N] [--layout eden|a64] [--aslr 0]
--layout eden (default): Eglantine's addresses, rtld included, from 0x80000000 + --aslr (image.py; an emulator
run must use the same offset: rng_seed_enabled=true, rng_seed=<aslr >> 12>);
--layout a64: a64-kit's own layout from 0x08000000 (its native runtime, below 4 GB)
Always: builds <work>/image (image.bin, image.json), discovers every function of rtld-less main + subsdk* + sdk,
and prints coverage: functions, instructions, jump tables, indirect branches, and the functions whose every
instruction translates (coverage.txt has the untranslated words by site count).
--write: also writes the lifted C to <work>/gen (about a gigabyte; game code: stays local, never committed).
--standalone: the standalone build's lift (<work>/sa/image, <work>/sa/gen): the SDK's OS-facing functions hooked
to the native runtime in host/ or to generated stubs, plus hos_image.c (nx2/standalone.py). It shares the
chunk boundaries of <work>/image, so its unhooked chunks are the emulator build's, byte for byte.
"""
import argparse
import collections
import glob
import json
import os
import struct
import sys
import time
import capstone
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # (python -I leaves the script folder off)
import kit
import nso2 # noqa: F401 (DT_RELR)
import insn
import lift_fixes
import chunks
import standalone
import adr_entries # noqa: E402
import nso
import poll
import nx_image
import image as eden_image
import a64emit
import a64_lift
import a64_rtgen
insn.install()
lift_fixes.install()
def module_of(img, pc):
for lo, hi, name in img.code:
if lo <= pc < hi:
return name
return "?"
def coverage(L, img, out_dir):
"""Per-module: functions, instructions, untranslated words; and the share of functions that translate fully."""
keys = {}
for body, _ in L.funcs.values():
for pc, (k, w) in body.items():
if k[0] == "op":
keys.setdefault(w, pc)
words = list(keys)
_, flags = a64emit.translate([(keys[w], w) for w in words])
bad = {w for w, f in zip(words, flags) if f & a64emit.UNHANDLED}
# UDF and unallocated encodings trap on the hardware too (code falls into them after noreturn calls): the
# lifter's trap is their exact translation, so they do not count against coverage
md = capstone.Cs(capstone.CS_ARCH_ARM64, capstone.CS_MODE_ARM)
undefined = set()
for w in bad:
ins = next(md.disasm(struct.pack("<I", w), 0), None)
if ins is None or ins.mnemonic == "udf":
undefined.add(w)
bad -= undefined
per = collections.defaultdict(collections.Counter)
sites = collections.Counter()
for e, (body, _) in L.funcs.items():
m = module_of(img, e)
n_bad = 0
for pc, (k, w) in body.items():
if k[0] == "op" and w in bad:
n_bad += 1
sites[w] += 1
elif k[0] == "op" and w in undefined:
per[m]["undefined"] += 1
kinds = collections.Counter(k[0] for k, _ in body.values())
c = per[m]
c["functions"] += 1
c["instructions"] += len(body)
c["untranslated"] += n_bad
c["complete"] += n_bad == 0
c["leaf"] += not (kinds["bl"] or kinds["blr"] or kinds["svc"])
c["indirect_br"] += sum(1 for pc, (k, _) in body.items() if k[0] == "br" and pc not in L.jt)
c["blr"] += kinds["blr"]
total = collections.Counter()
lines = ["module functions complete leaf instructions untranslated BLR BR(not table)"]
for m in sorted(per, key=lambda m: -per[m]["instructions"]):
c = per[m]
total.update(c)
lines.append("%-8s %9d %6.2f%% %8d %12d %12d %8d %8d" % (
m, c["functions"], 100.0 * c["complete"] / c["functions"], c["leaf"], c["instructions"], c["untranslated"],
c["blr"], c["indirect_br"]))
lines.append("%-8s %9d %6.2f%% %8d %12d %12d %8d %8d" % (
"all", total["functions"], 100.0 * total["complete"] / total["functions"], total["leaf"],
total["instructions"], total["untranslated"], total["blr"], total["indirect_br"]))
lines.append("undefined encodings reached (UDF, unallocated: trap, as on the hardware): %d sites" % total["undefined"])
lines.append("translated instructions: %.4f%% of %d (shared tails counted once per function)" % (
100.0 - 100.0 * total["untranslated"] / max(1, total["instructions"]), total["instructions"]))
print("\n".join(lines))
with open(os.path.join(out_dir, "coverage.txt"), "w") as f:
f.write("\n".join(lines) + "\n\nuntranslated words by site count\n")
f.write("".join("%08x %d\n" % (w, n) for w, n in sites.most_common()))
return total
def main():
kit.wait_for_quiet('the lift')
p = argparse.ArgumentParser()
p.add_argument("--exefs", default=kit.EXEFS)
p.add_argument("--work", default=kit.WORK)
p.add_argument("--write", action="store_true")
p.add_argument("--max-func", type=int, default=0)
p.add_argument("--layout", choices=("eden", "a64"), default="eden")
p.add_argument("--aslr", type=lambda v: int(v, 0), default=0)
p.add_argument("--no-polls", action="store_true", help="no loop-header interrupt polls (nx2/poll.py)")
p.add_argument("--rechunk", action="store_true", help="chunk boundaries from the sizes again (nx2/chunks.py)")
p.add_argument("--standalone", action="store_true", help="the standalone build's lift (nx2/standalone.py)")
p.add_argument("--entries", help="extra entry addresses, one hex address a line (e.g. the misses ArmRecomp logged)")
p.add_argument("--no-adr-entries", action="store_true",
help="no extra entries at the code addresses ADR / ADRP+ADD take (nx2/adr_entries.py)")
a = p.parse_args()
a.exefs = a.exefs or kit.need_exefs()
t0 = time.time()
hle = ()
if a.standalone:
hle, stubbed = standalone.hle_set(nso.load_exefs(a.exefs))
print("standalone: %d HLE functions in host/, %d OS-facing imports stubbed" % (len(hle), len(stubbed)))
hle = hle | stubbed
img = eden_image.EdenImage(a.exefs, a.aslr, hle=hle) if a.layout == "eden" else nx_image.NxImage(a.exefs)
out_dir = os.path.join(a.work, "sa") if a.standalone else a.work
img_dir = os.path.join(out_dir, "image")
meta = img.write(img_dir)
print("image: %s; %d entries, %d trampolines (%d unresolved non-weak imports) (%.0fs)" % (
", ".join("%s@%#x+%#x" % (m["name"], m["base"], m["size"]) for m in meta["modules"]),
len(img.entries), len(img.tramp), len(img.unresolved), time.time() - t0))
if img.unresolved:
print(" unresolved: " + ", ".join(sorted(img.unresolved)[:16]))
if a.entries:
extra = {int(l.split()[0], 16) for l in open(a.entries) if l.strip() and not l.startswith("#")}
new = {e for e in extra if img.in_code(e)} - img.entries
img.entries |= new
print("extra entries: %d of %d listed are new code addresses" % (len(new), len(extra)))
if not a.no_adr_entries: # (hookshot) callbacks reached only through a pointer the code builds
spans = adr_entries.fde_spans(img.modules)
refs = adr_entries.code_address_refs(open(os.path.join(img_dir, "image.bin"), "rb").read(), meta, spans=spans)
new = {e for e in refs if img.in_code(e)} - img.entries
img.entries |= new
print("ADR / ADRP+ADD code addresses outside the %d unwind-covered functions: %d, %d of them new entries" % (
len(spans), len(refs), len(new)))
L = a64_lift.Lifter(img)
if a.standalone: # (hookshot) widescreen and ultrawide: C after a few of the game's instructions
import oot_wide
oot_wide.install(L, img)
import oot_jitter # (the temporal upscalers' camera jitter, DLSS's and FSR 3.1's: none unless one runs, host/rx_backend.c)
oot_jitter.install(L, img)
import oot_occlusion # (no occlusion queries when HOOKSHOT_OCCLUSION_OFF=1: Fritchii's Occlusion Off)
oot_occlusion.install(L, img)
L.discover(max_insns=a.max_func)
print("discovered %d functions (%.0fs)" % (len(L.funcs), time.time() - t0))
total = coverage(L, img, img_dir)
jt = L.stats
print("jump tables %d (bounded %d, bounded by function %d, ADR table %d), BR without a table %d, PLT stubs %d" % (
len(L.jt), jt["jt_bounded"], jt["jt_unbounded"], jt["jt_adr_table"], len(L.jt_missing), len(L.plt)))
if jt["jt_table_reg_reused"]:
print("jump tables whose table register is reused before the br: %d" % jt["jt_table_reg_reused"])
for pc, reg in sorted(L.jt_missing.items()):
print(" BR without a table: %#x (x%d)" % (pc, reg))
with open(os.path.join(img_dir, "functions.json"), "w") as f:
json.dump({"%x" % e: [len(b), module_of(img, e)] for e, (b, _) in L.funcs.items()}, f)
if a.write:
if not a.no_polls:
poll.install()
gen = os.path.join(out_dir, "gen")
# ((Triforce) setup: what the emit's spawned processes do to the lifter's class before they emit, as this one did above)
files = chunks.write(L, gen, os.path.join(a.work, "image"), a.rechunk and not a.standalone,
counters=[] if a.no_polls else [poll.stats], setup=[] if a.no_polls else [poll.install])
import oot_patches
main_build = next((m["build_id"] for m in meta["modules"] if m["name"] == "main"), "")
# ((Triforce) wait=False: where the emit's processes were spawned they share the page marks while the files below are
# written, which read none of the C; marks_done() waits for them before the report says their counts)
chunks.edit_c(gen, L.stats, a.standalone and main_build == oot_patches.LIFE_BUILD_ID, whole=not a.max_func, wait=False)
a64_rtgen.write(gen)
if a.standalone:
start, exit_ = standalone.write_image_c(gen, img, meta, a.exefs)
print("standalone: hos_image.c (thread hooks %#x, %#x), %d import stubs" % (
start, exit_, standalone.gen_stubs(gen, meta)))
nvn = standalone.nvn_exports(nso.load_exefs(a.exefs))
standalone.write_nvn_exports(gen, nvn)
print("standalone: nvn2_exports.c (%d sdk NVN functions routed to host/nvn2.c)" % len(nvn))
chunks.marks_done()
L.report()
print("interrupt polls: %d loop headers in %d functions" % (poll.stats["polls"], poll.stats["functions"]))
print("wrote %d code files to %s (%.0fs)" % (len(files), gen, time.time() - t0))
print("done (%.0fs)" % (time.time() - t0))
if __name__ == "__main__":
main()