mirror of
https://git.triforceproject.com/triforce/source
synced 2026-11-20 00:29:57 -04:00
Triforce v0.2.0
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
204
nx2/lift.py
Normal file
204
nx2/lift.py
Normal file
@@ -0,0 +1,204 @@
|
||||
"""The lift driver: image (we are rtld) -> function discovery -> C, on the vendored a64-kit plus the Switch 2
|
||||
additions (nso2: DT_RELR, insn: ARMv8.1-8.5 forms, lift_fixes: jump tables).
|
||||
|
||||
Usage: python -I nx2/lift.py [--exefs <dir>] [--work <dir>] [--write] [--max-func N] [--layout eden|a64] [--aslr 0]
|
||||
--layout eden (default): Eglantine's addresses, rtld included, from 0x80000000 + --aslr (image.py; an emulator
|
||||
run must use the same offset: rng_seed_enabled=true, rng_seed=<aslr >> 12>);
|
||||
--layout a64: a64-kit's own layout from 0x08000000 (its native runtime, below 4 GB)
|
||||
Always: builds <work>/image (image.bin, image.json), discovers every function of rtld-less main + subsdk* + sdk,
|
||||
and prints coverage: functions, instructions, jump tables, indirect branches, and the functions whose every
|
||||
instruction translates (coverage.txt has the untranslated words by site count).
|
||||
--write: also writes the lifted C to <work>/gen (about a gigabyte; game code: stays local, never committed).
|
||||
--standalone: the standalone build's lift (<work>/sa/image, <work>/sa/gen): the SDK's OS-facing functions hooked
|
||||
to the native runtime in host/ or to generated stubs, plus hos_image.c (nx2/standalone.py). It shares the
|
||||
chunk boundaries of <work>/image, so its unhooked chunks are the emulator build's, byte for byte.
|
||||
"""
|
||||
import argparse
|
||||
import collections
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import struct
|
||||
import sys
|
||||
import time
|
||||
|
||||
import capstone
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # (python -I leaves the script folder off)
|
||||
import kit
|
||||
import nso2 # noqa: F401 (DT_RELR)
|
||||
import insn
|
||||
import lift_fixes
|
||||
import chunks
|
||||
import standalone
|
||||
import adr_entries # noqa: E402
|
||||
import nso
|
||||
import poll
|
||||
import nx_image
|
||||
import image as eden_image
|
||||
import a64emit
|
||||
import a64_lift
|
||||
import a64_rtgen
|
||||
|
||||
insn.install()
|
||||
lift_fixes.install()
|
||||
|
||||
|
||||
def module_of(img, pc):
|
||||
for lo, hi, name in img.code:
|
||||
if lo <= pc < hi:
|
||||
return name
|
||||
return "?"
|
||||
|
||||
|
||||
def coverage(L, img, out_dir):
|
||||
"""Per-module: functions, instructions, untranslated words; and the share of functions that translate fully."""
|
||||
keys = {}
|
||||
for body, _ in L.funcs.values():
|
||||
for pc, (k, w) in body.items():
|
||||
if k[0] == "op":
|
||||
keys.setdefault(w, pc)
|
||||
words = list(keys)
|
||||
_, flags = a64emit.translate([(keys[w], w) for w in words])
|
||||
bad = {w for w, f in zip(words, flags) if f & a64emit.UNHANDLED}
|
||||
# UDF and unallocated encodings trap on the hardware too (code falls into them after noreturn calls): the
|
||||
# lifter's trap is their exact translation, so they do not count against coverage
|
||||
md = capstone.Cs(capstone.CS_ARCH_ARM64, capstone.CS_MODE_ARM)
|
||||
undefined = set()
|
||||
for w in bad:
|
||||
ins = next(md.disasm(struct.pack("<I", w), 0), None)
|
||||
if ins is None or ins.mnemonic == "udf":
|
||||
undefined.add(w)
|
||||
bad -= undefined
|
||||
per = collections.defaultdict(collections.Counter)
|
||||
sites = collections.Counter()
|
||||
for e, (body, _) in L.funcs.items():
|
||||
m = module_of(img, e)
|
||||
n_bad = 0
|
||||
for pc, (k, w) in body.items():
|
||||
if k[0] == "op" and w in bad:
|
||||
n_bad += 1
|
||||
sites[w] += 1
|
||||
elif k[0] == "op" and w in undefined:
|
||||
per[m]["undefined"] += 1
|
||||
kinds = collections.Counter(k[0] for k, _ in body.values())
|
||||
c = per[m]
|
||||
c["functions"] += 1
|
||||
c["instructions"] += len(body)
|
||||
c["untranslated"] += n_bad
|
||||
c["complete"] += n_bad == 0
|
||||
c["leaf"] += not (kinds["bl"] or kinds["blr"] or kinds["svc"])
|
||||
c["indirect_br"] += sum(1 for pc, (k, _) in body.items() if k[0] == "br" and pc not in L.jt)
|
||||
c["blr"] += kinds["blr"]
|
||||
total = collections.Counter()
|
||||
lines = ["module functions complete leaf instructions untranslated BLR BR(not table)"]
|
||||
for m in sorted(per, key=lambda m: -per[m]["instructions"]):
|
||||
c = per[m]
|
||||
total.update(c)
|
||||
lines.append("%-8s %9d %6.2f%% %8d %12d %12d %8d %8d" % (
|
||||
m, c["functions"], 100.0 * c["complete"] / c["functions"], c["leaf"], c["instructions"], c["untranslated"],
|
||||
c["blr"], c["indirect_br"]))
|
||||
lines.append("%-8s %9d %6.2f%% %8d %12d %12d %8d %8d" % (
|
||||
"all", total["functions"], 100.0 * total["complete"] / total["functions"], total["leaf"],
|
||||
total["instructions"], total["untranslated"], total["blr"], total["indirect_br"]))
|
||||
lines.append("undefined encodings reached (UDF, unallocated: trap, as on the hardware): %d sites" % total["undefined"])
|
||||
lines.append("translated instructions: %.4f%% of %d (shared tails counted once per function)" % (
|
||||
100.0 - 100.0 * total["untranslated"] / max(1, total["instructions"]), total["instructions"]))
|
||||
print("\n".join(lines))
|
||||
with open(os.path.join(out_dir, "coverage.txt"), "w") as f:
|
||||
f.write("\n".join(lines) + "\n\nuntranslated words by site count\n")
|
||||
f.write("".join("%08x %d\n" % (w, n) for w, n in sites.most_common()))
|
||||
return total
|
||||
|
||||
|
||||
|
||||
|
||||
def main():
|
||||
kit.wait_for_quiet('the lift')
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--exefs", default=kit.EXEFS)
|
||||
p.add_argument("--work", default=kit.WORK)
|
||||
p.add_argument("--write", action="store_true")
|
||||
p.add_argument("--max-func", type=int, default=0)
|
||||
p.add_argument("--layout", choices=("eden", "a64"), default="eden")
|
||||
p.add_argument("--aslr", type=lambda v: int(v, 0), default=0)
|
||||
p.add_argument("--no-polls", action="store_true", help="no loop-header interrupt polls (nx2/poll.py)")
|
||||
p.add_argument("--rechunk", action="store_true", help="chunk boundaries from the sizes again (nx2/chunks.py)")
|
||||
p.add_argument("--standalone", action="store_true", help="the standalone build's lift (nx2/standalone.py)")
|
||||
p.add_argument("--entries", help="extra entry addresses, one hex address a line (e.g. the misses ArmRecomp logged)")
|
||||
p.add_argument("--no-adr-entries", action="store_true",
|
||||
help="no extra entries at the code addresses ADR / ADRP+ADD take (nx2/adr_entries.py)")
|
||||
a = p.parse_args()
|
||||
a.exefs = a.exefs or kit.need_exefs()
|
||||
t0 = time.time()
|
||||
hle = ()
|
||||
if a.standalone:
|
||||
hle, stubbed = standalone.hle_set(nso.load_exefs(a.exefs))
|
||||
print("standalone: %d HLE functions in host/, %d OS-facing imports stubbed" % (len(hle), len(stubbed)))
|
||||
hle = hle | stubbed
|
||||
img = eden_image.EdenImage(a.exefs, a.aslr, hle=hle) if a.layout == "eden" else nx_image.NxImage(a.exefs)
|
||||
out_dir = os.path.join(a.work, "sa") if a.standalone else a.work
|
||||
img_dir = os.path.join(out_dir, "image")
|
||||
meta = img.write(img_dir)
|
||||
print("image: %s; %d entries, %d trampolines (%d unresolved non-weak imports) (%.0fs)" % (
|
||||
", ".join("%s@%#x+%#x" % (m["name"], m["base"], m["size"]) for m in meta["modules"]),
|
||||
len(img.entries), len(img.tramp), len(img.unresolved), time.time() - t0))
|
||||
if img.unresolved:
|
||||
print(" unresolved: " + ", ".join(sorted(img.unresolved)[:16]))
|
||||
if a.entries:
|
||||
extra = {int(l.split()[0], 16) for l in open(a.entries) if l.strip() and not l.startswith("#")}
|
||||
new = {e for e in extra if img.in_code(e)} - img.entries
|
||||
img.entries |= new
|
||||
print("extra entries: %d of %d listed are new code addresses" % (len(new), len(extra)))
|
||||
if not a.no_adr_entries: # (hookshot) callbacks reached only through a pointer the code builds
|
||||
spans = adr_entries.fde_spans(img.modules)
|
||||
refs = adr_entries.code_address_refs(open(os.path.join(img_dir, "image.bin"), "rb").read(), meta, spans=spans)
|
||||
new = {e for e in refs if img.in_code(e)} - img.entries
|
||||
img.entries |= new
|
||||
print("ADR / ADRP+ADD code addresses outside the %d unwind-covered functions: %d, %d of them new entries" % (
|
||||
len(spans), len(refs), len(new)))
|
||||
L = a64_lift.Lifter(img)
|
||||
if a.standalone: # (hookshot) widescreen and ultrawide: C after a few of the game's instructions
|
||||
import oot_wide
|
||||
oot_wide.install(L, img)
|
||||
import oot_jitter # (the temporal upscalers' camera jitter, DLSS's and FSR 3.1's: none unless one runs, host/rx_backend.c)
|
||||
oot_jitter.install(L, img)
|
||||
import oot_occlusion # (no occlusion queries when HOOKSHOT_OCCLUSION_OFF=1: Fritchii's Occlusion Off)
|
||||
oot_occlusion.install(L, img)
|
||||
L.discover(max_insns=a.max_func)
|
||||
print("discovered %d functions (%.0fs)" % (len(L.funcs), time.time() - t0))
|
||||
total = coverage(L, img, img_dir)
|
||||
jt = L.stats
|
||||
print("jump tables %d (bounded %d, bounded by function %d, ADR table %d), BR without a table %d, PLT stubs %d" % (
|
||||
len(L.jt), jt["jt_bounded"], jt["jt_unbounded"], jt["jt_adr_table"], len(L.jt_missing), len(L.plt)))
|
||||
if jt["jt_table_reg_reused"]:
|
||||
print("jump tables whose table register is reused before the br: %d" % jt["jt_table_reg_reused"])
|
||||
for pc, reg in sorted(L.jt_missing.items()):
|
||||
print(" BR without a table: %#x (x%d)" % (pc, reg))
|
||||
with open(os.path.join(img_dir, "functions.json"), "w") as f:
|
||||
json.dump({"%x" % e: [len(b), module_of(img, e)] for e, (b, _) in L.funcs.items()}, f)
|
||||
if a.write:
|
||||
if not a.no_polls:
|
||||
poll.install()
|
||||
gen = os.path.join(out_dir, "gen")
|
||||
files = chunks.write(L, gen, os.path.join(a.work, "image"), a.rechunk and not a.standalone,
|
||||
counters=[] if a.no_polls else [poll.stats])
|
||||
import oot_patches
|
||||
main_build = next((m["build_id"] for m in meta["modules"] if m["name"] == "main"), "")
|
||||
chunks.edit_c(gen, L.stats, a.standalone and main_build == oot_patches.LIFE_BUILD_ID, whole=not a.max_func)
|
||||
a64_rtgen.write(gen)
|
||||
if a.standalone:
|
||||
start, exit_ = standalone.write_image_c(gen, img, meta, a.exefs)
|
||||
print("standalone: hos_image.c (thread hooks %#x, %#x), %d import stubs" % (
|
||||
start, exit_, standalone.gen_stubs(gen, meta)))
|
||||
nvn = standalone.nvn_exports(nso.load_exefs(a.exefs))
|
||||
standalone.write_nvn_exports(gen, nvn)
|
||||
print("standalone: nvn2_exports.c (%d sdk NVN functions routed to host/nvn2.c)" % len(nvn))
|
||||
L.report()
|
||||
print("interrupt polls: %d loop headers in %d functions" % (poll.stats["polls"], poll.stats["functions"]))
|
||||
print("wrote %d code files to %s (%.0fs)" % (len(files), gen, time.time() - t0))
|
||||
print("done (%.0fs)" % (time.time() - t0))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user