mirror of
https://git.triforceproject.com/triforce/source
synced 2026-11-22 17:19:58 -04:00
The 0.6 crashes after warps and area changes fixed (no write-back into memory the capture leaves out), invisible objects fixed (occlusion gate), DLSS frame generation flicker fixed, achievements, the renderer built with a play profile, faster translated game code, helper threads judged on settled scenes, memory, shader preparation, audio effects, launcher, controls and Linux fixes.
217 lines
11 KiB
Python
217 lines
11 KiB
Python
"""The lift driver: image (we are rtld) -> function discovery -> C, on the vendored a64-kit plus the Switch 2
|
|
additions (nso2: DT_RELR, insn: ARMv8.1-8.5 forms, lift_fixes: jump tables).
|
|
|
|
Usage: python -I nx2/lift.py [--exefs <dir>] [--work <dir>] [--write] [--max-func N] [--layout eden|a64] [--aslr 0]
|
|
--layout eden (default): Eglantine's addresses, rtld included, from 0x80000000 + --aslr (image.py; an emulator
|
|
run must use the same offset: rng_seed_enabled=true, rng_seed=<aslr >> 12>);
|
|
--layout a64: a64-kit's own layout from 0x08000000 (its native runtime, below 4 GB)
|
|
Always: builds <work>/image (image.bin, image.json), discovers every function of rtld-less main + subsdk* + sdk,
|
|
and prints coverage: functions, instructions, jump tables, indirect branches, and the functions whose every
|
|
instruction translates (coverage.txt has the untranslated words by site count).
|
|
--write: also writes the lifted C to <work>/gen (about a gigabyte; game code: stays local, never committed).
|
|
--standalone: the standalone build's lift (<work>/sa/image, <work>/sa/gen): the SDK's OS-facing functions hooked
|
|
to the native runtime in host/ or to generated stubs, plus hos_image.c (nx2/standalone.py). It shares the
|
|
chunk boundaries of <work>/image, so its unhooked chunks are the emulator build's, byte for byte.
|
|
"""
|
|
import argparse
|
|
import collections
|
|
import glob
|
|
import json
|
|
import os
|
|
import struct
|
|
import sys
|
|
import time
|
|
|
|
import capstone
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # (python -I leaves the script folder off)
|
|
import kit
|
|
import nso2 # noqa: F401 (DT_RELR)
|
|
import insn
|
|
import lift_fixes
|
|
import fastvec
|
|
import fma_soft
|
|
import fma_steps
|
|
import vecfp
|
|
import chunks
|
|
import standalone
|
|
import adr_entries # noqa: E402
|
|
import nso
|
|
import poll
|
|
import nx_image
|
|
import image as eden_image
|
|
import a64emit
|
|
import a64_lift
|
|
import a64_rtgen
|
|
|
|
insn.install()
|
|
lift_fixes.install()
|
|
fastvec.install() # ((Triforce, 11-21, LIFT-02) TBL, EXT, 128-bit loads and DUP as one instruction each: nx2/fastvec.py)
|
|
fma_soft.install() # ((Triforce, 11-22, LIFT-06) the exact FMA bodies' short way out where the CPU has no FMA3: nx2/fma_soft.py)
|
|
fma_steps.install() # ((Triforce, 11-22, LIFT-05) FRECPS and FRSQRTS S as one fused instruction where the build has FMA3: nx2/fma_steps.py)
|
|
vecfp.install() # ((Triforce, 11-22, LIFT-03) SSE fast paths before float forms lifted lane by lane, with A64_VECFP=1: nx2/vecfp.py)
|
|
|
|
|
|
def module_of(img, pc):
|
|
for lo, hi, name in img.code:
|
|
if lo <= pc < hi:
|
|
return name
|
|
return "?"
|
|
|
|
|
|
def coverage(L, img, out_dir):
|
|
"""Per-module: functions, instructions, untranslated words; and the share of functions that translate fully."""
|
|
keys = {}
|
|
for body, _ in L.funcs.values():
|
|
for pc, (k, w) in body.items():
|
|
if k[0] == "op":
|
|
keys.setdefault(w, pc)
|
|
words = list(keys)
|
|
_, flags = a64emit.translate([(keys[w], w) for w in words])
|
|
bad = {w for w, f in zip(words, flags) if f & a64emit.UNHANDLED}
|
|
# UDF and unallocated encodings trap on the hardware too (code falls into them after noreturn calls): the
|
|
# lifter's trap is their exact translation, so they do not count against coverage
|
|
md = capstone.Cs(capstone.CS_ARCH_ARM64, capstone.CS_MODE_ARM)
|
|
undefined = set()
|
|
for w in bad:
|
|
ins = next(md.disasm(struct.pack("<I", w), 0), None)
|
|
if ins is None or ins.mnemonic == "udf":
|
|
undefined.add(w)
|
|
bad -= undefined
|
|
per = collections.defaultdict(collections.Counter)
|
|
sites = collections.Counter()
|
|
for e, (body, _) in L.funcs.items():
|
|
m = module_of(img, e)
|
|
n_bad = 0
|
|
for pc, (k, w) in body.items():
|
|
if k[0] == "op" and w in bad:
|
|
n_bad += 1
|
|
sites[w] += 1
|
|
elif k[0] == "op" and w in undefined:
|
|
per[m]["undefined"] += 1
|
|
kinds = collections.Counter(k[0] for k, _ in body.values())
|
|
c = per[m]
|
|
c["functions"] += 1
|
|
c["instructions"] += len(body)
|
|
c["untranslated"] += n_bad
|
|
c["complete"] += n_bad == 0
|
|
c["leaf"] += not (kinds["bl"] or kinds["blr"] or kinds["svc"])
|
|
c["indirect_br"] += sum(1 for pc, (k, _) in body.items() if k[0] == "br" and pc not in L.jt)
|
|
c["blr"] += kinds["blr"]
|
|
total = collections.Counter()
|
|
lines = ["module functions complete leaf instructions untranslated BLR BR(not table)"]
|
|
for m in sorted(per, key=lambda m: -per[m]["instructions"]):
|
|
c = per[m]
|
|
total.update(c)
|
|
lines.append("%-8s %9d %6.2f%% %8d %12d %12d %8d %8d" % (
|
|
m, c["functions"], 100.0 * c["complete"] / c["functions"], c["leaf"], c["instructions"], c["untranslated"],
|
|
c["blr"], c["indirect_br"]))
|
|
lines.append("%-8s %9d %6.2f%% %8d %12d %12d %8d %8d" % (
|
|
"all", total["functions"], 100.0 * total["complete"] / total["functions"], total["leaf"],
|
|
total["instructions"], total["untranslated"], total["blr"], total["indirect_br"]))
|
|
lines.append("undefined encodings reached (UDF, unallocated: trap, as on the hardware): %d sites" % total["undefined"])
|
|
lines.append("translated instructions: %.4f%% of %d (shared tails counted once per function)" % (
|
|
100.0 - 100.0 * total["untranslated"] / max(1, total["instructions"]), total["instructions"]))
|
|
print("\n".join(lines))
|
|
with open(os.path.join(out_dir, "coverage.txt"), "w") as f:
|
|
f.write("\n".join(lines) + "\n\nuntranslated words by site count\n")
|
|
f.write("".join("%08x %d\n" % (w, n) for w, n in sites.most_common()))
|
|
return total
|
|
|
|
|
|
|
|
|
|
def main():
|
|
kit.wait_for_quiet('the lift')
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument("--exefs", default=kit.EXEFS)
|
|
p.add_argument("--work", default=kit.WORK)
|
|
p.add_argument("--write", action="store_true")
|
|
p.add_argument("--max-func", type=int, default=0)
|
|
p.add_argument("--layout", choices=("eden", "a64"), default="eden")
|
|
p.add_argument("--aslr", type=lambda v: int(v, 0), default=0)
|
|
p.add_argument("--no-polls", action="store_true", help="no loop-header interrupt polls (nx2/poll.py)")
|
|
p.add_argument("--rechunk", action="store_true", help="chunk boundaries from the sizes again (nx2/chunks.py)")
|
|
p.add_argument("--standalone", action="store_true", help="the standalone build's lift (nx2/standalone.py)")
|
|
p.add_argument("--entries", help="extra entry addresses, one hex address a line (e.g. the misses ArmRecomp logged)")
|
|
p.add_argument("--no-adr-entries", action="store_true",
|
|
help="no extra entries at the code addresses ADR / ADRP+ADD take (nx2/adr_entries.py)")
|
|
a = p.parse_args()
|
|
a.exefs = a.exefs or kit.need_exefs()
|
|
t0 = time.time()
|
|
hle = ()
|
|
if a.standalone:
|
|
hle, stubbed = standalone.hle_set(nso.load_exefs(a.exefs))
|
|
print("standalone: %d HLE functions in host/, %d OS-facing imports stubbed" % (len(hle), len(stubbed)))
|
|
hle = hle | stubbed
|
|
img = eden_image.EdenImage(a.exefs, a.aslr, hle=hle) if a.layout == "eden" else nx_image.NxImage(a.exefs)
|
|
out_dir = os.path.join(a.work, "sa") if a.standalone else a.work
|
|
img_dir = os.path.join(out_dir, "image")
|
|
meta = img.write(img_dir)
|
|
print("image: %s; %d entries, %d trampolines (%d unresolved non-weak imports) (%.0fs)" % (
|
|
", ".join("%s@%#x+%#x" % (m["name"], m["base"], m["size"]) for m in meta["modules"]),
|
|
len(img.entries), len(img.tramp), len(img.unresolved), time.time() - t0))
|
|
if img.unresolved:
|
|
print(" unresolved: " + ", ".join(sorted(img.unresolved)[:16]))
|
|
if a.entries:
|
|
extra = {int(l.split()[0], 16) for l in open(a.entries) if l.strip() and not l.startswith("#")}
|
|
new = {e for e in extra if img.in_code(e)} - img.entries
|
|
img.entries |= new
|
|
print("extra entries: %d of %d listed are new code addresses" % (len(new), len(extra)))
|
|
if not a.no_adr_entries: # (hookshot) callbacks reached only through a pointer the code builds
|
|
spans = adr_entries.fde_spans(img.modules)
|
|
refs = adr_entries.code_address_refs(open(os.path.join(img_dir, "image.bin"), "rb").read(), meta, spans=spans)
|
|
new = {e for e in refs if img.in_code(e)} - img.entries
|
|
img.entries |= new
|
|
print("ADR / ADRP+ADD code addresses outside the %d unwind-covered functions: %d, %d of them new entries" % (
|
|
len(spans), len(refs), len(new)))
|
|
L = a64_lift.Lifter(img)
|
|
if a.standalone: # (hookshot) widescreen and ultrawide: C after a few of the game's instructions
|
|
import oot_wide
|
|
oot_wide.install(L, img)
|
|
import oot_jitter # (the temporal upscalers' camera jitter, DLSS's and FSR 3.1's: none unless one runs, host/rx_backend.c)
|
|
oot_jitter.install(L, img)
|
|
import oot_occlusion # (no occlusion queries when HOOKSHOT_OCCLUSION_OFF=1: Fritchii's Occlusion Off)
|
|
oot_occlusion.install(L, img)
|
|
L.discover(max_insns=a.max_func)
|
|
print("discovered %d functions (%.0fs)" % (len(L.funcs), time.time() - t0))
|
|
total = coverage(L, img, img_dir)
|
|
jt = L.stats
|
|
print("jump tables %d (bounded %d, bounded by function %d, ADR table %d), BR without a table %d, PLT stubs %d" % (
|
|
len(L.jt), jt["jt_bounded"], jt["jt_unbounded"], jt["jt_adr_table"], len(L.jt_missing), len(L.plt)))
|
|
if jt["jt_table_reg_reused"]:
|
|
print("jump tables whose table register is reused before the br: %d" % jt["jt_table_reg_reused"])
|
|
for pc, reg in sorted(L.jt_missing.items()):
|
|
print(" BR without a table: %#x (x%d)" % (pc, reg))
|
|
with open(os.path.join(img_dir, "functions.json"), "w") as f:
|
|
json.dump({"%x" % e: [len(b), module_of(img, e)] for e, (b, _) in L.funcs.items()}, f)
|
|
if a.write:
|
|
if not a.no_polls:
|
|
poll.install()
|
|
gen = os.path.join(out_dir, "gen")
|
|
# ((Triforce) setup: what the emit's spawned processes do to the lifter's class before they emit, as this one did above)
|
|
files = chunks.write(L, gen, os.path.join(a.work, "image"), a.rechunk and not a.standalone,
|
|
counters=[] if a.no_polls else [poll.stats], setup=[] if a.no_polls else [poll.install])
|
|
import oot_patches
|
|
main_build = next((m["build_id"] for m in meta["modules"] if m["name"] == "main"), "")
|
|
# ((Triforce) wait=False: where the emit's processes were spawned they share the page marks while the files below are
|
|
# written, which read none of the C; marks_done() waits for them before the report says their counts)
|
|
chunks.edit_c(gen, L.stats, a.standalone and main_build == oot_patches.LIFE_BUILD_ID, whole=not a.max_func, wait=False)
|
|
a64_rtgen.write(gen)
|
|
if a.standalone:
|
|
start, exit_ = standalone.write_image_c(gen, img, meta, a.exefs)
|
|
print("standalone: hos_image.c (thread hooks %#x, %#x), %d import stubs" % (
|
|
start, exit_, standalone.gen_stubs(gen, meta)))
|
|
nvn = standalone.nvn_exports(nso.load_exefs(a.exefs))
|
|
standalone.write_nvn_exports(gen, nvn)
|
|
print("standalone: nvn2_exports.c (%d sdk NVN functions routed to host/nvn2.c)" % len(nvn))
|
|
chunks.marks_done()
|
|
L.report()
|
|
print("interrupt polls: %d loop headers in %d functions" % (poll.stats["polls"], poll.stats["functions"]))
|
|
print("wrote %d code files to %s (%.0fs)" % (len(files), gen, time.time() - t0))
|
|
print("done (%.0fs)" % (time.time() - t0))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|