Files
Triforce-Source/third_party/a64-kit/lift/a64_lift.py
ruckus7382sfgsgdf 34b3683441 Triforce v0.2.0
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-10 19:32:48 -06:00

1849 lines
98 KiB
Python

"""a64-kit lifter: AArch64 code image -> C.
Model (ppc-kit's, see ppc-kit/lifter/ppc_lift.py)
* Every guest function becomes `void f_XXXXXXXX(Ctx *restrict c)`. A "function" is an entry address plus
everything reachable from it without crossing another entry. Any address can be an entry and running from it
is exactly the original behaviour, so extra entries only cost code size. Branches and fallthrough into another
entry become tail calls.
* B/B.cond/CBZ/TBZ = goto; BL = C call (LR written first); BLR = A64_ICALL (a64_icall with a cache per site);
RET = C return.
* BR: a recovered jump table becomes a switch on the target address; any other BR is an indirect tail call
(A64_IJUMP). PLT stubs (adrp x16 / ldr x17 / add x16 / br x17) are resolved through the already-relocated
GOT, so a call to one calls its target directly.
* Every other instruction is translated by suyu's emitter (a64emit), which writes C against the context.
* SVC calls a64_svc; BRK, HLT and anything the emitter cannot translate stop the thread through a64_trap.
Input (image): .entries (set), .hooks {addr: c_name}, .word(addr), .in_code(addr), optional .names {addr: str}.
Output: <out>/code_NNN.c, <out>/a64_table.c, <out>/a64_hooks.c, <out>/a64_fpx.h, <out>/functions.txt
"""
import bisect
import os
import re
import struct
import sys
from collections import Counter
import a64emit
COND = ["(c->z)", "(!c->z)", "(c->c)", "(!c->c)", "(c->n)", "(!c->n)", "(c->v)", "(!c->v)",
"(c->c && !c->z)", "(!(c->c && !c->z))", "(c->n == c->v)", "(c->n != c->v)",
"((c->n == c->v) && !c->z)", "(!((c->n == c->v) && !c->z))", "(1)", "(1)"]
CHUNK_BYTES = 24 << 20
def sx(v, bits):
return v - (1 << bits) if v & (1 << (bits - 1)) else v
def classify(w, pc):
"""(kind, ...) for control flow and specials; ("op",) for everything the emitter translates."""
if (w & 0x7C000000) == 0x14000000:
return ("bl" if w & 0x80000000 else "b", pc + 4 * sx(w & 0x3FFFFFF, 26))
if (w & 0xFF000010) == 0x54000000:
return ("bcond", pc + 4 * sx((w >> 5) & 0x7FFFF, 19), w & 15)
if (w & 0x7E000000) == 0x34000000:
return ("cbz", pc + 4 * sx((w >> 5) & 0x7FFFF, 19), w & 31, w >> 31, (w >> 24) & 1)
if (w & 0x7E000000) == 0x36000000:
return ("tbz", pc + 4 * sx((w >> 5) & 0x3FFF, 14), w & 31, ((w >> 31) << 5) | ((w >> 19) & 31), (w >> 24) & 1)
if (w & 0xFFFFFC1F) == 0xD61F0000:
return ("br", (w >> 5) & 31)
if (w & 0xFFFFFC1F) == 0xD63F0000:
return ("blr", (w >> 5) & 31)
if (w & 0xFFFFFC1F) == 0xD65F0000:
return ("ret", (w >> 5) & 31)
if (w & 0xFFE0001F) == 0xD4000001:
return ("svc", (w >> 5) & 0xFFFF)
if (w & 0xFFE0001F) in (0xD4200000, 0xD4400000):
return ("trap",)
if (w & 0xFFFFF0FF) == 0xD50330DF:
return ("isb",)
if (w & 0xFFFFFFE0) == 0xD50B7520:
return ("ic",)
return ("op",)
def pc_relative(w):
return (w & 0x1F000000) == 0x10000000 or (w & 0x3B000000) == 0x18000000
# suyu's FPX1 guard around every FP lane op: native fast path, exact integer body as the fallback. Inline at every
# site it is most of the IR of FP-heavy functions (87k guards in TOTK's physics chunk, clang -O1 ran for >10 minutes
# on 20 functions), so each guard becomes one call of a helper per (fast op, exact op) pair, written to a64_fpx.h.
FPX_GUARD = re.compile(r"\{if\(RECOMP_FPX_OPEN\(c\)&&RECOMP_LIKELY\(recomp_fpx_(\w+)\(([^;]*?),RECOMP_FPX_FZ\(c\),"
r"&(\w+)\)\)\)\{RECOMP_FPX_PROBE\(\d+\);\}else\{RECOMP_FPX_PROBE\(\d+\);(\w+)=recomp_fpe_(\w+)"
r"\(c,([^;]*?)\);\}\}")
def outline_fpx(text, used):
def sub(m):
fast, fast_args, out, out2, exact, exact_args = m.groups()
nf, ne = fast_args.count(",") + 1, exact_args.count(",") + 1
if out != out2 or exact_args.split(",")[:nf] != fast_args.split(","):
return m.group(0)
used.add((fast, exact, nf, ne))
return "%s=a64_fpx_%s_%s(c,%s);" % (out, fast, exact, exact_args)
return FPX_GUARD.sub(sub, text) if "RECOMP_FPX_OPEN" in text else text
# Straight 2- and 4-lane NEON FADD / FSUB and 4-lane FMUL (.2S / .4S) become one SSE op with the scalar fast path's
# exactness tests on every lane (a64_vfpx_* in a64_fpx.h); when a lane needs the exact body, one out-of-line call runs
# each lane's helper as before. They were one helper call per lane (TOTK: ~124k such ops, ~500k of its 690k FP calls).
VFPX_ADD = re.compile(r"\{ uint32_t _n\[4\],_m\[4\],_r\[4\]=\{0\};memcpy\(_n,c->vreg\[(\d+)\],16\);memcpy\(_m,c->vreg\[(\d+)\],16\);\s*"
r"((?:\{uint64_t _a=_n\[\d\],_b=_m\[\d\],_v=0;_v=a64_fpx_(?:add32_add32|sub32_sub32)\(c,_a,_b\);"
r"_r\[\d\]=\(uint32_t\)_v;\}\s*)+)memcpy\(c->vreg\[(\d+)\],_r,16\);\}")
VFPX_LANE = re.compile(r"_a=_n\[(\d)\],_b=_m\[(\d)\],_v=0;_v=a64_fpx_(add32_add32|sub32_sub32)\(c,_a,_b\);_r\[(\d)\]")
VFPX_MUL = re.compile(r"\{uint32_t _n\[4\],_m\[4\],_d\[4\];memcpy\(_n,c->vreg\[(\d+)\],16\);memcpy\(_m,c->vreg\[(\d+)\],16\);"
r"for\(unsigned _j=0;_j<4;\+\+_j\)\{uint64_t _a=_n\[_j\],_b=_m\[_j\],_z=\(_a\^_b\)&0x80000000ULL,_v;\s*"
r"_v=a64_fpx_mul32_fma32\(c,_a,_b,_z\);\s*_d\[_j\]=\(uint32_t\)_v;\}memcpy\(c->vreg\[(\d+)\],_d,16\);\s*\}")
# Vector float compares (FCMGE / FCMGT / FCMEQ, .2S / .4S) were exact integer code per lane (NaN and FZ checks, FPSR
# flags): one SSE compare when no lane is a NaN (and, under FZ, none is subnormal), the exact block otherwise.
VFCMP = re.compile(r"\{ uint32_t _n\[4\],_m\[4\],_r\[4\]=\{0\}; memcpy\(_n,c->vreg\[(\d+)\],16\); memcpy\(_m,c->vreg\[(\d+)\],16\); "
r"for\(int _i=0;_i<([24]);_i\+\+\) \{ uint64_t _a=_n\[_i\],_b=_m\[_i\],_v=0; const uint64_t _sign=0x80000000ULL,"
r"_exp=0x7f800000ULL,_frac=0x7fffffULL,_quiet=0x400000ULL; .*?"
r"if\(_an\|\|_bn\) \{ if\([^;]*\) c->fpsr\|=1; \} else \{ int _eq=_a==_b\|\|!\(\(_a\|_b\)&~_sign\); "
r"int _gt=!_eq&&\(\(\(_a\^_b\)&_sign\)\?!\(_a&_sign\):\(\(_a&_sign\)\?_a<_b:_a>_b\)\); \(void\)_gt; \(void\)_eq; "
r"if\((_eq\|\|_gt|_gt|_eq)\) _v=~\(uint64_t\)0; \} _r\[_i\]=\(uint32_t\)_v; \} memcpy\(c->vreg\[(\d+)\],_r,16\); \}")
VFCMP_OP = {"_eq||_gt": 0, "_gt": 1, "_eq": 2}
# ...and against zero (FCMxx #0.0): one source, the other operand the constant 0 (on either side)
VFCMPZ = re.compile(r"\{ uint32_t _n\[4\],_r\[4\]=\{0\}; memcpy\(_n,c->vreg\[(\d+)\],16\); "
r"for\(int _i=0;_i<([24]);_i\+\+\) \{ uint64_t (_a=0,_b=_n\[_i\]|_a=_n\[_i\],_b=0),_v=0; const uint64_t _sign=0x80000000ULL,"
r"_exp=0x7f800000ULL,_frac=0x7fffffULL,_quiet=0x400000ULL; .*?"
r"if\(_an\|\|_bn\) \{ if\([^;]*\) c->fpsr\|=1; \} else \{ int _eq=_a==_b\|\|!\(\(_a\|_b\)&~_sign\); "
r"int _gt=!_eq&&\(\(\(_a\^_b\)&_sign\)\?!\(_a&_sign\):\(\(_a&_sign\)\?_a<_b:_a>_b\)\); \(void\)_gt; \(void\)_eq; "
r"if\((_eq\|\|_gt|_gt|_eq)\) _v=~\(uint64_t\)0; \} _r\[_i\]=\(uint32_t\)_v; \} memcpy\(c->vreg\[(\d+)\],_r,16\); \}")
# Scalar FCMP / FCMPE (S and D, against a register or #0.0): exact integer code too (FZ, NaN and ordering tests, ~5
# branches; ~65k sites in TOTK). One native compare sets the flags when neither operand is a NaN (and, under FZ, neither
# is subnormal); the exact block stays behind it. A64_NO_SFCMP=1 at lift time: off.
SFCMP = re.compile(r"\{ uint64_t _a=0,_b=0; memcpy\(&_a,([^;]+?),([48])\);(?: memcpy\(&_b,([^;]+?),\2\);)? "
r"const uint64_t _sign=0x80+ULL,[^\n]*? c->n=\(uint8_t\)_lt; c->z=\(uint8_t\)_eq; "
r"c->c=\(uint8_t\)!_lt; c->v=0; \} \}")
def vectorize_fcmp(text):
if "_quiet=" not in text or os.environ.get("A64_NO_VFPX"):
return text
if not os.environ.get("A64_NO_SFCMP"): # (braced: FCCMP writes "if (cond) <block> else ...")
if local_flags(): # (the flags stored in the text, where the passes see them: dead ones dropped, locals)
text = SFCMP.sub(lambda m: "{ const uint32_t _fk = a64_fcmp%dk(c,%s,%s); if (_fk) { c->n=(uint8_t)(_fk&1); "
"c->z=(uint8_t)(_fk>>1&1); c->c=(uint8_t)!(_fk&1); c->v=0; } else %s }"
% (32 if m.group(2) == "4" else 64, m.group(1), m.group(3) or "a64_vzero", m.group(0)), text)
else:
text = SFCMP.sub(lambda m: "{ if (!a64_fcmp%d(c,%s,%s)) %s }" % (32 if m.group(2) == "4" else 64, m.group(1),
m.group(3) or "a64_vzero", m.group(0)), text)
if "_quiet=0x400000ULL" not in text:
return text
text = VFCMP.sub(lambda m: "if (!a64_vfcmp(c,c->vreg[%s],c->vreg[%s],c->vreg[%s],%s,%d)) %s"
% (m.group(5), m.group(1), m.group(2), m.group(3), VFCMP_OP[m.group(4)], m.group(0)), text)
def zero(m):
src = "c->vreg[%s]" % m.group(1)
a, b = ("a64_vzero", src) if m.group(3).startswith("_a=0") else (src, "a64_vzero")
return "if (!a64_vfcmp(c,c->vreg[%s],%s,%s,%s,%d)) %s" % (m.group(5), a, b, m.group(2), VFCMP_OP[m.group(4)],
m.group(0))
return VFCMPZ.sub(zero, text)
# Writes of a whole guest vector register as one 16-byte store. The translation writes a scalar result as two 8-byte
# stores (value, then the zeroed top) and an element insert as load-modify-store, which the compiler narrows to a 4-byte
# store; a 16-byte read of that register soon after (a vector op, a q-register store) then cannot take its data from
# those stores (a store-forwarding stall: ~a quarter of the hottest castle function's samples sat on two such reads).
VSET_PAIR = re.compile(r"c->vreg\[(\d+)\]\[0\] ?= ?([^;]+); ?c->vreg\[\1\]\[1\] ?= ?([^;]+);")
VSET_LOW = re.compile(r"c->vreg\[(\d+)\]\[0\]=0; ?c->vreg\[\1\]\[1\]=0; ?memcpy\(c->vreg\[\1\],(&?\w+),([48])\);")
VSET_FULL = re.compile(r"memcpy\(c->vreg\[(\d+)\],(\w+),16\);")
def widen_vstores(text):
if "c->vreg[" not in text or os.environ.get("A64_NO_VSET"):
return text
text = VSET_LOW.sub(lambda m: "a64_vset_lo(c->vreg[%s],%s,%s);" % m.groups(), text)
def pair(m): # (both halves from the old value: the top half's expression must not read the new bottom half;
d, lo, hi = m.groups() # with two expressions no calls in them: their order as arguments is unspecified)
if hi.strip() == "0":
return "a64_vset64(c->vreg[%s],%s);" % (d, lo)
if "c->vreg[%s][0]" % d in hi or re.search(r"\w\s*\(", lo + hi):
return m.group(0)
return "a64_vset128(c->vreg[%s],%s,%s);" % (d, lo, hi)
text = VSET_PAIR.sub(pair, text)
return VSET_FULL.sub(lambda m: "a64_vst16(c->vreg[%s],%s);" % m.groups(), text)
# Stores through SP (register 31 as a base is always SP) go to the guest stack, which the GPU never reads: the
# _sp variants (rt/a64.h) leave out the page marks of the soft write watch (two byte stores each, a quarter of TOTK's
# store sites, nearly all prologue STPs). A64_MARK_STACK=1 at lift time: marked like any store.
SP_STORE_PRE = re.compile(r"(\{ uint64_t _b=c->x\[31\]; int64_t _o=-?\d+; )recomp_(store8|store16|store32|store64|stp32|stp64)\(")
SP_STORE = re.compile(r"recomp_(store8|store16|store32|store64|stp32|stp64)\(c,(\(?)c->x\[31\]") # ((sp + imm), sp + imm)
# Stores through x29 go to the stack too where x29 is the frame pointer: in a function whose every write of x29 is the
# frame setup (x29 = sp + imm) or its restore from the stack (LDP from SP), emit_function gives its x29-based stores
# the _sp variants as well (88k sites in TOTK; a few functions use x29 as an ordinary register and keep their marks).
# Castle, 3 rounds: 66.9 -> 67.4 fps (within noise), exe -2 MB. A64_MARK_FRAME=1 at lift time: marked like any store.
FP_WRITE = re.compile(r"c->x\[29\]\s*=(?!=)|&c->x\[29\]")
FP_SETUP = re.compile(r"\{ uint64_t _a=c->x\[31\], _b=\d+ULL; uint64_t _r=_a\+_b; c->x\[29\]=_r; \}")
FP_RESTORE = re.compile(r"\{ uint64_t _b=c->x\[31\]; int64_t _o=-?\d+; \{ uint64_t _p0,_p1; recomp_ldp64\(c,[^;]*;"
r" c->x\[29\]=_p0; c->x\[30\]=_p1; \}")
FP_STORE_PRE = re.compile(r"(\{ uint64_t _b=c->x\[29\]; int64_t _o=-?\d+; )recomp_(store8|store16|store32|store64|stp32|stp64)\(")
FP_STORE = re.compile(r"recomp_(store8|store16|store32|store64|stp32|stp64)\(c,(\(?)c->x\[29\]")
def fp_is_frame(text):
"""an instruction's C text writes x29 only as frame setup or restore (or not at all)"""
if not FP_WRITE.search(text):
return True
rest = FP_RESTORE.sub("", FP_SETUP.sub("", text))
return not FP_WRITE.search(rest)
def frame_stores(text):
text = FP_STORE_PRE.sub(r"\1recomp_\2_sp(", text)
return FP_STORE.sub(r"recomp_\1_sp(c,\2c->x[29]", text)
def stack_stores(text):
if "c->x[31]" not in text or os.environ.get("A64_MARK_STACK"):
return text
text = SP_STORE_PRE.sub(r"\1recomp_\2_sp(", text)
return SP_STORE.sub(r"recomp_\1_sp(c,\2c->x[31]", text)
# Page marks per group (A2). In a straight-line run of a function's C the marked stores through the same base register
# plus a constant keep their data stores but share one A64_MARK(base + lo, hi - lo) for their byte span [lo, hi) (when it
# is at most 4 KB), placed after the group's last store and before the first later line that writes the base register or
# ends the run. The pages marked are the same: every store of the group lies in [base+lo, base+hi), at most two pages,
# which the stores at lo and ending at hi mark themselves. Each data store is still followed, in program order and before
# any guest synchronisation, by a release store of a mark of its page, so a collection that clears the mark sees the data
# (rt/a64.h A64_MARK).
# It runs on the finished C files, last (nx2/lift.py: after nx2/oot_patches.py and every other edit of the lifted text,
# which match the lift's exact text), so it is not part of emit_function, and the kit's own write() leaves it out: a
# lift without nx2 keeps a mark per store unless it calls group_marks itself. A64_MARK_EACH=1 at lift time: a mark per
# store. Running it again changes nothing (its own marks are recognised, and are not run ends).
# A run ends at anything not shown to be straight-line code without synchronisation: a label, goto, return, a line
# whose braces do not balance (a multi-line block opens or closes: a mark never moves into or out of one), STLR / LDAR
# in suyu's spelling (a plain recomp_storeN(c,c->x[N],...) / recomp_loadN(c,c->x[N])), the register file as a whole,
# a local register (A64_LOCAL_REGS), and every call-like token but MG_PLAIN's: calls (f_*, TAILCALL, hos_* from the
# patches), polls, atomics, barriers, exclusives, SVCs, traps and any helper this list does not know.
MG_STORE = re.compile(r"recomp_(store8|store16|store32|store64|stp32|stp64)\(c,")
MG_SIZE = {"store8": 1, "store16": 2, "store32": 4, "store64": 8, "stp32": 8, "stp64": 16}
# address forms (the text right after "recomp_X(c,")
MG_ADDR = [
(re.compile(r"c->x\[(\d+)\] \+ (-?\d+),"), "x"), # c->x[N] + IMM,
(re.compile(r"\(c->x\[(\d+)\]\+(\d+)ULL\),"), "x"), # (c->x[N]+IMMULL),
(re.compile(r"\(_b\+_o\)\+(\d+),"), "bo+"), # (_b+_o)+IMM,
(re.compile(r"\(_b\+_o\),"), "bo"), # (_b+_o),
(re.compile(r"_b,"), "b"), # _b,
(re.compile(r"_a\+(\d+),"), "a+"), # _a+IMM,
(re.compile(r"_a,"), "a"), # _a,
]
MG_BDEF = re.compile(r"uint64_t _b=c->x\[(\d+)\]; int64_t _o=(-?\d+);")
MG_ADEF = re.compile(r"uint64_t _a=c->x\[(\d+)\];")
# (call-like tokens that are plain: keywords, guest loads and stores without ordering (the stores are what is grouped),
# and helpers that only compute or move values between locals, registers and vector registers)
MG_PLAIN = re.compile(r"(?:if|for|while|sizeof|memcpy|memset|UINT64_C|INT64_C|fabsf|fabs|__builtin_add_overflow|"
r"recomp_(?:load|ldp|store|stp)(?:8|16|32|64)(?:_sp)?|"
r"recomp_(?:fixed_to_fp|fp_to_fixed|fp_to_int|fp_estimate|umulh|smulh|aes_sbox)|"
r"a64_(?:vset64|vset128|vset_lo|vst16|fcmp32|fcmp64|vfcmp)|a64_v?fpx_\w+)\Z")
MG_CALL = re.compile(r"\b([A-Za-z_]\w*)\s*\(")
_mg_plain = {}
# (the other run ends: a label, goto, return, STLR and LDAR, the register file as a whole)
MG_END = re.compile(r"^\s*L_[0-9a-f]+:|\bgoto\b|\breturn\b|recomp_(?:store|stp)\d+\(c,c->x\[\d+\],|"
r"recomp_load\d+\(c,c->x\[\d+\]\)|c->x(?!\[)")
MG_LOCAL = re.compile(r"\bx\d+_ = c->x\[") # (localize_regs's locals: such a function is left as it is)
MG_BRACE = re.compile(r"[{}]")
# a line whose stores may not all run, or run more than once: its stores keep their own marks
MG_NOT_STRAIGHT = re.compile(r"\bif\b|\?|\bfor\b|\bwhile\b|\bswitch\b|\bcase\b")
MG_NOT_STRAIGHT_WORDS = ("if", "?", "for", "while", "switch", "case")
MG_MARK = " A64_MARK((c->x[%d] + (uint64_t)(int64_t)(%d)), %d);"
MG_OWN_MARK = re.compile(r" A64_MARK\(\(c->x\[\d+\] \+ \(uint64_t\)\(int64_t\)\(-?\d+\)\), \d+\);\Z")
MG_HEAD = re.compile(r"\nvoid f_[0-9a-f]+\(") # (a definition or declaration: its line begins after the newline)
def mg_balanced(line):
"""the line's braces close what it opens, in order (a block on one line, or none)"""
if "{" not in line and "}" not in line:
return True
d = 0
for b in MG_BRACE.findall(line):
d += 1 if b == "{" else -1
if d < 0:
return False
return d == 0
def mg_ends_run(line):
"""the line ends a straight-line run (each test only on a line that has its text: this runs on most lines of the lift)"""
if ("{" in line or "}" in line) and not mg_balanced(line):
return True
if "(" in line:
for t in MG_CALL.findall(line):
p = _mg_plain.get(t)
if p is None:
p = _mg_plain[t] = MG_PLAIN.match(t) is not None
if not p:
return True
if "goto" in line or "return" in line or "L_" in line or "(c,c->x[" in line or line.count("c->x") != line.count("c->x["):
return MG_END.search(line) is not None
return False
def mg_parse_store(line, m):
"""(base register, offset, size) of the marked store at match m, or None"""
rest = line[m.end():]
size = MG_SIZE[m.group(1)]
for rx, kind in MG_ADDR:
a = rx.match(rest)
if not a:
continue
if kind == "x":
return int(a.group(1)), int(a.group(2)), size
d = None # (the nearest definition before the store)
for d in (MG_BDEF if kind in ("bo", "bo+", "b") else MG_ADEF).finditer(line, 0, m.start()):
pass
if d is None:
return None
if kind in ("bo", "bo+", "b"):
off = 0 if kind == "b" else int(d.group(2)) + (int(a.group(1)) if kind == "bo+" else 0)
return int(d.group(1)), off, size
return int(d.group(1)), (int(a.group(1)) if kind == "a+" else 0), size
return None
def mark_groups_lines(lines, st):
"""the lines of a function (or of any C text) with the page marks of each group shared"""
inserts = {} # line index -> mark lines to put before it
unmark = {} # line index -> set of match start offsets to turn into _sp
groups = {} # base register -> [(line index, match start, off, size)]
def flush(reg, at):
g = groups.pop(reg, None)
if not g:
return
lo = min(o for _, _, o, _ in g)
hi = max(o + n for _, _, o, n in g)
st["stores"] += len(g)
if len(g) < 2 or hi - lo > 4096:
return
for li, ms, _, _ in g:
unmark.setdefault(li, set()).add(ms)
inserts.setdefault(at, []).append(MG_MARK % (reg, lo, hi - lo))
st["groups"] += 1
st["grouped_stores"] += len(g)
for i, line in enumerate(lines):
if not groups and "recomp_st" not in line:
continue # (nothing to end or to add to)
if line.startswith(" A64_MARK((") and MG_OWN_MARK.match(line):
continue
if mg_ends_run(line):
for r in list(groups):
flush(r, i)
continue
writes = {int(a or b) for a, b in REG_WRITE.findall(line)} if (
"=" in line or "&c->x" in line or "++" in line or "--" in line) else ()
stores = list(MG_STORE.finditer(line)) if "recomp_st" in line else ()
if stores and not writes and not (any(w in line for w in MG_NOT_STRAIGHT_WORDS) and MG_NOT_STRAIGHT.search(line)):
for m in stores:
p = mg_parse_store(line, m)
if p is None or p[0] == 31:
continue # (an address this pass does not follow, or SP: the store keeps what it has)
groups.setdefault(p[0], []).append((i, m.start(), p[1], p[2]))
continue
for r in writes: # (the base changes here: its group's mark goes before this line)
if r in groups:
flush(r, i)
for r in list(groups):
flush(r, len(lines))
if not unmark:
return lines
out = []
for i, line in enumerate(lines):
out.extend(inserts.get(i, []))
if i in unmark:
for ms in sorted(unmark[i], reverse=True):
m = MG_STORE.match(line, ms)
line = line[:m.start()] + "recomp_%s_sp(c," % m.group(1) + line[m.end():]
out.append(line)
out.extend(inserts.get(len(lines), []))
return out
def mark_groups(text, stats=None):
"""text (a C file of lifted functions, or one function) with the page marks of each group shared. stats: counts of
the stores looked at, the groups and their stores"""
st = stats if stats is not None else Counter()
for k in ("stores", "groups", "grouped_stores"):
st[k] = st.get(k, 0)
starts = [0] + [m.start() + 1 for m in MG_HEAD.finditer(text)]
starts.append(len(text))
out = []
for a, b in zip(starts, starts[1:]):
part = text[a:b]
if part.count("recomp_st") < 2 or len(MG_STORE.findall(part)) < 2 or (
"_ = c->x[" in part and MG_LOCAL.search(part)): # (no group possible: most functions)
out.append(part)
continue
out.append("\n".join(mark_groups_lines(part.split("\n"), st)))
return "".join(out)
def group_marks(paths, stats=None):
"""mark_groups over C files, in place (nx2/lift.py: the lift's last step). A64_MARK_EACH=1: none. The files changed"""
if os.environ.get("A64_MARK_EACH"):
return 0
changed = 0
for path in paths:
with open(path, encoding="utf-8", newline="") as f:
text = f.read()
new = mark_groups(text, stats)
if new != text:
with open(path, "w", encoding="utf-8", newline="") as f:
f.write(new)
changed += 1
return changed
# Dead flags. Each flag-setting instruction stores all of N, Z, C and V in the context, where the compiler must keep
# them: a callee or the caller could read them. The guest's ABI says otherwise (calls clobber NZCV, and nothing reads
# flags across a return), so emit_function computes per function which flags are live after each instruction and
# drops the stores of the dead ones (the computations go with them). A call (BL, BLR) clobbers all four; RET reads
# none; anything else that leaves the function (tail calls, which also join the pieces of a split function, indirect
# jumps, traps, SVC) reads all four. A64_KEEP_FLAGS=1 at lift time: every store kept.
FLAG_BIT = {"n": 8, "z": 4, "c": 2, "v": 1}
FLAG_WRITE = re.compile(r"c->([nzcv])\s*=(?!=)[^;]*;")
FLAG_READ = re.compile(r"c->([nzcv])\b(?!\s*=(?!=))")
COND_READS = [4, 4, 2, 2, 8, 8, 1, 1, 6, 6, 9, 9, 13, 13, 0, 0] # (EQ NE CS CC MI PL VS VC HI LS GE LT GT LE AL NV)
_flag_masks = {}
def flag_masks(text):
"""(flags read, flags written) by an instruction's C text; control flow inside it: reads all, kills none"""
m = _flag_masks.get(text)
if m is None:
if "goto" in text or "return" in text or "TAILCALL" in text:
m = (15, 0)
else:
w = 0
for f in FLAG_WRITE.findall(text):
w |= FLAG_BIT[f]
r = 0
for f in FLAG_READ.findall(text):
r |= FLAG_BIT[f]
m = (r, w)
_flag_masks[text] = m
return m
def drop_flag_stores(text, dead):
return FLAG_WRITE.sub(lambda m: "" if FLAG_BIT[m.group(1)] & dead else m.group(0), text)
# A64_LOCAL_REGS=<k> (experiment, lift time): a function's k most used guest registers (and SP) as C locals, written
# back to the context only where other code can see them. All of them (k >= 31) measured no gain on 10-21: the
# compiler spills them and the syncs stay; totk-pc-recomp (recomp/regs.py, MIT) found 4-6 the best over 300 hot
# functions (1.4.3 fight 13.5 -> 11.3 ms). The compiler had to store every changed register before each call and reload every
# register it reads after one. The guest's ABI needs less: x9-x15 are dead at calls and returns, and x19-x29 and SP come
# back from a call unchanged. emit_function marks each place code outside the function sees the registers, renames
# c->x[N] to xN_ and expands the marks: before a call or a return, every register but x9-x15 is written back; after a
# call x0-x18 and x30 are read again; around tail calls, indirect jumps, SVCs and traps everything (after an SVC, read
# again too). Functions with code spliced in (build.py SPLICE_AFTER) keep their registers in the context.
SYNC_CALL, SYNC_BACK, SYNC_RET, SYNC_ALL, SYNC_RELOAD = "\x01C%d\x01", "\x01B%d\x01", "\x01R%d\x01", "\x01A%d\x01", "\x01L%d\x01"
SYNC_TAIL = "\x01T%d\x01" # (control leaving after the instruction: what it wrote too)
SYNC_MARK = re.compile("\x01([CBRALT])(\\d+)\x01")
REG_REF = re.compile(r"c->x\[(\d+)\]")
# (a write of a guest register: its address taken (LDP's outputs), ++ / --, = and the compound assignments)
REG_WRITE = re.compile(r"(?:(?:&|\+\+\s*|--\s*)c->x\[(\d+)\])|(?:c->x\[(\d+)\]\s*(?:(?:[-+*/%&|^]|<<|>>)?=(?!=)|\+\+|--))")
SCRATCH = 0xFE00 # x9-x15: dead at calls and returns
_reg_writes = {}
def reg_writes(text):
"""the guest registers an instruction's C text may write (a mask)"""
m = _reg_writes.get(text)
if m is None:
m = 0
for a, b in REG_WRITE.findall(text):
m |= 1 << int(a or b)
_reg_writes[text] = m
return m
# A64_LOCAL_FLAGS=1 (experiment, lift time): N, Z, C and V as locals of each function (totk-pc-recomp keeps NZCV in a
# local: recomp/flags.py, MIT; 1.4.3 .text 342 -> 299 MiB). The ABI leaves the flags undefined across calls and
# returns (the dead-flag pass relies on it too), so they are written back only where control leaves otherwise: tail
# calls, indirect jumps, SVCs and traps (the SYNC_ALL / SYNC_TAIL marks), and read again after an SVC.
FLAG_REF = re.compile(r"(?<![\w>.])c->([nzcv])\b")
FLAG_STORE = "c->n = fn_; c->z = fz_; c->c = fc_; c->v = fv_; "
FLAG_LOAD = "fn_ = c->n; fz_ = c->z; fc_ = c->c; fv_ = c->v; "
def local_flags():
return bool(os.environ.get("A64_LOCAL_FLAGS"))
def local_regs():
"""A64_LOCAL_REGS: how many of a function's guest registers become locals (0: none)"""
v = os.environ.get("A64_LOCAL_REGS", "")
return int(v) if v.isdigit() else 0
def localize_regs(text, local, dirty):
"""Expand the context marks (see SYNC_CALL): with `local`, the registers become locals first, and each mark writes
back what may have changed since the last one (dirty: per instruction index, the masks before and after it) or
reads again."""
flags = local_flags() and FLAG_REF.search(text) is not None
if flags:
text = FLAG_REF.sub(r"f\1_", text)
fstore, fload = (FLAG_STORE, FLAG_LOAD) if flags else ("", "")
fdecl = "\n uint8_t fn_ = c->n, fz_ = c->z, fc_ = c->c, fv_ = c->v;" if flags else ""
if not local:
text = SYNC_MARK.sub(lambda m: fstore if m.group(1) in "AT" else fload if m.group(1) == "L" else "", text)
head, _, rest = text.partition("\n")
return head + fdecl + "\n" + rest
uses = {}
for n in REG_REF.findall(text):
uses[int(n)] = uses.get(int(n), 0) + 1
top = sorted((n for n in uses if n != 31), key=lambda n: (-uses[n], n))[:local_regs()]
regs = sorted(top + ([31] if 31 in uses else []))
chosen = set(regs)
text = REG_REF.sub(lambda m: "x%s_" % m.group(1) if int(m.group(1)) in chosen else m.group(0), text)
def store(mask):
return "".join("c->x[%d] = x%d_; " % (n, n) for n in regs if mask >> n & 1)
def load(mask):
return "".join("x%d_ = c->x[%d]; " % (n, n) for n in regs if mask >> n & 1)
def expand(m):
kind, i = m.group(1), int(m.group(2))
before, after = dirty[0][i], dirty[1][i]
if kind == "C": # a call: what it may read (x30 is set just before it)
return store((before | 1 << 30) & ~SCRATCH)
if kind == "R":
return store(before & ~SCRATCH)
if kind == "A":
return store(before) + fstore
if kind == "T":
return store(after) + fstore
if kind == "B": # after a call: x0-x18 and x30 (x19-x29 and SP come back as they were)
return load(0x4007FFFF)
return load(0xFFFFFFFF) + fload
text = SYNC_MARK.sub(expand, text)
head, _, rest = text.partition("\n")
decl = ("\n uint64_t " + ", ".join("x%d_ = c->x[%d]" % (n, n) for n in regs) + ";") if regs else ""
return head + decl + fdecl + "\n" + rest
# A64_PACKED_FLAGS=1 (experiment, lift time): the four NZCV stores of a flag-setting instruction become one 32-bit
# store (Ctx keeps n, z, c, v as adjacent bytes); byte stores to the context were ~5% of lifted-code samples.
# Measured 2026-10-21 (castle, 2 interleaved rounds): slower, game threads 39.6 -> 41.0 ms CPU a frame, 72.5 -> 71.1
# fps (exe -12 MB): the compiler no longer forwards a single flag from the packed store. Off.
FLAGS4 = re.compile(r"c->([zn])=([^;]+); ?c->([zn])=([^;]+); ?c->c=([^;]+); ?c->v=([^;]+);")
def pack_flags(text):
if "c->v=" not in text or not os.environ.get("A64_PACKED_FLAGS"):
return text
def sub(m):
a, ea, b, eb, ec, ev = m.groups()
if a == b or any("c->" in e for e in (ea, eb, ec, ev)):
return m.group(0)
en, ez = (ea, eb) if a == "n" else (eb, ea)
return "a64_set_nzcv(c,%s,%s,%s,%s);" % (en, ez, ec, ev)
return FLAGS4.sub(sub, text)
VSET_HEADER = r"""
/* the four flags in one store (pack_flags in a64_lift.py): n, z, c, v are adjacent bytes of the context */
static inline __attribute__((always_inline)) void a64_set_nzcv(Ctx *c, uint32_t n, uint32_t z, uint32_t cf, uint32_t v) {
const uint32_t w = n | z << 8 | cf << 16 | v << 24;
memcpy(&c->n, &w, 4);
}
/* ---- whole-register vector writes (widen_vstores in a64_lift.py): one 16-byte store, kept whole (A64_OPAQUE hides
the value from the compiler's store narrowing) ---- */
#if defined(__x86_64__) || defined(_M_X64)
#define A64_OPAQUE(x) __asm__("" : "+x"(x))
#else
#define A64_OPAQUE(x) ((void)0)
#endif
#if RECOMP_FPX_HOST == 2
static inline __attribute__((always_inline)) void a64_vst16(uint64_t *d, const void *s) {
__m128i x = _mm_loadu_si128((const __m128i *)s);
A64_OPAQUE(x);
_mm_storeu_si128((__m128i *)d, x);
}
static inline __attribute__((always_inline)) void a64_vset64(uint64_t *d, uint64_t v) {
__m128i x = _mm_set_epi64x(0, (long long)v);
A64_OPAQUE(x);
_mm_storeu_si128((__m128i *)d, x);
}
static inline __attribute__((always_inline)) void a64_vset128(uint64_t *d, uint64_t lo, uint64_t hi) {
__m128i x = _mm_set_epi64x((long long)hi, (long long)lo);
A64_OPAQUE(x);
_mm_storeu_si128((__m128i *)d, x);
}
static inline __attribute__((always_inline)) void a64_vst_lo64(uint64_t *d, __m128i r) { /* the low half, top zeroed */
r = _mm_move_epi64(r);
A64_OPAQUE(r);
_mm_storeu_si128((__m128i *)d, r);
}
#else
static inline void a64_vst16(uint64_t *d, const void *s) { memcpy(d, s, 16); }
static inline void a64_vset64(uint64_t *d, uint64_t v) { d[0] = v; d[1] = 0; }
static inline void a64_vset128(uint64_t *d, uint64_t lo, uint64_t hi) { d[0] = lo; d[1] = hi; }
#endif
static inline __attribute__((always_inline)) void a64_vset_lo(uint64_t *d, const void *s, int n) {
uint64_t v = 0;
memcpy(&v, s, n);
a64_vset64(d, v);
}
"""
VFCMP_HEADER = r"""
/* ---- vector float compares on SSE (vectorize_fcmp): 1 when done (no NaN lane; under FZ no subnormal lane) ---- */
static const uint64_t a64_vzero[2] = {0, 0}; /* (the operand of a compare against zero) */
/* scalar FCMP / FCMPE: the flags from one native compare; 0 for the exact block. (Byte stores, as everywhere: one
packed store of the four was slower, the compiler no longer forwarded single flags from it.) */
static inline __attribute__((always_inline)) int a64_fcmp_set(Ctx *c, uint32_t lt, uint32_t eq) {
c->n = (uint8_t)lt; /* lt: N, eq: Z, else C; V clear */
c->z = (uint8_t)eq;
c->c = (uint8_t)!lt;
c->v = 0;
return 1;
}
static inline __attribute__((always_inline)) int a64_fcmp32(Ctx *c, const void *pa, const void *pb) {
uint32_t a, b;
float fa, fb;
memcpy(&a, pa, 4);
memcpy(&b, pb, 4);
memcpy(&fa, &a, 4);
memcpy(&fb, &b, 4);
if (__builtin_isunordered(fa, fb)) return 0;
if ((c->fpcr & (1ULL << 24)) && ((a & 0x7fffffffu) - 1u < 0x7fffffu || (b & 0x7fffffffu) - 1u < 0x7fffffu)) return 0;
return a64_fcmp_set(c, fa < fb, fa == fb);
}
static inline __attribute__((always_inline)) int a64_fcmp64(Ctx *c, const void *pa, const void *pb) {
uint64_t a, b;
double fa, fb;
memcpy(&a, pa, 8);
memcpy(&b, pb, 8);
memcpy(&fa, &a, 8);
memcpy(&fb, &b, 8);
if (__builtin_isunordered(fa, fb)) return 0;
if ((c->fpcr & (1ULL << 24)) && ((a & 0x7fffffffffffffffULL) - 1u < 0xfffffffffffffULL ||
(b & 0x7fffffffffffffffULL) - 1u < 0xfffffffffffffULL)) return 0;
return a64_fcmp_set(c, fa < fb, fa == fb);
}
static inline __attribute__((always_inline)) int a64_vfcmp(Ctx *c, uint64_t *d, const uint64_t *n, const uint64_t *m, int k, int op) {
#if RECOMP_FPX_HOST == 2
const __m128i a = _mm_loadu_si128((const __m128i *)n), b = _mm_loadu_si128((const __m128i *)m);
const __m128i abs_mask = _mm_set1_epi32(0x7fffffff), inf = _mm_set1_epi32(0x7f800000);
const __m128i abs_a = _mm_and_si128(a, abs_mask), abs_b = _mm_and_si128(b, abs_mask);
__m128i bad = _mm_or_si128(_mm_cmpgt_epi32(abs_a, inf), _mm_cmpgt_epi32(abs_b, inf)); /* NaN */
if (c->fpcr & (1ULL << 24)) { /* FZ: a subnormal is flushed (and flagged) by the exact block */
const __m128i z = _mm_setzero_si128();
const __m128i sub_a = _mm_andnot_si128(_mm_cmpeq_epi32(_mm_and_si128(a, inf), inf), _mm_and_si128(_mm_cmpeq_epi32(_mm_and_si128(a, inf), z), _mm_cmpgt_epi32(abs_a, z)));
const __m128i sub_b = _mm_andnot_si128(_mm_cmpeq_epi32(_mm_and_si128(b, inf), inf), _mm_and_si128(_mm_cmpeq_epi32(_mm_and_si128(b, inf), z), _mm_cmpgt_epi32(abs_b, z)));
bad = _mm_or_si128(bad, _mm_or_si128(sub_a, sub_b));
}
if (_mm_movemask_epi8(bad) & (k == 4 ? 0xFFFF : 0x00FF)) return 0;
const __m128 fa = _mm_castsi128_ps(a), fb = _mm_castsi128_ps(b);
const __m128i r = _mm_castps_si128(op == 0 ? _mm_cmpge_ps(fa, fb) : op == 1 ? _mm_cmpgt_ps(fa, fb) : _mm_cmpeq_ps(fa, fb));
if (k == 4) _mm_storeu_si128((__m128i *)d, r); else a64_vst_lo64(d, r);
return 1;
#else
(void)c; (void)d; (void)n; (void)m; (void)k; (void)op;
return 0;
#endif
}
"""
def vectorize_fpx(text):
text = vectorize_fcmp(text)
if "a64_fpx_" not in text or os.environ.get("A64_NO_VFPX"):
return text
def add(m):
lanes = VFPX_LANE.findall(m.group(3))
ops = {op for _, _, op, _ in lanes}
if len(lanes) not in (2, 4) or len(ops) != 1 or \
any(not (int(i) == int(j) == int(r) == x) for x, (i, j, _, r) in enumerate(lanes)):
return m.group(0)
return "a64_vfpx_%s(c,c->vreg[%s],c->vreg[%s],c->vreg[%s],%d);" % (ops.pop()[:5], m.group(4), m.group(1),
m.group(2), len(lanes))
text = VFPX_ADD.sub(add, text)
return VFPX_MUL.sub(lambda m: "a64_vfpx_mul32(c,c->vreg[%s],c->vreg[%s],c->vreg[%s],4);"
% (m.group(3), m.group(1), m.group(2)), text)
# A64_LOCAL_FLAGS: the fast scalar FCMP as a code (0: the exact block; else 4 | lt | eq << 1), appended to a64_fpx.h
FCMPK_HEADER = r"""
static inline __attribute__((always_inline)) uint32_t a64_fcmp32k(Ctx *c, const void *pa, const void *pb) {
uint32_t a, b;
float fa, fb;
memcpy(&a, pa, 4);
memcpy(&b, pb, 4);
memcpy(&fa, &a, 4);
memcpy(&fb, &b, 4);
if (__builtin_isunordered(fa, fb)) return 0;
if ((c->fpcr & (1ULL << 24)) && ((a & 0x7fffffffu) - 1u < 0x7fffffu || (b & 0x7fffffffu) - 1u < 0x7fffffu)) return 0;
return 4u | (uint32_t)(fa < fb) | (uint32_t)(fa == fb) << 1;
}
static inline __attribute__((always_inline)) uint32_t a64_fcmp64k(Ctx *c, const void *pa, const void *pb) {
uint64_t a, b;
double fa, fb;
memcpy(&a, pa, 8);
memcpy(&b, pb, 8);
memcpy(&fa, &a, 8);
memcpy(&fb, &b, 8);
if (__builtin_isunordered(fa, fb)) return 0;
if ((c->fpcr & (1ULL << 24)) && ((a & 0x7fffffffffffffffULL) - 1u < 0xfffffffffffffULL ||
(b & 0x7fffffffffffffffULL) - 1u < 0xfffffffffffffULL)) return 0;
return 4u | (uint32_t)(fa < fb) | (uint32_t)(fa == fb) << 1;
}
"""
VFPX_HEADER = r"""
/* ---- 2/4-lane single-precision ops on SSE (vectorize_fpx in a64_lift.py): a lane kept only where the scalar fast
path would keep it (the same tests, on every lane); otherwise each lane goes through its scalar helper ---- */
#if RECOMP_FPX_HOST == 2
static inline __m128i a64_v_ult(__m128i x, uint32_t k) { /* unsigned x < k, per lane */
return _mm_cmplt_epi32(_mm_xor_si128(x, _mm_set1_epi32((int)0x80000000u)), _mm_set1_epi32((int)(k ^ 0x80000000u)));
}
static inline __m128i a64_v_abs(__m128i x) { return _mm_and_si128(x, _mm_set1_epi32(0x7fffffff)); }
static inline __m128i a64_v_big(__m128i t) { return a64_v_ult(_mm_sub_epi32(a64_v_abs(t), _mm_set1_epi32(0x00800001)), 0x7effffffu); }
static inline __m128i a64_v_zero(__m128i x) { return _mm_cmpeq_epi32(a64_v_abs(x), _mm_setzero_si128()); }
static inline __m128i a64_v_nonfin(__m128i t) { const __m128i e = _mm_set1_epi32(0x7f800000); return _mm_cmpeq_epi32(_mm_and_si128(t, e), e); }
static inline __m128i a64_v_subn(__m128i x) { return a64_v_ult(_mm_sub_epi32(_mm_slli_epi32(x, 1), _mm_set1_epi32(1)), 0x00ffffffu); }
/* lanes that overflowed (an infinite result of finite operands), kept once FPSR.OFC is raised (FPX_FZ_HEADER) */
static inline __m128i a64_v_ovf(const Ctx *c, __m128i a, __m128i b, __m128i t) {
if (!(c->fpsr & 4u)) return _mm_setzero_si128();
const __m128i inf = _mm_cmpeq_epi32(a64_v_abs(t), _mm_set1_epi32(0x7f800000));
return _mm_andnot_si128(_mm_or_si128(a64_v_nonfin(a), a64_v_nonfin(b)), inf);
}
#define A64_VFPX_SUM(name, op, scalar) \
static __attribute__((noinline, cold)) void a64_vfpx_##name##_lanes(Ctx *c, uint64_t *d, const uint64_t *n, const uint64_t *m, int k) { \
uint32_t a[4], b[4], r[4] = {0}; memcpy(a, n, 16); memcpy(b, m, 16); \
for (int j = 0; j < k; j++) r[j] = (uint32_t)scalar(c, a[j], b[j]); \
memcpy(d, r, 16); } \
static inline __attribute__((always_inline)) void a64_vfpx_##name(Ctx *c, uint64_t *d, const uint64_t *n, const uint64_t *m, int k) { \
if (RECOMP_LIKELY(RECOMP_FPX_OPEN(c))) { \
const __m128i a = _mm_loadu_si128((const __m128i *)n), b = _mm_loadu_si128((const __m128i *)m); \
const __m128i t = _mm_castps_si128(op(_mm_castsi128_ps(a), _mm_castsi128_ps(b))); \
__m128i bad = a64_v_nonfin(t), r = t; \
if (RECOMP_FPX_FZ(c)) { \
bad = _mm_or_si128(bad, _mm_or_si128(a64_v_subn(a), a64_v_subn(b))); \
if (c->fpsr & 8u) { /* (flags raised: a sum below the normal range, exact there, is a zero of its sign) */ \
const __m128i tiny = _mm_cmplt_epi32(a64_v_abs(t), _mm_set1_epi32(0x00800000)); \
r = _mm_or_si128(_mm_andnot_si128(tiny, t), _mm_and_si128(tiny, _mm_and_si128(t, _mm_set1_epi32((int)0x80000000u)))); \
} else { \
bad = _mm_or_si128(bad, _mm_andnot_si128(_mm_or_si128(a64_v_big(t), a64_v_zero(t)), _mm_set1_epi32(-1))); \
} \
} \
const int lanes = k == 4 ? 0xFFFF : 0x00FF; /* (overflowed lanes looked at only once a lane is out) */ \
if (RECOMP_LIKELY(!(_mm_movemask_epi8(bad) & lanes)) || \
!(_mm_movemask_epi8(_mm_andnot_si128(a64_v_ovf(c, a, b, t), bad)) & lanes)) { \
if (k == 4) _mm_storeu_si128((__m128i *)d, r); else a64_vst_lo64(d, r); \
return; } } \
a64_vfpx_##name##_lanes(c, d, n, m, k); }
#else
#define A64_VFPX_SUM(name, op, scalar) \
static __attribute__((noinline)) void a64_vfpx_##name(Ctx *c, uint64_t *d, const uint64_t *n, const uint64_t *m, int k) { \
uint32_t a[4], b[4], r[4] = {0}; memcpy(a, n, 16); memcpy(b, m, 16); \
for (int j = 0; j < k; j++) r[j] = (uint32_t)scalar(c, a[j], b[j]); \
memcpy(d, r, 16); }
#endif
"""
VFPX_MUL_HEADER = r"""
static __attribute__((noinline, cold)) void a64_vfpx_mul32_lanes(Ctx *c, uint64_t *d, const uint64_t *n, const uint64_t *m) {
uint32_t a[4], b[4], r[4]; memcpy(a, n, 16); memcpy(b, m, 16);
for (int j = 0; j < 4; j++) r[j] = (uint32_t)a64_fpx_mul32_fma32(c, a[j], b[j], (a[j] ^ b[j]) & 0x80000000u);
memcpy(d, r, 16); }
static inline __attribute__((always_inline)) void a64_vfpx_mul32(Ctx *c, uint64_t *d, const uint64_t *n, const uint64_t *m, int k) {
(void)k;
#if RECOMP_FPX_HOST == 2
if (RECOMP_LIKELY(RECOMP_FPX_OPEN(c))) {
const __m128i a = _mm_loadu_si128((const __m128i *)n), b = _mm_loadu_si128((const __m128i *)m);
const __m128i t = _mm_castps_si128(_mm_mul_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b)));
/* kept: big, or a finite product of a zero (RECOMP_FPX_KEEP32); under FZ no subnormal operand */
const __m128i keep = _mm_or_si128(a64_v_big(t), _mm_andnot_si128(a64_v_nonfin(t), _mm_or_si128(a64_v_zero(a), a64_v_zero(b))));
__m128i bad = _mm_andnot_si128(keep, _mm_set1_epi32(-1)), r = t;
if (RECOMP_FPX_FZ(c)) {
bad = _mm_or_si128(bad, _mm_or_si128(a64_v_subn(a), a64_v_subn(b)));
if (c->fpsr & 8u) { /* (flags raised: a tiny product is a zero of its sign; the smallest normal: exact) */
const __m128i abs_t = a64_v_abs(t), minn = _mm_set1_epi32(0x00800000);
const __m128i tiny = _mm_cmplt_epi32(abs_t, minn);
const __m128i fin = _mm_andnot_si128(a64_v_nonfin(t), _mm_set1_epi32(-1));
bad = _mm_andnot_si128(_mm_and_si128(tiny, fin), bad); /* (still bad if a subnormal operand) */
bad = _mm_or_si128(bad, _mm_or_si128(a64_v_subn(a), a64_v_subn(b)));
bad = _mm_or_si128(bad, _mm_cmpeq_epi32(abs_t, minn));
r = _mm_or_si128(_mm_andnot_si128(tiny, t), _mm_and_si128(tiny, _mm_and_si128(t, _mm_set1_epi32((int)0x80000000u))));
}
}
if (RECOMP_LIKELY(!_mm_movemask_epi8(bad)) || !_mm_movemask_epi8(_mm_andnot_si128(a64_v_ovf(c, a, b, t), bad))) {
_mm_storeu_si128((__m128i *)d, r);
return;
}
}
#endif
a64_vfpx_mul32_lanes(c, d, n, m);
}
"""
# Results past the fast path's range once their flags are already raised (TOTK's threads: FPSR = 0x9f, every cumulative
# flag set; FPCR.FZ = 1). FADD/FSUB S (a64_fpx_fz_add32 / a64_fpx_fz_sub32, OFC or UFC raised): an infinite result of
# finite operands is the native one, the round-to-nearest ±inf (~1M multiplies a second at the castle went through the
# exact body for it); under FZ with UFC raised a result below the normal range is the native one flushed to a zero of its
# sign. Sums are exact there (a tiny sum of normal numbers is a subnormal exactly). No subnormal operand (those flush
# first). FMUL/FNMUL/FMLA-family S: a64_fpx_sticky32 below.
FPX_FZ_HEADER = r"""
static inline __attribute__((always_inline)) int a64_fpx_fz_result(const Ctx *c, uint64_t a, uint64_t b, uint64_t t, uint64_t *v) {
const uint32_t m = (uint32_t)t & 0x7fffffffu;
if (m == 0x7f800000u) return (c->fpsr & 4u) && RECOMP_FPX_FIN32(a) && RECOMP_FPX_FIN32(b) ? (*v = t, 1) : 0;
if (!RECOMP_FPX_FZ(c) || !(c->fpsr & 8u) || m > 0x7f800000u || RECOMP_FPX_SUBN32(a) || RECOMP_FPX_SUBN32(b)) return 0;
if (m >= 0x00800000u) return (*v = t, 1);
*v = t & 0x80000000u;
return 1;
}
#if RECOMP_FPX_HOST
static inline __attribute__((always_inline)) int a64_fpx_fz_add32(const Ctx *c, uint64_t a, uint64_t b, uint64_t *v) {
return a64_fpx_fz_result(c, a, b, recomp_fpx_fb(recomp_fpx_f(a) + recomp_fpx_f(b)), v); }
static inline __attribute__((always_inline)) int a64_fpx_fz_sub32(const Ctx *c, uint64_t a, uint64_t b, uint64_t *v) {
return a64_fpx_fz_result(c, a, b, recomp_fpx_fb(recomp_fpx_f(a) - recomp_fpx_f(b)), v); }
#else
static inline int a64_fpx_fz_add32(const Ctx *c, uint64_t a, uint64_t b, uint64_t *v) { (void)c; (void)a; (void)b; (void)v; return 0; }
static inline int a64_fpx_fz_sub32(const Ctx *c, uint64_t a, uint64_t b, uint64_t *v) { (void)c; (void)a; (void)b; (void)v; return 0; }
#endif
/* FMUL/FNMUL/FMLA-family S past the fast path (the "sticky" rescue of the outlined wrappers a64_fpx_mul32_fma32 and
a64_fpx_fma32_fma32, inside their RECOMP_FPX_OPEN gate): one hardware multiply or fused multiply-add, kept when every
FPSR bit the Arm op could raise is already set and the value is the Arm one.
FZ: a subnormal operand is flushed to a zero of its sign (IDC). NaN: never. Infinite: exact with an infinite operand,
else an overflow (OFC). A zero product or an exact cancellation (a*b == -z in binary64, where the product is exact):
exact. Otherwise a result at most the smallest normal may be tiny (UFC): FZ clear, the IEEE gradual-underflow result is
Arm's; FZ set, below the smallest normal the exact value is tiny and Arm returns a zero of its sign, and exactly the
smallest normal is ambiguous (falls back). Rejected before computing: a NaN operand, a subnormal operand under FZ with
IDC clear, and FMUL of finite operands with neither OFC nor UFC set and nothing flushed (past the fast path, which keeps
a large product and a finite product of a zero, such a product is tiny or infinite). Without a hardware FMA
(RECOMP_FPX_FMA64; binary32 fmaf is the same instruction class) only the multiply is rescued. Returning 0 is always
sound: the wrapper then takes the exact body. */
#ifndef A64_FPX_STICKY_PROBE
#define A64_FPX_STICKY_PROBE() ((void)0) /* (tests/test_fpx_sticky.py counts the rescues the wrappers keep here) */
#endif
static inline __attribute__((always_inline)) int a64_fpx_sticky32(const Ctx *c, uint64_t a, uint64_t b, uint64_t z, int mul, uint64_t *v) {
#if RECOMP_FPX_HOST
const uint64_t s = c->fpsr;
uint32_t x = (uint32_t)a, y = (uint32_t)b, w = (uint32_t)z, t;
#if !RECOMP_FPX_FMA64
if (!mul) return 0;
#endif
if ((x & 0x7fffffffu) > 0x7f800000u || (y & 0x7fffffffu) > 0x7f800000u || (w & 0x7fffffffu) > 0x7f800000u) return 0;
if (RECOMP_FPX_FZ(c) && (RECOMP_FPX_SUBN32(x) || RECOMP_FPX_SUBN32(y) || (!mul && RECOMP_FPX_SUBN32(w)))) {
if (!(s & 0x80u)) return 0;
if (RECOMP_FPX_SUBN32(x)) x &= 0x80000000u;
if (RECOMP_FPX_SUBN32(y)) y &= 0x80000000u;
if (RECOMP_FPX_SUBN32(w)) w &= 0x80000000u;
} else if (mul && !(s & 0xCu) && RECOMP_FPX_FIN32(x) && RECOMP_FPX_FIN32(y)) {
return 0;
}
/* (FMUL: w is the zero of the product's sign, so x*y + w is x*y) */
#if RECOMP_FPX_FMA64
t = (uint32_t)recomp_fpx_fb(mul ? recomp_fpx_f(x) * recomp_fpx_f(y)
: __builtin_fmaf(recomp_fpx_f(x), recomp_fpx_f(y), recomp_fpx_f(w)));
#else
t = (uint32_t)recomp_fpx_fb(recomp_fpx_f(x) * recomp_fpx_f(y));
#endif
const uint32_t m = t & 0x7fffffffu;
if (m > 0x7f800000u) return 0;
if (m == 0x7f800000u) {
if (RECOMP_FPX_FIN32(x) && RECOMP_FPX_FIN32(y) && RECOMP_FPX_FIN32(w) && !(s & 4u)) return 0;
} else if (m <= 0x00800000u && !RECOMP_FPX_ZERO32(x) && !RECOMP_FPX_ZERO32(y)) {
if (!(s & 8u)) { /* UFC clear: only an exact cancellation (+0, no bit) is kept */
if (mul || m != 0 || (double)recomp_fpx_f(x) * (double)recomp_fpx_f(y) != -(double)recomp_fpx_f(w)) return 0;
} else if (RECOMP_FPX_FZ(c)) {
if (m == 0x00800000u) return 0;
t &= 0x80000000u;
}
}
*v = t;
return 1;
#else
(void)c; (void)a; (void)b; (void)z; (void)mul; (void)v; return 0;
#endif
}
"""
# Hot functions (A64_HOT_FUNCS=<file>: one function name a line, e.g. the hottest few hundred of a HOS_PROFILE_PCS or
# HOS_PROFILE_ORDER profile; ~70% of TOTK's lifted-code time is in its 300 hottest) call the scalar FP helpers' inline
# copy: a call makes the caller store and reload the guest registers it keeps in host registers around every float
# op; inlining the fast path everywhere made the exe 40% bigger and slower (RX_FPX_INLINE, build.py).
HOT_FPX = re.compile(r"\ba64_fpx_(?!hot_|fz_)(\w+)\(c,")
def hot_text(text):
"""a hot function's text: its FPX helper calls go through the copies with the fast path inline"""
return HOT_FPX.sub(r"a64_fpx_hot_\1(c,", text)
def hot_functions():
path = os.environ.get("A64_HOT_FUNCS")
if not path:
return set()
return {w for line in open(path) for w in line.split()[:1] if w.startswith("f_")}
# Profile-guided devirtualization (A64_DEVIRT=<file>: "site target" a line, hex, from scripts/devirt.py over an
# A64_SITE_PROFILE run): a BLR at a listed site compares its target with the one the profile saw there almost always and
# calls that function directly, so the ThinLTO link can inline it (C++ virtual calls: getters, small overrides); any
# other target still goes through A64_ICALL. Measured 2026-10-22 (TOTK castle, 3 interleaved rounds): ~100k indirect
# calls a frame, 7799 sites listed covering 88% of them; 73.6 -> 73.5 fps, game threads -0.3 ms CPU a frame (noise):
# a site's cached indirect call is already well predicted. Not on by default.
def devirt_sites():
path = os.environ.get("A64_DEVIRT")
if not path:
return {}
sites = {}
for line in open(path):
p = line.split()
if len(p) >= 2 and not line.startswith("#"):
sites[int(p[0], 16)] = int(p[1], 16)
return sites
def fpx_header(used):
out = ["/* Generated by a64_lift.py: one helper per FPX1 guard shape (see FPX_GUARD). A64_FPX_FN decides whether",
" the guard is inlined (fast code, slow compile) or called (default). */",
"#ifndef A64_FPX_FN", "#define A64_FPX_FN static __attribute__((noinline))", "#endif",
FPX_FZ_HEADER]
for fast, exact, nf, ne in sorted(used):
params = ", ".join("uint64_t a%d" % i for i in range(ne))
# past the fast path, still inside its gate (tested once): the rescues above, else the exact body
rescue = ""
if (fast, exact) in (("mul32", "fma32"), ("fma32", "fma32")):
rescue = "if (a64_fpx_sticky32(c, a0, a1, a2, %d, &v)) { A64_FPX_STICKY_PROBE(); return v; } " % (fast == "mul32")
elif fast in ("add32", "sub32"):
rescue = "if ((c->fpsr & 0xCu) && a64_fpx_fz_%s(c, a0, a1, &v)) return v; " % fast
out.append("A64_FPX_FN uint64_t a64_fpx_%s_%s(Ctx *c, %s) { uint64_t v; "
"if (RECOMP_FPX_OPEN(c)) { if (RECOMP_LIKELY(recomp_fpx_%s(%s, RECOMP_FPX_FZ(c), &v))) return v; %s} "
"return recomp_fpe_%s(c, %s); }" % (fast, exact, params, fast,
", ".join("a%d" % i for i in range(nf)), rescue, exact,
", ".join("a%d" % i for i in range(ne))))
# (the hot functions' copy, see HOT_FPX: the fast path inline, everything else through the helper)
out.append("static inline __attribute__((always_inline)) uint64_t a64_fpx_hot_%s_%s(Ctx *c, %s) { uint64_t v; "
"if (RECOMP_FPX_OPEN(c) && RECOMP_LIKELY(recomp_fpx_%s(%s, RECOMP_FPX_FZ(c), &v))) return v; "
"return a64_fpx_%s_%s(c, %s); }" % (fast, exact, params, fast, ", ".join("a%d" % i for i in range(nf)),
fast, exact, ", ".join("a%d" % i for i in range(ne))))
out.append(VSET_HEADER)
out.append(VFCMP_HEADER)
if any(e[:2] in (("add32", "add32"), ("sub32", "sub32"), ("mul32", "fma32")) for e in used):
out.append(VFPX_HEADER)
if ("add32", "add32", 2, 2) in used:
out.append("A64_VFPX_SUM(add32, _mm_add_ps, a64_fpx_add32_add32)")
if ("sub32", "sub32", 2, 2) in used:
out.append("A64_VFPX_SUM(sub32, _mm_sub_ps, a64_fpx_sub32_sub32)")
if ("mul32", "fma32", 2, 3) in used:
out.append(VFPX_MUL_HEADER)
return "\n".join(out) + "\n"
def op_text(text, fpx):
"""one instruction's emitted C as lifted code has it: FPX guards outlined (their shapes added to fpx, which
fpx_header turns into a64_fpx.h), NEON FP vectorized, stores widened, flags packed"""
return stack_stores(pack_flags(widen_vstores(vectorize_fpx(outline_fpx(text, fpx)))))
class Lifter:
def __init__(self, img, comments=False):
self.img = img
self.entries = set(img.entries) | set(img.hooks)
self.hooks = dict(img.hooks)
self.names = getattr(img, "names", {}) or {}
self.comments = comments
# (hookshot) C emitted right after the instruction at a guest address ({pc: text}): a game patch that adds to an
# instruction instead of replacing a function (nx2/oot_wide.py). Only after plain instructions (kind "op")
self.after = {}
self.funcs = {} # entry -> (body {pc: kind}, labels set)
self.jt = {} # br pc -> [targets]
self.jt_missing = {} # br pc -> its register, for BRs whose table address was not found
self.ijump = Counter()
self.unhandled = Counter()
self.plt = {} # stub entry -> final target
self.stats = Counter()
self.no_local = set() # functions whose registers stay in the context (A64_LOCAL_REGS; code spliced in)
# function starts from unwind info bound jump-table targets; without them, every entry does
self.eh_starts = sorted(getattr(img, "eh_starts", None) or self.entries)
self.devirt = devirt_sites() # BLR pc -> its usual target (A64_DEVIRT)
# ---- discovery ----------------------------------------------------------------------------------
def discover(self, max_insns=0):
for e in sorted(self.entries):
if e in self.hooks:
continue
self.funcs[e] = self.walk(e)
# A table found through another switch's edges needs those resolved first: retry the misses once with
# every function's tables known, and re-walk the functions that contain a newly resolved one.
missing, self.jt_missing = dict(self.jt_missing), {}
self.__dict__.pop("_preds", None)
redo = set()
for pc, reg in missing.items():
targets = self.jump_table(pc, reg)
if targets:
self.jt[pc] = targets
redo |= {e for e, (body, _) in self.funcs.items() if pc in body}
for e in redo:
self.funcs[e] = self.walk(e)
self.stats["jt_second_pass"] = len(missing) - len(self.jt_missing)
self.stats["jt_no_table"] = len(self.jt_missing)
if max_insns:
self.split_large(max_insns)
for e in self.funcs:
t = self.plt_target(e)
if t is not None:
self.plt[e] = t
self.find_setjmp_owners() # (after the PLT stubs: their callees are resolved through them)
SETJMP_NAMES = ("__setjmp", "setjmp", "_setjmp")
def find_setjmp_owners(self):
"""(hookshot) the functions that call __setjmp, with the call's return addresses: entries (walked here), so
a longjmp can run the owner's code after its setjmp (emit_function wraps the owner's callers)"""
self.setjmp_owner = {}
for e, (body, _) in list(self.funcs.items()):
for pc, (k, w) in body.items():
if k[0] == "bl" and self.names.get(self.callee(k[1])) in self.SETJMP_NAMES:
self.setjmp_owner.setdefault(e, []).append(pc + 4)
for rets in self.setjmp_owner.values():
for r in rets:
if r not in self.funcs:
self.entries.add(r)
self.funcs[r] = self.walk(r)
self.stats["setjmp_owners"] = len(self.setjmp_owner)
def split_large(self, max_insns):
"""Give functions longer than max_insns extra entries, about every max_insns / 2 instructions, only where
no branch of the function crosses the cut (so neither half re-walks code of the other). Extra entries are
always correct in this model: control reaching one tail-calls it. A cut trades one direct tail call for
compile time, which grows faster than linearly with function size at -O1 (TOTK's largest function is
18,156 instructions). Functions that are one big loop or switch cannot be cut and stay whole."""
new = set()
for e, (body, _) in self.funcs.items():
if len(body) <= max_insns:
continue
pcs = sorted(body)
diff = [0] * (len(pcs) + 1)
for pc, (k, _w) in body.items():
targets = [k[1]] if k[0] in ("b", "bcond", "cbz", "tbz") else self.jt.get(pc, []) if k[0] == "br" else []
for t in targets:
if t not in body or t == pc:
continue
if t > pc: # forward: a cut strictly between pc and t splits the edge
lo, hi = bisect.bisect_right(pcs, pc), bisect.bisect_left(pcs, t)
else: # backward: a cut in (t, pc] puts the branch after the cut and its target before
lo, hi = bisect.bisect_right(pcs, t), bisect.bisect_right(pcs, pc)
if lo < hi:
diff[lo] += 1
diff[hi] -= 1
step = max_insns // 2
last, cover = 0, 0
for i, pc in enumerate(pcs):
cover += diff[i]
if i - last >= step and cover == 0 and i and pcs[i - 1] == pc - 4:
new.add(pc)
last = i
new -= self.entries
if not new:
return
self.entries |= new
self.stats["split_entries"] += len(new)
affected = {e for e, (body, _) in self.funcs.items() if not new.isdisjoint(body.keys())}
for a in sorted(new | affected):
self.funcs[a] = self.walk(a)
def walk(self, entry):
img, entries = self.img, self.entries
body, labels, todo = {}, set(), [entry]
def visit(t):
if t != entry and t in entries:
return
if img.in_code(t):
labels.add(t)
todo.append(t)
while todo:
pc = todo.pop()
while pc not in body:
if (pc != entry and pc in entries) or not img.in_code(pc):
break
w = img.word(pc)
k = classify(w, pc)
body[pc] = (k, w)
kind = k[0]
if kind == "b":
visit(k[1])
break
if kind in ("bcond", "cbz", "tbz"):
visit(k[1])
elif kind == "br":
targets = self.jump_table(pc, k[1])
if targets:
self.jt[pc] = targets
self.add_jt_edges(pc, targets)
for t in targets:
visit(t)
break
elif kind in ("ret", "trap"):
break
pc += 4
return body, labels
# ---- jump tables ---------------------------------------------------------------------------------
def jump_table(self, br_pc, reg, window=32):
"""Symbolically run the straight-line code before a BR. Recognises LLVM's AArch64 tables: the target is
base + (entry << k), entry loaded from a constant table. Seen in TOTK:
- the table address hoisted out of the window (loop-invariant): found by scanning further back;
- the bound on a copy of the index (mov w8, w26 / and x12, x1, #0xff) or set by CCMP;
- no bound at all (index from cset/bfi): entries are read until one leaves the function.
- one register holding a different table on each path into a shared dispatch: every constant reaching
the br through the CFG is a candidate (reaching_constants).
Targets must lie in the containing function ([start, next .eh_frame function start))."""
img = self.img
R = {}
bound = None
bounds = {} # register -> number of table entries its value can index (cmp + b.hi/b.hs, and-mask, copies)
for pc in range(br_pc - 4 * window, br_pc, 4):
if not img.in_code(pc):
R.clear()
bounds.clear()
continue
w = img.word(pc)
if (w & 0xFC000000) == 0x14000000 or (w & 0xFFFFFC1F) in (0xD65F0000, 0xD61F0000):
# ret / b / br: what follows is reached from elsewhere, so nothing above is known there
R.clear()
bounds.clear()
continue
d = w & 31
n = (w >> 5) & 31
m = (w >> 16) & 31
# index bounds, tracked beside the values
if (w & 0x7F80001F) == 0x7100001F or (w & 0x7FE00C10) == 0x7A400800: # CMP / CCMP #imm
last_cmp = (n, (w >> 10) & 0xFFF if (w & 0x7F80001F) == 0x7100001F else (w >> 16) & 31)
bounds["cmp"] = last_cmp
elif (w & 0xFF00001F) in (0x54000008, 0x54000002) and "cmp" in bounds: # B.HI / B.HS
r, imm = bounds["cmp"]
bounds[r] = imm + 1 if (w & 15) == 8 else imm
elif (w & 0x7F800000) == 0x12000000 and not (w >> 16) & 63: # AND #(2^k - 1)
imms, nbit = (w >> 10) & 63, (w >> 22) & 1
if (nbit or not w >> 31) and imms < 31:
bounds[d] = 1 << (imms + 1)
else:
bounds.pop(d, None)
elif (w & 0x7FE0FFE0) == 0x2A0003E0: # MOV Wd/Xd, Wm/Xm
if m in bounds:
bounds[d] = bounds[m]
else:
bounds.pop(d, None)
elif d != 31 and self.writes(w, d):
bounds.pop(d, None)
if (w & 0x9F000000) == 0x90000000: # ADRP
imm = ((w >> 3) & 0x1FFFFC) | ((w >> 29) & 3)
R[d] = ("c", (pc & ~0xFFF) + (sx(imm, 21) << 12))
elif (w & 0x9F000000) == 0x10000000: # ADR
imm = ((w >> 3) & 0x1FFFFC) | ((w >> 29) & 3)
R[d] = ("c", pc + sx(imm, 21))
elif (w & 0xFF800000) == 0x91000000: # ADD Xd, Xn, #imm{, lsl 12}
v = R.get(n, ("reg", n))
imm = ((w >> 10) & 0xFFF) << (12 if w & (1 << 22) else 0)
if v and v[0] == "c":
R[d] = ("c", v[1] + imm)
elif v and v[0] == "reg" and d == n:
R.pop(d, None) # adrp before the window, add inside it: resolved at the br (reaching_constants)
elif v and v[0] == "reg" and len(v) == 2:
R[d] = ("reg", n, imm)
else:
R[d] = None
elif (w & 0x3B200C00) == 0x38200800 and not (w >> 26) & 1: # LDR* (register offset)
size, opc, S = w >> 30, (w >> 22) & 3, (w >> 12) & 1
v = R.get(n, ("reg", n)) # an undefined base: hoisted
R[d] = ("ld", v, m, 1 << size, opc >= 2, size if S else 0) if opc != 0 and v else None
elif (w & 0xFF200000) == 0x8B000000 and (w >> 22) & 3 == 0: # ADD Xd, Xn, Xm, LSL #k
R[d] = ("add", R.get(n), R.get(m), (w >> 10) & 63)
elif (w & 0xFFE00000) == 0x8B200000: # ADD Xd, Xn, Wm, <ext> #k
R[d] = ("add", R.get(n), R.get(m), (w >> 10) & 7)
elif (w & 0x7F80001F) == 0x7100001F: # CMP Wn/Xn, #imm
bound = {"imm": (w >> 10) & 0xFFF}
elif (w & 0x7FE00C10) == 0x7A400800: # CCMP Wn/Xn, #imm5, #nzcv, cond
bound = {"imm": (w >> 16) & 31}
elif (w & 0xFF00001F) in (0x54000008, 0x54000002) and bound and "count" not in bound: # B.HI / B.HS
bound["count"] = bound["imm"] + 1 if (w & 15) == 8 else bound["imm"]
elif self.writes(w, d):
# anything else that writes Xd forgets it (stores, compares, branches and SIMD ops do not)
R[d] = None
rt2 = (w >> 10) & 31
if ((w >> 27) & 7) == 5 and (w >> 22) & 1 and not (w >> 26) & 1: # LDP also writes Rt2
R[rt2] = None
bounds.pop(rt2, None)
v = R.get(reg)
if v and v[0] == "c": # br to a code constant: `adr x19, label ... br x19` (a continuation in a register)
lo, hi = self.function_extent(br_pc)
if lo <= v[1] < hi and img.in_code(v[1]):
self.stats["br_constant"] += 1
return [v[1]]
return None
if not v or v[0] != "add":
return self.adr_targets(br_pc, reg)
a, b, k = v[1], v[2], v[3]
if a and a[0] == "ld":
a, b = b, a
if not (a and a[0] == "c" and b and b[0] == "ld"):
self.stats["jt_unrecognized"] += 1
self.jt_missing[br_pc] = reg
return None
base, (_, table, idx, size, signed, _shift) = a[1], b
if isinstance(bounds.get(idx), int): # a bound on the very register that indexes the table wins
bound = {"count": bounds[idx]}
if table[0] == "reg":
# The table address was set outside the window. Every ADRP(+ADD) constant reaching the br through the
# function's CFG is a candidate (one register can hold a different table on each path into a shared
# dispatch), plus the address-order guesses; the switch takes the union of their targets.
reg_t = table[1]
off = table[2] if len(table) > 2 else 0 # add xT, xN, #off in the window, xN set before it
tables = set(self.reaching_constants(br_pc, reg_t))
guess = self.hoisted_constant(br_pc, reg_t) # from the br itself: the load does not write reg_t
if guess is None:
guess = self.function_constant(br_pc, reg_t)
if guess is not None:
tables.add(guess)
tables = {t + off for t in tables}
if not tables:
self.stats["jt_no_table"] += 1
self.jt_missing[br_pc] = reg
return None
if len(tables) > 1:
self.stats["jt_multi_table"] += 1
else:
tables = {table[1]}
lo, hi = self.function_extent(br_pc)
count = bound.get("count") if bound else None
if not (count and 0 < count <= 4096):
count = None
fmt = {1: "b", 2: "h", 4: "i", 8: "q"}[size]
fmt = fmt if signed else fmt.upper()
def read(table, limit, strict):
out = []
for i in range(limit):
try:
e = struct.unpack("<" + fmt, img.read(table + i * size, size))[0]
except ValueError:
break
t = (base + (e << k)) & 0xFFFFFFFFFFFFFFFF
if not (lo <= t < hi and img.in_code(t)):
if strict:
return None
break
out.append(t)
return out
targets, bounded = [], True
for table in sorted(tables):
got = read(table, count, True) if count else None
if got is None:
# no bound, or one that over-reads into the next table: read until an entry leaves the function
if count:
self.stats["jt_out_of_function"] += 1
got = read(table, 256 if size == 1 else 1024, False)
bounded = False
targets += [t for t in got if t not in targets]
if not targets:
return None
self.stats["jt_bounded" if bounded and count else "jt_unbounded"] += 1
return targets
def preds(self, lo, hi):
"""Predecessor map of the code in [lo, hi): fall-through, direct branches, and jump tables resolved so far.
Cached per function extent."""
cache = self.__dict__.setdefault("_preds", {})
if lo in cache:
return cache[lo]
img = self.img
p = {}
for pc in range(lo, hi, 4):
if not img.in_code(pc):
continue
k = classify(img.word(pc), pc)
kind = k[0]
if kind == "b":
succ = (k[1],)
elif kind in ("bcond", "cbz", "tbz"):
succ = (k[1], pc + 4)
elif kind == "br":
succ = tuple(self.jt.get(pc, ()))
elif kind in ("ret", "trap"):
succ = ()
else:
succ = (pc + 4,)
for s in succ:
p.setdefault(s, []).append(pc)
cache[lo] = p
return p
def add_jt_edges(self, br_pc, targets):
"""Keep a cached predecessor map current when a jump table of its function is resolved."""
cache = self.__dict__.get("_preds")
if cache:
lo, _ = self.function_extent(br_pc)
if lo in cache:
for t in targets:
cache[lo].setdefault(t, []).append(br_pc)
def reaching_constants(self, at, reg, limit=50000, depth=0):
"""Every ADRP (+ ADD #imm) constant that reaches `at` in `reg` along the function's CFG, walking back from
`at` until each path writes `reg`. A value copied (mov) or spilled to a stack slot and reloaded (ldr/ldp
from [sp, #off], LLVM's spill of a hoisted table address under register pressure) is followed to where it
was set. Paths that write something else, or a call that clobbers a caller-saved `reg`, contribute
nothing (the switch's default covers them)."""
return sorted(self._reaching(at, ("reg", reg), limit, depth))
def _reaching(self, at, loc, limit, depth):
img = self.img
lo, hi = self.function_extent(at)
if hi - lo > 0x40000 or depth > 4:
return set()
preds = self.preds(lo, hi)
found, seen, todo = set(), set(), [at]
kind_loc, x = loc
while todo and len(seen) < limit:
for p in preds.get(todo.pop(), ()):
if p in seen:
continue
seen.add(p)
w = img.word(p)
if kind_loc == "reg":
if classify(w, p)[0] in ("bl", "blr") and (x <= 18 or x == 30):
continue
if not self.writes(w, x):
todo.append(p)
continue
if (w & 0x9F000000) == 0x90000000: # ADRP
imm = ((w >> 3) & 0x1FFFFC) | ((w >> 29) & 3)
found.add((p & ~0xFFF) + (sx(imm, 21) << 12))
elif (w & 0xFF800000) == 0x91000000: # ADD Xreg, Xn, #imm
v = self.hoisted_constant(p, (w >> 5) & 31, 64)
if v is not None:
found.add(v + (((w >> 10) & 0xFFF) << (12 if w & (1 << 22) else 0)))
elif (w & 0xFFE0FFE0) == 0xAA0003E0: # MOV Xreg, Xm
found |= self._reaching(p, ("reg", (w >> 16) & 31), limit, depth + 1)
elif (w & 0xFFC003E0) == 0xF94003E0: # LDR Xreg, [sp, #off]
found |= self._reaching(p, ("slot", ((w >> 10) & 0xFFF) * 8), limit, depth + 1)
elif (w & 0xFFC003E0) == 0xA94003E0: # LDP X, X, [sp, #off]
off = sx((w >> 15) & 0x7F, 7) * 8 + (8 if (w & 31) != x else 0)
found |= self._reaching(p, ("slot", off), limit, depth + 1)
continue
# a stack slot: the store that last wrote it
if (w & 0xFFC003E0) == 0xF90003E0 and ((w >> 10) & 0xFFF) * 8 == x: # STR Xt, [sp, #off]
found |= self._reaching(p, ("reg", w & 31), limit, depth + 1)
continue
if (w & 0xFFC003E0) == 0xA90003E0: # STP Xt1, Xt2, [sp, #off]
off = sx((w >> 15) & 0x7F, 7) * 8
if x in (off, off + 8):
found |= self._reaching(p, ("reg", w & 31 if x == off else (w >> 10) & 31), limit, depth + 1)
continue
if ((w & 31) == 31 and (w & 0x7F800000) in (0x51000000, 0x11000000)) or ( # ADD/SUB sp, #imm
(w >> 5) & 31 == 31 and ((w & 0x3B200C00) in (0x38000C00, 0x38000400)
or (w & 0x3A800000) == 0x28800000)): # [sp] pre/post-index
continue # sp moved: slots differ
todo.append(p)
return found
@classmethod
def writes(cls, w, reg):
"""True if the instruction w may write X`reg`: Rd/Rt (bits 0-4) of anything but a store, compare or
branch, and Rt2 of a load pair."""
if ((w >> 25) & 7) == 7:
# SIMD & FP data processing: Rd is a V register, except FP<->integer conversions/moves and UMOV/SMOV
return (w & 31) == reg and ((w & 0x5F20FC00) == 0x1E200000 or (w & 0xBFE0DC00) == 0x0E002C00)
if (w & 0x0A000000) == 0x08000000 and (w >> 26) & 1:
return False # SIMD & FP load/store: Rt (and Rt2) are V registers
if (w & 31) == reg and not cls.reads_rt_only(w):
return True
return ((w >> 27) & 7) == 5 and (w >> 22) & 1 and not (w >> 26) & 1 and ((w >> 10) & 31) == reg
def function_constant(self, br_pc, reg):
"""The one ADRP(+ADD) constant the function ever assigns to `reg`, or None if it assigns none or several.
Catches table addresses hoisted far from their use, where a backward scan in address order runs into an
unrelated path's write first (an epilogue's `ldp x26, ...` after a `ret`)."""
img = self.img
lo, hi = self.function_extent(br_pc)
if hi - lo > 0x40000:
return None
page = {}
found = set()
for pc in range(lo, hi, 4):
w = img.word(pc)
d = w & 31
if (w & 0x9F000000) == 0x90000000: # ADRP
imm = ((w >> 3) & 0x1FFFFC) | ((w >> 29) & 3)
page[d] = (pc & ~0xFFF) + (sx(imm, 21) << 12)
elif (w & 0xFF800000) == 0x91000000 and ((w >> 5) & 31) in page: # ADD Xd, Xn, #imm
v = page[(w >> 5) & 31] + (((w >> 10) & 0xFFF) << (12 if w & (1 << 22) else 0))
if d == reg:
found.add(v)
return found.pop() if len(found) == 1 else None
def adr_targets(self, br_pc, reg):
"""Every `adr xreg, label` into the same function: the candidate targets of a `br xreg` whose value the
straight-line window could not pin down (a continuation set on several paths). The switch keeps a
default case, so a candidate list that misses one still falls back to an indirect jump."""
img = self.img
lo, hi = self.function_extent(br_pc)
if hi - lo > 0x40000:
return None
targets = []
for pc in range(lo, hi, 4):
w = img.word(pc)
if (w & 0x9F00001F) == (0x10000000 | reg):
imm = ((w >> 3) & 0x1FFFFC) | ((w >> 29) & 3)
t = pc + sx(imm, 21)
if lo <= t < hi and img.in_code(t):
targets.append(t)
if targets:
self.stats["br_adr_candidates"] += 1
return targets or None
def hoisted_constant(self, before, reg, scan=1024):
"""The constant ADRP (+ ADD) last wrote to `reg` before address `before`, scanning back in address order.
Any other write to `reg` on the way makes it unknown. Address order is a heuristic for 'earlier': it is
how LLVM lays out the hoisted setup of a loop-invariant table address."""
img = self.img
pc = before - 4
for _ in range(scan):
if not img.in_code(pc):
return None
w = img.word(pc)
if (w & 31) == reg and not self.reads_rt_only(w):
if (w & 0x9F000000) == 0x90000000: # ADRP Xreg, page
imm = ((w >> 3) & 0x1FFFFC) | ((w >> 29) & 3)
return (pc & ~0xFFF) + (sx(imm, 21) << 12)
if (w & 0xFF800000) == 0x91000000: # ADD Xreg, Xn, #imm
imm = ((w >> 10) & 0xFFF) << (12 if w & (1 << 22) else 0)
v = self.hoisted_constant(pc, (w >> 5) & 31, 64)
return None if v is None else v + imm
return None
pc -= 4
return None
@staticmethod
def reads_rt_only(w):
"""True for encodings whose bits 0-4 are not a destination: stores, compares, branches."""
return ((w & 0x3A000000) == 0x38000000 and not (w >> 22) & 1) or (w & 0x1C000000) == 0x14000000 \
or (w & 0x7E000000) in (0x34000000, 0x36000000) \
or (((w >> 27) & 7) == 5 and not (w >> 22) & 1) # STP / STNP
def function_extent(self, pc):
starts = self.eh_starts
i = bisect.bisect_right(starts, pc)
lo = starts[i - 1] if i else 0
hi = starts[i] if i < len(starts) else 1 << 64
return lo, hi
# ---- PLT stubs -----------------------------------------------------------------------------------
def plt_target(self, e):
img = self.img
try:
w0, w1, w2, w3 = (img.word(e + 4 * i) for i in range(4))
except Exception:
return None
if (w0 & 0x9F00001F) != 0x90000010 or (w1 & 0xFFC003FF) != 0xF9400211 or (w2 & 0xFFC003FF) != 0x91000210 \
or w3 != 0xD61F0220:
return None
imm = ((w0 >> 3) & 0x1FFFFC) | ((w0 >> 29) & 3)
slot = (e & ~0xFFF) + (sx(imm, 21) << 12) + ((w1 >> 10) & 0xFFF) * 8
t = struct.unpack("<Q", img.read(slot, 8))[0]
return (t, slot) if t in self.entries else None
def callee(self, t):
seen = 0
while t in self.plt and seen < 4:
t = self.plt[t][0]
seen += 1
return t
def fname(self, t):
# Hooked addresses keep their f_ name too (a64_hooks.c defines it as a tail call of the hook), so hooking
# or unhooking a function changes one small file instead of every call site in the lifted code.
return "f_%08x" % t
# ---- emission ------------------------------------------------------------------------------------
def transfer(self, pc, t, labels, body):
"""C for control reaching t from an instruction at pc (self.cur_i: its index, for the context mark)."""
if t in body and (t not in self.entries or t == self.cur):
return "goto L_%08x;" % t
if t in self.entries:
name = self.fname(self.callee(t))
self.refs.add(name)
return SYNC_TAIL % self.cur_i + "TAILCALL(%s);" % name
self.stats["bad_target"] += 1
return SYNC_TAIL % self.cur_i + "a64_trap(c, 0x%xULL, 0); return;" % t
def dead_flags(self, body, pcs, texts):
"""pc -> the flags an instruction stores that nothing reads (FLAG_BIT bits; see FLAG_WRITE)"""
e = self.cur
def inner(t): # the instruction a branch to t goes to here (a goto), or None: it leaves the function
return t if t in body and (t not in self.entries or t == e) else None
n = len(pcs)
at = {pc: i for i, pc in enumerate(pcs)}
reads, kills, succ, exits = [0] * n, [0] * n, [()] * n, [0] * n
for i, pc in enumerate(pcs):
k, w = body[pc]
kind = k[0]
targets, falls = [], True
if kind == "op":
text, fl = texts[(w, pc) if pc_relative(w) else w]
if fl & a64emit.UNHANDLED:
reads[i], falls = 15, False
else:
reads[i], kills[i] = flag_masks(text)
elif kind == "b":
targets, falls = [k[1]], False
elif kind == "bcond":
reads[i], targets = COND_READS[k[2]], [k[1]]
elif kind in ("cbz", "tbz"):
targets = [k[1]]
elif kind in ("bl", "blr"):
if kind == "bl" and self.callee(k[1]) not in self.entries:
reads[i], falls = 15, False # (a trap)
else:
kills[i] = 15 # a call clobbers NZCV
elif kind == "ret":
falls = False
elif kind == "br":
targets, falls = list(self.jt.get(pc) or ()), False
exits[i] = 15 # (the switch's default: an indirect jump)
elif kind in ("svc", "trap"):
reads[i] = 15
falls = kind == "svc"
if falls:
targets.append(pc + 4)
s = []
for t in targets:
t2 = inner(t)
if t2 is None:
exits[i] = 15
else:
s.append(at[t2])
succ[i] = tuple(s)
live_in = [0] * n
changed = True
while changed:
changed = False
for i in range(n - 1, -1, -1):
out = exits[i]
for j in succ[i]:
out |= live_in[j]
v = reads[i] | (out & ~kills[i])
if v != live_in[i]:
live_in[i] = v
changed = True
dead = {}
for i, pc in enumerate(pcs):
if kills[i] and body[pc][0][0] == "op":
out = exits[i]
for j in succ[i]:
out |= live_in[j]
if kills[i] & ~out:
dead[pc] = kills[i] & ~out
return dead
def dirty_regs(self, body, pcs, texts):
"""A64_LOCAL_REGS: per instruction index, the registers that may differ from the context before it and after it
(written since the function's entry or the last call or SVC, which write back and read again)"""
e = self.cur
def inner(t):
return t if t in body and (t not in self.entries or t == e) else None
n = len(pcs)
at = {pc: i for i, pc in enumerate(pcs)}
writes, sync, succ = [0] * n, [False] * n, [()] * n
for i, pc in enumerate(pcs):
k, w = body[pc]
kind = k[0]
targets, falls = [], True
if kind == "op":
text, fl = texts[(w, pc) if pc_relative(w) else w]
if fl & a64emit.UNHANDLED:
falls = False
else:
writes[i] = reg_writes(text)
elif kind == "b":
targets, falls = [k[1]], False
elif kind in ("bcond", "cbz", "tbz"):
targets = [k[1]]
elif kind in ("bl", "blr", "svc"):
sync[i] = True
elif kind in ("ret", "trap"):
falls = False
elif kind == "br":
targets, falls = list(self.jt.get(pc) or ()), False
if falls:
targets.append(pc + 4)
succ[i] = tuple(at[t2] for t2 in (inner(t) for t in targets) if t2 is not None)
dirty_in = [0] * n
changed = True
while changed:
changed = False
for i in range(n):
out = 0 if sync[i] else dirty_in[i] | writes[i]
for j in succ[i]:
if out & ~dirty_in[j]:
dirty_in[j] |= out
changed = True
return dirty_in, [0 if sync[i] else dirty_in[i] | writes[i] for i in range(n)]
def emit_function(self, e, texts):
"""C text of one function, and the set of C names it calls."""
body, labels = self.funcs[e]
self.cur = e
self.refs = set()
out = ["void f_%08x(Ctx *__restrict A64_NOESC c) {" % e]
if e in self.names and self.comments:
out[0] = "/* %s */\n" % self.names[e] + out[0]
if e in self.plt: # a PLT stub reached indirectly: x16 = GOT slot, x17 = target, as the stub leaves them
t, slot = self.plt[e]
name = self.fname(self.callee(t))
self.refs.add(name)
out.append("c->x[16] = 0x%xULL; c->x[17] = 0x%xULL; TAILCALL(%s);\n}" % (slot, t, name))
return "\n".join(out) + "\n", self.refs
pcs = sorted(body)
dead = {} if os.environ.get("A64_KEEP_FLAGS") else self.dead_flags(body, pcs, texts)
frame = not os.environ.get("A64_MARK_FRAME")
local = local_regs() and e not in self.no_local
dirty = self.dirty_regs(body, pcs, texts) if local else None
for i, pc in enumerate(pcs):
self.cur_i = i
(k, w) = body[pc]
kind = k[0]
if pc in labels or (pc == e and e in labels):
out.append("L_%08x:;" % pc)
if self.comments:
out.append("/* %08x: %08x */" % (pc, w))
nxt = pc + 4
falls = True
if kind == "op":
text, flags = texts[(w, pc) if pc_relative(w) else w]
if flags & a64emit.UNHANDLED:
self.unhandled[w] += 1
out.append(SYNC_ALL % i + "a64_trap(c, 0x%xULL, 0x%08xu); return;" % (pc, w))
falls = False
elif flags & a64emit.TERMINATOR:
raise SystemExit("a64emit closed a block at %#x (%08x): classify() must handle it" % (pc, w))
else:
if frame and not fp_is_frame(text):
frame = False
if pc in dead:
text = drop_flag_stores(text, dead[pc])
self.stats["dead_flag_stores"] += bin(dead[pc]).count("1")
out.append(text.rstrip("\n"))
if pc in self.after:
out.append(self.after[pc])
self.stats["after_patches"] = self.stats.get("after_patches", 0) + 1
elif kind == "b":
out.append(self.transfer(pc, k[1], labels, body))
falls = False
elif kind == "bcond":
out.append("if %s { %s }" % (COND[k[2]], self.transfer(pc, k[1], labels, body)))
elif kind == "cbz":
_, t, rt, sf, nz = k
v = "0" if rt == 31 else ("c->x[%d]" % rt if sf else "(uint32_t)c->x[%d]" % rt)
out.append("if (%s %s 0) { %s }" % (v, "!=" if nz else "==", self.transfer(pc, t, labels, body)))
elif kind == "tbz":
_, t, rt, bit, nz = k
v = "0" if rt == 31 else "((c->x[%d] >> %d) & 1)" % (rt, bit)
out.append("if (%s%s) { %s }" % ("" if nz else "!", v, self.transfer(pc, t, labels, body)))
elif kind == "bl":
t = self.callee(k[1])
if t in self.entries and t in getattr(self, "setjmp_owner", {}):
# (hookshot) a call of a function that calls __setjmp: this frame takes a host setjmp. A guest
# longjmp's BR to one of the owner's setjmp return addresses comes back here (host/hos_setjmp.c),
# with the registers it restored, and runs the owner's code from there, as the owner would.
# (recomp_sj_setjmp, not __builtin_setjmp: on Win64 clang's frame pointer sits above the frame
# address the builtin saves, and its longjmp restored rbp to that, so the frame's spill slots
# were read 128 bytes off: c and _sj came back as garbage)
rets = (self.setjmp_owner[t] + [0, 0, 0, 0])[:4]
out.append("c->x[30] = 0x%xULL; %s{ void **_sj = recomp_sj_push(c, 0x%xULL, 0x%xULL, 0x%xULL, 0x%xULL); "
"if (recomp_sj_setjmp(_sj) == 0) %s(c); else a64_icall(c, recomp_sj_target(_sj)); "
"recomp_sj_pop(_sj); }%s"
% (nxt, SYNC_CALL % i, rets[0], rets[1], rets[2], rets[3], self.fname(t), SYNC_BACK % i))
self.refs.add(self.fname(t))
self.stats["setjmp_call_sites"] = self.stats.get("setjmp_call_sites", 0) + 1
elif t in self.entries:
out.append("c->x[30] = 0x%xULL; %s%s(c);%s" % (nxt, SYNC_CALL % i, self.fname(t), SYNC_BACK % i))
self.refs.add(self.fname(t))
else:
self.stats["bad_target"] += 1
out.append(SYNC_ALL % i + "a64_trap(c, 0x%xULL, 0x%08xu); return;" % (pc, w))
elif kind == "blr":
usual = self.devirt.get(pc)
if usual in self.entries:
name = self.fname(self.callee(usual))
self.refs.add(name)
self.stats["devirt_sites"] += 1
out.append("{ uint64_t _t = c->x[%d]; c->x[30] = 0x%xULL; %sif (RECOMP_LIKELY(_t == 0x%xULL)) %s(c); "
"else A64_ICALL(c, _t); %s}" % (k[1], nxt, SYNC_CALL % i, usual, name, SYNC_BACK % i))
else:
out.append("{ uint64_t _t = c->x[%d]; c->x[30] = 0x%xULL; %sA64_ICALL(c, _t); %s}"
% (k[1], nxt, SYNC_CALL % i, SYNC_BACK % i))
elif kind == "ret":
out.append(SYNC_RET % i + "return;")
falls = False
elif kind == "br":
targets = self.jt.get(pc)
if targets:
cases = " ".join("case 0x%xULL: %s" % (t, self.transfer(pc, t, labels, body))
for t in sorted(set(targets)))
out.append("switch (c->x[%d]) { %s default: %sA64_IJUMP(c, c->x[%d]); return; }"
% (k[1], cases, SYNC_ALL % i, k[1]))
else:
self.ijump[pc] += 1
out.append(SYNC_ALL % i + "A64_IJUMP(c, c->x[%d]); return;" % k[1])
falls = False
elif kind == "svc":
out.append(SYNC_ALL % i + "a64_svc(c, %du);" % k[1] + SYNC_RELOAD % i)
elif kind == "trap":
out.append(SYNC_ALL % i + "a64_trap(c, 0x%xULL, 0x%08xu); return;" % (pc, w))
falls = False
elif kind == "isb":
out.append("recomp_barrier();")
elif kind == "ic":
out.append("/* ic ivau: no self-modifying code */")
if falls and (i + 1 == len(pcs) or pcs[i + 1] != nxt):
out.append(self.transfer(pc, nxt, labels, body))
out.append("}")
text = "\n".join(out)
if frame:
text = frame_stores(text)
return localize_regs(text, local, dirty) + "\n", self.refs
def write(self, out_dir, chunk_bytes=CHUNK_BYTES):
os.makedirs(out_dir, exist_ok=True)
# translate every distinct instruction once (per pc only where the result depends on it)
keys = {}
for body, _ in self.funcs.values():
for pc, (k, w) in body.items():
if k[0] == "op":
keys.setdefault((w, pc) if pc_relative(w) else w, (pc, w))
order = list(keys)
tx, fl = a64emit.translate([keys[x] for x in order])
fpx = set()
texts = dict(zip(order, zip((op_text(t, fpx) for t in tx), fl)))
del tx, fl, keys, order
with open(os.path.join(out_dir, "a64_fpx.h"), "w", newline="\n") as f:
f.write(fpx_header(fpx))
if local_flags():
f.write(FCMPK_HEADER)
chunk, size, n, decls, files = [], 0, 0, set(), []
def flush():
nonlocal chunk, size, n, decls
if not chunk:
return
path = os.path.join(out_dir, "code_%03d.c" % n)
with open(path, "w", newline="\n") as f:
f.write('#include "a64.h"\n')
text = "".join(chunk)
if "recomp_sj_setjmp(" in text:
f.write("__attribute__((returns_twice)) int recomp_sj_setjmp(void **slot);\n")
f.write("".join("void %s(Ctx *A64_NOESC);\n" % d for d in sorted(decls)))
f.write(text)
files.append(path)
chunk, size, n, decls = [], 0, n + 1, set()
hot = hot_functions()
# A64_HOT_CHUNK=1 (with A64_HOT_FUNCS): the hot functions in a file of their own, code_hot.c, so that the
# compiler sees the hot callees' bodies where the hot callers are and can inline them (build.py compiles it
# with RX_HOT_CFLAGS)
hot_chunk = [] if hot and os.environ.get("A64_HOT_CHUNK") else None
hot_decls = set()
nhot = 0
for e in sorted(self.funcs):
text, refs = self.emit_function(e, texts)
if hot and self.fname(e) in hot:
text = hot_text(text)
nhot += 1
if hot_chunk is not None:
hot_chunk.append(text)
hot_decls.update(refs)
continue
decls.update(refs)
chunk.append(text)
size += len(text)
if size >= chunk_bytes:
flush()
flush()
if hot_chunk:
path = os.path.join(out_dir, "code_hot.c")
with open(path, "w", newline="\n") as f:
f.write('#include "a64.h"\n')
text = "".join(hot_chunk)
if "recomp_sj_setjmp(" in text:
f.write("__attribute__((returns_twice)) int recomp_sj_setjmp(void **slot);\n")
f.write("".join("void %s(Ctx *A64_NOESC);\n" % d for d in sorted(hot_decls)))
f.write(text)
files.append(path)
elif os.path.exists(os.path.join(out_dir, "code_hot.c")):
os.remove(os.path.join(out_dir, "code_hot.c")) # (a lift without the hot file: no stale copy linked twice)
if hot:
print("hot functions (A64_HOT_FUNCS): %d of %d listed found%s" % (nhot, len(hot),
", in code_hot.c" if hot_chunk else ""))
self._write_table(out_dir)
with open(os.path.join(out_dir, "functions.txt"), "w") as f:
for e in sorted(self.funcs):
f.write("%08x %d\n" % (e, len(self.funcs[e][0])))
return files
def _write_table(self, out_dir):
fns = sorted(set(self.funcs) | set(self.hooks))
with open(os.path.join(out_dir, "a64_table.c"), "w", newline="\n") as f:
f.write('#include "a64.h"\n')
f.write("".join("void %s(Ctx *A64_NOESC);\n" % self.fname(a) for a in fns))
f.write("const uint32_t a64_fn_count = %d;\n" % len(fns))
f.write("const uint32_t a64_fn_addr[] = {\n%s};\n" % "".join("0x%08x,\n" % a for a in fns))
f.write("const A64Fn a64_fn_ptr[] = {\n%s};\n" % "".join("%s,\n" % self.fname(a) for a in fns))
with open(os.path.join(out_dir, "a64_hooks.c"), "w", newline="\n") as f:
f.write("/* Hooked guest addresses: f_<address> is the host function (an HLE or a generated stub). */\n")
f.write('#include "a64.h"\n')
f.write("".join("void %s(Ctx *A64_NOESC);\n" % n for n in sorted(set(self.hooks.values()))))
f.write("".join("void f_%08x(Ctx *A64_NOESC c) { TAILCALL(%s); }\n" % (a, n) for a, n in sorted(self.hooks.items())))
def report(self):
nins = sum(len(b) for b, _ in self.funcs.values())
br = sum(1 for b, _ in self.funcs.values() for k, _ in b.values() if k[0] == "br")
print("functions %d, instructions %d (with shared tails), jump tables %d, indirect BR %d of %d, "
"PLT stubs %d, unhandled words %d (%d sites), bad targets %d" % (
len(self.funcs), nins, len(self.jt), len(self.ijump), br, len(self.plt),
len(self.unhandled), sum(self.unhandled.values()), self.stats["bad_target"]))
print("jump tables: %d bounded, %d bounded by the function, %d rejected (target outside), %d table not found" % (
self.stats["jt_bounded"], self.stats["jt_unbounded"], self.stats["jt_out_of_function"],
self.stats["jt_no_table"]))
if self.stats["jt_unrecognized"]:
print("table-shaped BRs whose load was not recognized: %d" % self.stats["jt_unrecognized"])
if self.stats["jt_multi_table"]:
print("jump tables with several candidate table addresses: %d" % self.stats["jt_multi_table"])
print("br to a code constant: %d; br with adr candidates: %d" % (self.stats["br_constant"],
self.stats["br_adr_candidates"]))
if self.stats["split_entries"]:
print("split: %d extra entries in long functions; largest function now %d instructions" % (
self.stats["split_entries"], max(len(b) for b, _ in self.funcs.values())))
if self.stats["stores"]: # (group_marks, when the lift ran it before this report)
print("page marks: %d groups share one mark for %d of %d stores through a base register" % (
self.stats["groups"], self.stats["grouped_stores"], self.stats["stores"]))