"""Every instruction form a game uses, run in Unicorn and as lifted C, compared (Rexcomp's test_insns.py method). Samples real encodings from a game's code (up to --per encodings per mnemonic form, NEON keyed by arrangement), gives each --states random register states (FPCR 0 and FPCR.FZ, as TOTK runs), runs one instruction in Unicorn's AArch64 and in C from a64emit + rt/a64.h, and compares X0-X30, SP, NZCV, FPSR, V0-V31 and the scratch memory. Memory operands get a base register inside the scratch area and a small index, so loads and stores are tested too. Usage: python insn_test.py [--modules main,sdk] [--per 6] [--states 12] [--seed 1] [--fpsr] (also compare FPSR cumulative flags) [--list] (show every failing form) Needs: unicorn, capstone, emsdk clang (native). Builds in a temp folder. """ import argparse import collections import hashlib import json import os import random import struct import subprocess import sys import tempfile import capstone import unicorn from unicorn import arm64_const as A HERE = os.path.dirname(os.path.abspath(__file__)) KIT = os.path.join(HERE, "..") sys.path.insert(0, os.path.join(KIT, "lift")) sys.path.insert(0, os.path.join(KIT, "tools")) import a64emit, a64_lift, a64_rtgen, toolchain CODE = 0x10000 SCRATCH = 0x100000 SCRATCH_SIZE = 0x10000 MEM_SIZE = SCRATCH + SCRATCH_SIZE FZ = 1 << 24 XREG = [getattr(A, "UC_ARM64_REG_X%d" % i) for i in range(29)] + [A.UC_ARM64_REG_X29, A.UC_ARM64_REG_X30] QREG = [getattr(A, "UC_ARM64_REG_Q%d" % i) for i in range(32)] STATE = struct.Struct("<32Q4B4xQQ64Q") # x0-x30, sp | n z c v | fpcr fpsr | v0-v31 (lo, hi) def sample(image_dir, modules, per, rng): md = capstone.Cs(capstone.CS_ARCH_ARM64, capstone.CS_MODE_ARM) md.detail = True groups = collections.defaultdict(list) for mod in modules: meta = json.load(open(os.path.join(image_dir, mod + ".json"))) img = open(os.path.join(image_dir, mod + ".bin"), "rb").read() t = next(s for s in meta["segments"] if s["name"] == ".text") words = set(struct.unpack_from("<%dI" % (t["size"] // 4), img, t["offset"])) for w in words: if a64_lift.classify(w, CODE)[0] != "op": continue ins = next(md.disasm(struct.pack("> 31) & 1, (f >> 30) & 1, (f >> 29) & 1, (f >> 28) & 1] ov = [] for i in range(32): q = uc.reg_read(QREG[i]) ov += [q & (2 ** 64 - 1), q >> 64] return (ox, on, uc.reg_read(A.UC_ARM64_REG_FPCR), uc.reg_read(A.UC_ARM64_REG_FPSR), ov, hashlib.sha1(bytes(uc.mem_read(SCRATCH, SCRATCH_SIZE))).digest()), None def pack(state): x, nzcv, fpcr, fpsr, v = state return STATE.pack(*(x + nzcv + [fpcr, fpsr] + v)) def build_harness(tmp, picked): a64_rtgen.write(tmp) texts, flags = a64emit.translate([(CODE, w) for _, w in picked]) used = set() texts = [a64_lift.outline_fpx(t, used) for t in texts] # the same FPX helpers the lifter emits open(os.path.join(tmp, "a64_fpx.h"), "w", newline="\n").write(a64_lift.fpx_header(used)) src = ['#include "a64.h"', "#include ", "#include ", "#include "] for i, ((key, w), t, f) in enumerate(zip(picked, texts, flags)): body = "c->halted = 7;" if f else t src.append("static void t%d(Ctx *__restrict c) { %s }" % (i, body)) src.append("static void (*const T[])(Ctx *) = {%s};" % ",".join("t%d" % i for i in range(len(picked)))) src.append(r""" void a64_trap(Ctx *c, uint64_t pc, uint32_t insn) { (void)pc; (void)insn; c->halted = 9; } void recomp_unhandled(GuestContext *c, uint32_t i, uint64_t pc) { (void)i; (void)pc; c->halted = 7; } void recomp_barrier(void) {} void recomp_ic_ivau(GuestContext *c, uint64_t a, int v) { (void)c; (void)a; (void)v; } uint64_t recomp_cntpct(GuestContext *c) { (void)c; return 0; } static uint64_t ex_addr; static int ex_valid; uint64_t recomp_ldxr(GuestContext *c, uint64_t a, uint32_t s) { uint64_t v = 0; memcpy(&v, A64_PTR(a), s); ex_addr = a; ex_valid = 1; return v; } uint32_t recomp_stxr(GuestContext *c, uint64_t a, uint32_t s, uint64_t v) { if (!ex_valid || ex_addr != a) return 1; ex_valid = 0; memcpy(A64_PTR(a), &v, s); return 0; } void recomp_ldxp(GuestContext *c, uint64_t a, uint32_t s, uint64_t *lo, uint64_t *hi) { *lo = *hi = 0; memcpy(lo, A64_PTR(a), s); memcpy(hi, A64_PTR(a + s), s); ex_addr = a; ex_valid = 1; } uint32_t recomp_stxp(GuestContext *c, uint64_t a, uint32_t s, uint64_t lo, uint64_t hi) { if (!ex_valid || ex_addr != a) return 1; ex_valid = 0; memcpy(A64_PTR(a), &lo, s); memcpy(A64_PTR(a + s), &hi, s); return 0; } void recomp_clrex(GuestContext *c) { ex_valid = 0; } int main(int argc, char **argv) { FILE *in = fopen(argv[1], "rb"), *out = fopen(argv[2], "wb"); if (VirtualAlloc(A64_MEM, %d, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE) != A64_MEM) return 3; static uint8_t state[%d]; unsigned idx; while (fread(&idx, 4, 1, in) == 1) { fread(state, sizeof state, 1, in); fread(A64_MEM + %d, %d, 1, in); Ctx c; memset(&c, 0, sizeof c); memcpy(c.x, state, 256); memcpy(&c.n, state + 256, 4); memcpy(&c.fpcr, state + 264, 8); memcpy(&c.fpsr, state + 272, 8); memcpy(c.vreg, state + 280, 512); ex_valid = 0; T[idx](&c); memcpy(state, c.x, 256); memcpy(state + 256, &c.n, 4); memcpy(state + 264, &c.fpcr, 8); memcpy(state + 272, &c.fpsr, 8); memcpy(state + 280, c.vreg, 512); fwrite(&c.halted, 4, 1, out); fwrite(state, sizeof state, 1, out); fwrite(A64_MEM + %d, %d, 1, out); } fclose(out); return 0; }""" % (MEM_SIZE, STATE.size, SCRATCH, SCRATCH_SIZE, SCRATCH, SCRATCH_SIZE)) path = os.path.join(tmp, "harness.c") open(path, "w", newline="\n").write("\n".join(src)) exe = os.path.join(tmp, "harness.exe") cmd = [toolchain.need(toolchain.find("clang"))] + toolchain.native_target() + toolchain.native_link() + \ ["-O1", "-g0", "-Wno-everything", "-I" + tmp, "-I" + os.path.join(KIT, "rt"), path, os.path.join(tmp, "a64_suyu.c"), "-o", exe] subprocess.run(cmd, check=True) return exe, flags def main(): p = argparse.ArgumentParser() p.add_argument("image_dir") p.add_argument("--modules", default="main,sdk") p.add_argument("--per", type=int, default=6) p.add_argument("--states", type=int, default=12) p.add_argument("--seed", type=int, default=1) p.add_argument("--fpsr", action="store_true") p.add_argument("--list", action="store_true") a = p.parse_args() rng = random.Random(a.seed) picked = sample(a.image_dir, a.modules.split(","), a.per, rng) print("%d encodings in %d forms" % (len(picked), len(set(k for k, _ in picked))), flush=True) tmp = tempfile.mkdtemp(prefix="a64_insn_") exe, flags = build_harness(tmp, picked) cases, expect = [], [] base_mem = bytes(rng.getrandbits(8) for _ in range(4096)) * (SCRATCH_SIZE // 4096) for i, (key, w) in enumerate(picked): for _ in range(a.states): st = rand_state(rng, w) res, err = run_unicorn(w, st, base_mem) if res is None: continue # Unicorn faulted (e.g. an unmapped address): not a comparable case cases.append((i, st)) expect.append(res) inp = os.path.join(tmp, "in.bin") with open(inp, "wb") as f: for i, st in cases: f.write(struct.pack("