Import the base tree (MIT, see LICENSE)

This commit is contained in:
Valiantsin Dzerakh
2026-11-04 22:16:41 +03:00
committed by chrg6
commit 0478c13e45
734 changed files with 233953 additions and 0 deletions

221
tests/diff/atomic.py Normal file
View File

@@ -0,0 +1,221 @@
"""Differential test of exclusives and atomics against Unicorn (max CPU model, which has the ARMv8.1 LSE atomics).
Single steps cannot cover these: a store exclusive only succeeds after a load exclusive, and the A57 model rejects
LSE. So this test enumerates every variant and runs short sequences, with the monitor cleared before each case:
- LDXR/LDAXR then STXR/STLXR to the same address (succeeds), all sizes;
- LDXP/LDAXP then STXP/STLXP to the same address (succeeds) or 16 bytes further (fails), W and X pairs;
- STXR and STXP alone (fail without a monitor);
- LDADD, LDCLR, LDEOR, LDSET, LDSMAX, LDSMIN, LDUMAX, LDUMIN and SWP, every size and ordering;
- CAS and CASP, every size and ordering, with the compare value equal to memory in half of the cases.
Needs no game files. Usage: atomic.py [--cases K] [--seed S] [--jobs J]
"""
import argparse
import os
import platform
import random
import struct
import sys
import time
import zlib
import unicorn as uc
from unicorn import arm64_const as A
from common import BUF, BUF_SIZE, ROOT, Emu, build_harness, diff_state, dis, rand_fpcr, rand_v, rand_x, run_harness
from lifter import Lifter
BUF_MID = BUF + 0x10000
CODE = 0x7220000000
SLOT = 16 # bytes of code per test
CLREX = 0xD5033F5F
def lse(size, a, r, rs, o3, opc, rn, rt):
return 0x38200000 | size << 30 | a << 23 | r << 22 | rs << 16 | o3 << 15 | opc << 12 | rn << 5 | rt
def cas(size, l, rs, o0, rn, rt):
return 0x08A07C00 | size << 30 | l << 22 | rs << 16 | o0 << 15 | rn << 5 | rt
def casp(sz, l, rs, o0, rn, rt):
return 0x08207C00 | sz << 30 | l << 22 | rs << 16 | o0 << 15 | rn << 5 | rt
def ldxp(sz, o0, rt2, rn, rt):
return 0x887F0000 | sz << 30 | o0 << 15 | rt2 << 10 | rn << 5 | rt
def stxp(sz, o0, rs, rt2, rn, rt):
return 0x88200000 | sz << 30 | rs << 16 | o0 << 15 | rt2 << 10 | rn << 5 | rt
def ldxr(size, o0, rn, rt):
return 0x085F7C00 | size << 30 | o0 << 15 | rn << 5 | rt
def stxr(size, o0, rs, rn, rt):
return 0x08007C00 | size << 30 | rs << 16 | o0 << 15 | rn << 5 | rt
def regs(r, n, exclude=()):
"""n distinct general registers 0-30 outside exclude."""
return r.sample([i for i in range(31) if i not in exclude], n)
def tests(r):
"""[(words, rn, kind)]: kind 'cas'/'casp' marks compare-and-swap tests (state gets a matching compare value)."""
out = []
for size in range(4):
for a in (0, 1):
for rel in (0, 1):
for o3, opc in [(0, k) for k in range(8)] + [(1, 0)]:
rn = r.choice(regs(r, 1) + [31])
rs, rt = r.randrange(32), r.randrange(32)
out.append(([lse(size, a, rel, rs, o3, opc, rn, rt)], rn, 'lse'))
rn = r.choice(regs(r, 1) + [31])
rs, rt = regs(r, 2, {rn})
out.append(([cas(size, a, rs, rel, rn, rt)], rn, 'cas'))
for sz in (0, 1):
for l in (0, 1):
for o0 in (0, 1):
rn = r.choice(regs(r, 1) + [31])
rs, rt = (2 * i for i in r.sample(range(15), 2))
if rn in (rs, rs + 1):
rs = 30 if rn < 30 else 0 # Rs+1 = XZR or away from Rn
out.append(([casp(sz, l, rs, o0, rn, rt)], rn, 'casp'))
for size in range(4):
for o0 in (0, 1):
rn = r.choice(regs(r, 1) + [31])
rt, rs, rt_st = regs(r, 3, {rn})
out.append(([ldxr(size, o0, rn, rt), stxr(size, o0, rs, rn, rt_st)], rn, 'excl'))
out.append(([stxr(size, o0, rs, rn, rt_st)], rn, 'excl'))
for sz in (0, 1):
for o0 in (0, 1):
rn = r.choice(regs(r, 1) + [31])
rt, rt2, rs, d1, d2 = regs(r, 5, {rn})
out.append(([ldxp(sz, o0, rt2, rn, rt)], rn, 'excl'))
out.append(([ldxp(sz, o0, rt2, rn, rt), stxp(sz, o0, rs, d2, rn, d1)], rn, 'excl'))
out.append(([stxp(sz, o0, rs, d2, rn, d1)], rn, 'excl'))
# A store exclusive to another address fails; Rn is advanced between the two by an ADD.
if rn != 31:
add = 0x91004000 | rn << 5 | rn # add xn, xn, #16
out.append(([ldxp(sz, o0, rt2, rn, rt), add, stxp(sz, o0, rs, d2, rn, d1)], rn, 'excl'))
return out
def unicorn_wrong(w):
"""LDSMAX/LDSMIN on bytes, halfwords and words: Unicorn 2.1 (QEMU 5.0) compares them unsigned."""
return (w & 0x3F200C00) == 0x38200000 and (w >> 30) < 3 and not (w >> 15) & 1 and (w >> 12) & 7 in (4, 5)
def model_signed_minmax(w, x, sp, nzcv, v, buf0):
"""Reference for unicorn_wrong(w) words, from the ARM pseudocode: returns the expected state and memory CRC."""
nb = 1 << (w >> 30)
rs, rn, rt = (w >> 16) & 31, (w >> 5) & 31, w & 31
addr = sp if rn == 31 else x[rn]
off = addr - BUF
old = int.from_bytes(buf0[off:off + nb], 'little')
operand = (x[rs] if rs != 31 else 0) & ((1 << 8 * nb) - 1)
sign = lambda u: u - (1 << 8 * nb) if u >> (8 * nb - 1) else u
pick = max if (w >> 12) & 7 == 4 else min
new = old if pick(sign(old), sign(operand)) == sign(old) else operand
mem = bytearray(buf0)
mem[off:off + nb] = new.to_bytes(nb, 'little')
ox = list(x[:31])
if rt != 31:
ox[rt] = old
return (ox, sp, nzcv, 0, list(v), zlib.crc32(bytes(mem)))
def make_case(r, words, rn, kind, buf0):
addr = BUF_MID + r.randrange(0, 0x100, 16)
x = [rand_x(r) for _ in range(31)]
sp = addr if rn == 31 else BUF_MID + r.randrange(0, 0x100, 16)
if rn != 31:
x[rn] = addr
w = words[0]
if kind in ('cas', 'casp') and r.random() < 0.5:
off = addr - BUF
rs = (w >> 16) & 31
if kind == 'cas':
nb = 1 << (w >> 30)
if rs != 31:
x[rs] = int.from_bytes(buf0[off:off + nb], 'little')
else:
nb = 8 if (w >> 30) & 1 else 4
for i in (0, 1):
if rs + i != 31:
x[rs + i] = int.from_bytes(buf0[off + i * nb:off + (i + 1) * nb], 'little')
return rand_fpcr(r), x, sp, r.getrandbits(4) << 28, [rand_v(r) for _ in range(32)]
def main():
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
ap.add_argument('--cases', type=int, default=64)
ap.add_argument('--seed', type=int, default=1)
ap.add_argument('--jobs', type=int, default=os.cpu_count() or 4)
ap.add_argument('--cflags', default='-march=x86-64-v3' if platform.machine().lower() in ('amd64', 'x86_64') else '')
args = ap.parse_args()
r = random.Random(args.seed)
t0 = time.time()
ts = tests(r)
code = [CLREX, 0, 0, 0]
bodies = []
lifter = Lifter(0, 'c')
for i, (words, _, _) in enumerate(ts):
pc = CODE + SLOT * (i + 1)
code += words + [0] * (SLOT // 4 - len(words))
bodies.append(' '.join(lifter.lift(w, pc + 4 * j).code for j, w in enumerate(words)))
code_bytes = struct.pack(f'<{len(code)}I', *code)
work = ROOT / 'build' / 'test' / 'atomic'
work.mkdir(parents=True, exist_ok=True)
code_path = work / 'code.bin'
code_path.write_bytes(code_bytes)
exe = build_harness(work, bodies, args.jobs, args.cflags)
emu = Emu([(CODE, code_bytes)], A.UC_CPU_ARM64_MAX)
mu = emu.mu
buf0 = bytes(r.getrandbits(8) for _ in range(4096)) * (BUF_SIZE // 4096)
mu.mem_write(BUF, buf0)
cases, expect = [], []
for k, (words, rn, kind) in enumerate(ts):
pc = CODE + SLOT * (k + 1)
for _ in range(args.cases):
fpcr, x, sp, nzcv, v = make_case(r, words, rn, kind, buf0)
if unicorn_wrong(words[0]):
cases.append((k, fpcr, x, sp, nzcv, v, True))
expect.append(model_signed_minmax(words[0], x, sp, nzcv, v, buf0))
continue
mu.emu_start(CODE, CODE + 4, count=1) # clear the exclusive monitor
try:
ox, osp, onz, opc, ov = emu.run(pc, fpcr, x, sp, nzcv, v, count=len(words))
except uc.UcError as e:
raise SystemExit(f'unicorn rejected {" ; ".join(dis(w) for w in words)}: {e}')
mh = zlib.crc32(bytes(mu.mem_read(BUF, BUF_SIZE)))
mu.mem_write(BUF, buf0)
cases.append((k, fpcr, x, sp, nzcv, v, True))
expect.append((ox, osp, onz, opc, ov, mh))
recs, err = run_harness(exe, work, buf0, cases, [(CODE, code_path)])
if err:
k = cases[len(recs)][0] if len(recs) < len(cases) else None
print(f'harness failed: {err}' + (f', stopped at {" ; ".join(dis(w) for w in ts[k][0])}' if k is not None else ''))
fails, nfail = {}, 0
for case, want, got in zip(cases, expect, recs):
d = diff_state(got, want, check_pc=False, check_mem=True)
if d:
nfail += 1
fails.setdefault(case[0], d[:4])
print(f'{len(ts)} tests, {len(cases)} cases: {nfail} failing cases, {len(fails)} failing tests '
f'({time.time() - t0:.0f}s)')
for k, d in fails.items():
print(f' {" ; ".join(dis(w) for w in ts[k][0]):70} {"; ".join(d)}')
return 1 if fails or err else 0
if __name__ == '__main__':
sys.exit(main())

276
tests/diff/common.py Normal file
View File

@@ -0,0 +1,276 @@
"""Shared parts of the differential tests: Unicorn reference, operand generators, build and run of harness.c.
Adapted from LAS Recomp tools/test_lift.py and tools/test_simd.py @ da35b83 (MIT, Copyright (c) 2026 Peiikai).
"""
import os
import shutil
import struct
import subprocess
import sys
from pathlib import Path
import capstone
import unicorn as uc
from unicorn import arm64_const as A
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / 'recomp'))
BUF, BUF_SIZE = 0x7200000000, 0x40000
UC_INVALID = (uc.UC_ERR_INSN_INVALID, uc.UC_ERR_EXCEPTION)
# Record layouts of struct In / struct Out in harness.c.
IN_FMT = '<QQ32QQQ'
IN_TAIL = '<QQ'
OUT_SIZE = 32 * 8 + 4 * 8 + 32 * 16
MSYS_BIN = Path(r'C:\msys64\ucrt64\bin')
F32 = [0, 0x80000000, 0x7f800000, 0xff800000, 0x7fc00000, 0xffc00000, 0x7fa00001, 0xff812345, 0x7fd00abc,
0x00000001, 0x80000001, 0x007fffff, 0x807fffff, 0x00400000, 0x00800000, 0x80800000, 0x00800001, 0x7f7fffff,
0xff7fffff, 0x3f800000, 0xbf800000, 0x3f000000, 0xbf000000, 0x40000000, 0x40400000, 0x3fc00000, 0xbfc00000,
0x40200000, 0x40100000, 0x4f000000, 0xcf000000, 0x4f800000, 0x5f000000, 0xdf000000, 0x5f800000, 0x4effffff,
0x4f7fffff, 0x5effffff, 0x5f7fffff, 0x3effffff, 0x3f7fffff, 0x7effffff, 0x01000000, 0x3e800000, 0x7e800000,
0x7e7fffff, 0x00200000, 0x001fffff, 0x00200001, 0x47800000, 0x477fe000, 0x477ff000, 0x38800000, 0x33800000,
0x33000000, 0x387fc000, 0x3f7ff000, 0x46fffe00, 0x80000000 | 0x00200000, 0x7f000000, 0x7f7ffffe]
F64 = [0, 1 << 63, 0x7ff0000000000000, 0xfff0000000000000, 0x7ff8000000000000, 0xfff8000000000000,
0x7ff4000000000001, 0xfff0000000012345, 0x7ffc00000abcdef0, 1, (1 << 63) | 1, 0x000fffffffffffff,
0x0008000000000000, 0x0010000000000000, 0x8010000000000000, 0x7fefffffffffffff, 0xffefffffffffffff,
0x3ff0000000000000, 0xbff0000000000000, 0x3fe0000000000000, 0x4000000000000000, 0x4008000000000000,
0x3ff8000000000000, 0xbff8000000000000, 0x4004000000000000, 0x41e0000000000000, 0xc1e0000000000000,
0x41f0000000000000, 0x43e0000000000000, 0xc3e0000000000000, 0x43f0000000000000, 0x41dfffffffc00000,
0x41efffffffe00000, 0x43dfffffffffffff, 0x43efffffffffffff, 0x3fdfffffffffffff, 0x7fdfffffffffffff,
0x7fd0000000000000, 0x0004000000000000, 0x0003ffffffffffff, 0x47efffffe0000000, 0x47efffff00000000,
0x36a0000000000000, 0x3810000000000000, 0x380fffffffffffff, 0x47f0000000000000, 0x3e70000000000000,
0x3f0ffc0000000000, 0x40effc0000000000, 0x40effe0000000000, 0x3e6ffffffff00000, 0x369fffffffffffff,
0x3810000010000000, 0x381000000fffffff, 0x3e80000000000000]
F16 = [0, 0x8000, 0x7c00, 0xfc00, 0x7e00, 0xfe00, 0x7d00, 0x7e55, 0x0001, 0x8001, 0x03ff, 0x0400, 0x7bff, 0xfbff,
0x3c00, 0xbc00, 0x3800, 0x4000, 0x3e00, 0x7fff, 0xffff]
_md = capstone.Cs(capstone.CS_ARCH_ARM64, capstone.CS_MODE_ARM)
def dis(w, pc=0):
d = next(_md.disasm(struct.pack('<I', w), pc), None)
return f'{d.mnemonic} {d.op_str}' if d else '?'
def mnemonic(w):
return dis(w).split(' ')[0]
def rand_lane(r, sz):
"""Random interesting integer lane of 8 << sz bits."""
bits_ = 8 << sz
k = r.random()
if k < 0.35:
return r.choice([0, 1, 2, 3, (1 << bits_) - 1, (1 << bits_) - 2, 1 << (bits_ - 1), (1 << (bits_ - 1)) - 1,
(1 << (bits_ - 1)) + 1, 0x80 % (1 << bits_), 0x7f, bits_ - 1, bits_, bits_ + 1,
(1 << bits_) - bits_, 1 << (bits_ // 2)])
if k < 0.5:
return r.getrandbits(r.randint(1, bits_))
if k < 0.6:
return ((1 << bits_) - r.getrandbits(r.randint(1, bits_ - 1))) & ((1 << bits_) - 1)
return r.getrandbits(bits_)
def rand_f32(r):
k = r.random()
if k < 0.4:
return r.choice(F32)
if k < 0.8:
e = r.choice([r.randint(110, 145), r.randint(0, 255), r.randint(120, 135)])
return (r.getrandbits(1) << 31) | (e << 23) | (r.getrandbits(23) if r.random() < 0.7 else
r.getrandbits(4) << 19)
return struct.unpack('<I', struct.pack('<f', r.choice([r.uniform(-4, 4), float(r.randint(-300, 300)) + 0.5,
float(r.randint(-5, 5)), r.uniform(-1e6, 1e6)])))[0]
def rand_f64(r):
k = r.random()
if k < 0.4:
return r.choice(F64)
if k < 0.8:
e = r.choice([r.randint(1000, 1050), r.randint(0, 2047), r.randint(1020, 1030), r.randint(0, 40),
r.randint(2000, 2047), r.randint(860, 920)])
return (r.getrandbits(1) << 63) | (e << 52) | (r.getrandbits(52) if r.random() < 0.7 else
r.getrandbits(6) << 46)
return struct.unpack('<Q', struct.pack('<d', r.choice([r.uniform(-4, 4), float(r.randint(-300, 300)) + 0.5,
r.uniform(-1e12, 1e12)])))[0]
def rand_v(r):
k = r.random()
out = 0
if k < 0.3:
for i in range(4):
out |= rand_f32(r) << (32 * i)
elif k < 0.55:
for i in range(2):
out |= rand_f64(r) << (64 * i)
elif k < 0.6:
for i in range(8):
out |= (r.choice(F16) if r.random() < 0.5 else r.getrandbits(16)) << (16 * i)
elif k < 0.9:
sz = r.randrange(4)
for i in range(16 >> sz):
out |= rand_lane(r, sz) << ((8 << sz) * i)
else:
out = r.getrandbits(128)
if r.random() < 0.15: # upper half zero, like most 64-bit results
out &= (1 << 64) - 1
return out
def rand_x(r):
k = r.random()
if k < 0.3:
return r.choice([0, 1, 2, 0xFFFFFFFFFFFFFFFF, 0x7FFFFFFFFFFFFFFF, 0x8000000000000000, 0xFFFFFFFF,
0x7FFFFFFF, 0x80000000, 0x100000000, 0xFFFFFFFF80000000, 0x8000000000000001, 0xFFFFFF,
0x1000001, 0x20000000000001, 0xFFFFFFFFFF000001, 63, 64, 31, 32, 0xFF, 0xFFFF])
if k < 0.45:
return rand_f32(r) | (r.getrandbits(32) << 32 if r.random() < 0.3 else 0)
if k < 0.6:
return rand_f64(r)
if k < 0.75:
return r.getrandbits(r.randint(1, 64))
if k < 0.8:
return (-r.getrandbits(r.randint(1, 63))) & 0xFFFFFFFFFFFFFFFF
return r.getrandbits(64)
def rand_fpcr(r):
k = r.random()
if k < 0.55:
return 0
v = 0
if r.random() < 0.4:
v |= 1 << 25 # DN
if r.random() < 0.4:
v |= 1 << 24 # FZ
if r.random() < 0.4:
v |= r.randrange(4) << 22 # RMode
if r.random() < 0.15:
v |= 1 << 26 # AHP
return v
class Emu:
"""Unicorn as a Cortex-A57, the Switch's CPU, unless another model is given (LSE atomics need
UC_CPU_ARM64_MAX). Images are (base, bytes) pairs mapped at their guest address."""
def __init__(self, images, model=A.UC_CPU_ARM64_A57):
mu = uc.Uc(uc.UC_ARCH_ARM64, uc.UC_MODE_ARM)
mu.ctl_set_cpu_model(model)
mu.mem_map(BUF, BUF_SIZE)
for base, data in images:
mu.mem_map(base, (len(data) + 0xFFFF) & ~0xFFFF)
mu.mem_write(base, bytes(data))
mu.reg_write(A.UC_ARM64_REG_CPACR_EL1, 0x300000) # enable FP/SIMD
self.mu = mu
self.xregs = [A.UC_ARM64_REG_X0 + i for i in range(29)] + [A.UC_ARM64_REG_X29, A.UC_ARM64_REG_X30]
self.qregs = [A.UC_ARM64_REG_Q0 + i for i in range(32)]
self.out_regs = self.xregs + [A.UC_ARM64_REG_SP, A.UC_ARM64_REG_NZCV, A.UC_ARM64_REG_PC] + self.qregs
def run(self, pc, fpcr, x, sp, nzcv, v, count=1):
"""Run count instructions from pc. Returns (x[31], sp, nzcv, pc, v[32])."""
mu = self.mu
mu.reg_write_batch(list(zip(self.xregs, x[:31])) + list(zip(self.qregs, v)) +
[(A.UC_ARM64_REG_SP, sp), (A.UC_ARM64_REG_NZCV, nzcv), (A.UC_ARM64_REG_FPCR, fpcr)])
mu.emu_start(pc, pc + 4 * count + 4, count=count)
o = mu.reg_read_batch(self.out_regs)
return list(o[:31]), o[31], o[32] & 0xF0000000, o[33], list(o[34:])
def compiler():
cc = os.environ.get('CC')
if cc:
return cc
if os.name == 'nt' and (MSYS_BIN / 'clang.exe').exists():
return str(MSYS_BIN / 'clang.exe')
return shutil.which('clang') or 'clang'
def build_harness(work, bodies, jobs, cflags):
"""Compile one C function per test body plus harness.c into work/harness[.exe]."""
cc = compiler()
env = dict(os.environ)
if os.name == 'nt' and MSYS_BIN.exists():
env['PATH'] = str(MSYS_BIN) + os.pathsep + env.get('PATH', '')
flags = ['-O1', '-fno-strict-aliasing', '-Wno-unused-label', '-I', str(ROOT / 'runtime')] + cflags.split()
n = max(1, min(jobs, len(bodies) // 200 + 1))
per = (len(bodies) + n - 1) // n
srcs = []
for j in range(n):
path = work / f'tests_{j}.c'
with open(path, 'w') as f:
f.write('#include "cpu.h"\n')
for k in range(j * per, min(len(bodies), (j + 1) * per)):
f.write(f'void t{k}(Cpu *c) {{ {bodies[k]} }}\n')
srcs.append(path)
with open(work / 'table.c', 'w') as f:
f.write('#include "cpu.h"\n')
f.write(''.join(f'void t{k}(Cpu *c);\n' for k in range(len(bodies))))
f.write('GuestFn tests[] = {' + ','.join(f't{k}' for k in range(len(bodies))) + '};\n')
f.write(f'int ntests = {len(bodies)};\n')
# fma.c: a --cflags=-march=x86-64-v2 run must exercise the software fma the v2 build links, not the C runtime's.
srcs += [work / 'table.c', ROOT / 'tests' / 'diff' / 'harness.c', ROOT / 'runtime' / 'fma.c']
objs, procs = [], []
for s in srcs:
o = work / (s.stem + '.o')
objs.append(o)
procs.append(subprocess.Popen([cc] + flags + ['-c', str(s), '-o', str(o)], env=env))
if any(p.wait() for p in procs):
raise SystemExit('compile failed')
exe = work / ('harness.exe' if os.name == 'nt' else 'harness')
subprocess.check_call([cc, '-fuse-ld=lld', '-o', str(exe)] + [str(o) for o in objs], env=env)
return exe
def write_cases(path, cases, stubs=False):
"""cases: (k, fpcr, x[31], sp, nzcv, v[32], memchk). stubs: calls go to harness.c's call stub (rt_stub)."""
with open(path, 'wb') as f:
for k, fpcr, x, sp, nzcv, v, memchk in cases:
f.write(struct.pack(IN_FMT, k, fpcr, *(list(x[:31]) + [0]), sp, nzcv))
for q in v:
f.write(q.to_bytes(16, 'little'))
f.write(struct.pack(IN_TAIL, int(memchk), int(stubs)))
def run_harness(exe, work, buf0, cases, images, stubs=False):
"""Run the lifted tests. Returns (records, stderr tail or None). images: (base, path) pairs."""
with open(work / 'buf.bin', 'wb') as f:
f.write(buf0)
write_cases(work / 'cases.bin', cases, stubs)
out = work / 'out.bin'
args = [str(exe), str(work / 'buf.bin'), str(work / 'cases.bin'), str(out)]
for base, path in images:
args += [str(path), f'{base:x}']
p = subprocess.run(args, capture_output=True)
err = None if p.returncode == 0 else f'exit {p.returncode:#x}: {p.stderr.decode(errors="replace")[-2000:]}'
data = out.read_bytes() if out.exists() else b''
recs = []
for i in range(len(data) // OUT_SIZE):
rec = data[i * OUT_SIZE:(i + 1) * OUT_SIZE]
x = struct.unpack_from('<31Q', rec, 0)
sp, nzcv, pc, mem = struct.unpack_from('<4Q', rec, 256)
v = [int.from_bytes(rec[288 + 16 * j:304 + 16 * j], 'little') for j in range(32)]
recs.append((x, sp, nzcv, pc, mem, v))
return recs, err
def diff_state(got, want, check_pc, check_mem, check_nzcv=True):
"""got: harness record; want: (x, sp, nzcv, pc, v, mem). Returns a list of mismatches."""
hx, hsp, hnz, hpc, hmem, hv = got
ex, esp, enz, epc, ev, emem = want
d = [f'x{j}: got {hx[j]:x} want {ex[j]:x}' for j in range(31) if hx[j] != ex[j]]
if hsp != esp:
d.append(f'sp: got {hsp:x} want {esp:x}')
if check_nzcv and hnz != enz:
d.append(f'nzcv: got {hnz:x} want {enz:x}')
if check_pc and hpc != epc:
d.append(f'pc: got {hpc:x} want {epc:x}')
if check_mem and hmem != emem:
d.append('memory differs')
d += [f'v{j}: got {hv[j]:032x} want {ev[j]:032x}' for j in range(32) if hv[j] != ev[j]]
return d

428
tests/diff/func.py Normal file
View File

@@ -0,0 +1,428 @@
"""Differential test of whole emitted functions: recomp/gen.py's C for a guest function against Unicorn running the
same function from its entry until it returns.
Picks functions from the ExeFS modules: no branches out of the function, no SVC, BRK, system register, exclusive or
atomic instructions; with conditional branches, and loops, jump tables, SIMD and calls where the module has them.
Emits them with the generator's own Emitter, compiles them with tests/diff/harness.c, and runs each from random
states: x0-x7 point into the scratch buffer (or hold small integers), the buffer holds pointers into itself and random
data, SP sits near the buffer's top and LR is a sentinel address. Unicorn (Cortex-A57) stops when PC reaches the
sentinel. A case is skipped when Unicorn faults, writes outside the scratch buffer (module images are mapped
read-only), leaves the function, or runs longer than --steps instructions. Compares X0-X30, SP, V0-V31 and the
scratch buffer; NZCV is local to a generated function (recomp/flags.py).
Every BL and BLR goes to one stub on both sides (rt_stub in harness.c, Runner.stub here) instead of its callee. The
stub folds the guest state at the call into a digest and sets the caller-saved registers from it, so the test sees
whether the emitted code had every register in the Cpu at the call and reads them back after it. Callers of another
picked function are dropped, as the C side would run that function instead of the stub.
With --patches, the spin waits of the table (recomp/patches.py `spin`) are emitted with their gates, and functions
that hold one come first (a quarter of the picks at most); a case whose loop does not leave runs into --steps.
Usage: func.py [--exefs DIR] [--modules main,subsdk0,sdk] [--functions N] [--cases K] [--seed S] [--jobs J]
[--patches FILE]
"""
import argparse
import os
import platform
import random
import subprocess
import sys
import time
import zlib
from collections import Counter
from pathlib import Path
import numpy as np
import unicorn as uc
from unicorn import arm64_const as A
from common import BUF, BUF_SIZE, MSYS_BIN, ROOT, Emu, compiler, diff_state, rand_fpcr, rand_v, rand_x, run_harness
import gen
import patches
from layout import layout
from module import Module
SENTINEL = 0x7210000000 # LR on entry; never executed
PTR_LO, PTR_HI = BUF + 0x8000, BUF + 0x28000
STACK_TOP = BUF + 0x3F000
MIN_INSNS, MAX_INSNS = 6, 600
FNV = 0x100000001B3
M64 = (1 << 64) - 1
def splitmix(z):
z = (z + 0x9E3779B97F4A7C15) & M64
z = ((z ^ (z >> 30)) * 0xBF58476D1CE4E5B9) & M64
z = ((z ^ (z >> 27)) * 0x94D049BB133111EB) & M64
return z ^ (z >> 31)
def call_mask(w):
return ((w & 0xFC000000) == 0x94000000) | ((w & 0xFFFFFC1F) == 0xD63F0000) # BL, BLR
def forbidden_mask(w):
"""Words a picked function must not contain."""
m = (w & 0xFF000000) == 0xD4000000 # SVC, HVC, SMC, BRK, HLT, DCPS
sysi = (w & 0xFFC00000) == 0xD5000000 # system: MRS, MSR, SYS (DC ZVA), hints, barriers
hint_or_barrier = ((w & 0xFFFFF000) == 0xD5032000) | ((w & 0xFFFFF000) == 0xD5033000)
m |= sysi & ~hint_or_barrier
m |= (w & 0x3F000000) == 0x08000000 # exclusives, load-acquire / store-release, CAS
m |= (w & 0x3F200C00) == 0x38200000 # LSE atomics
return m
def cond_mask(w):
return (((w & 0xFE000010) == 0x54000000) | ((w & 0x7E000000) == 0x34000000) |
((w & 0x7E000000) == 0x36000000))
def simd_mask(w):
return ((w & 0x0E000000) == 0x0E000000) | (((w & 0x0A000000) == 0x08000000) & (((w >> 26) & 1) == 1))
def pick_functions(name, mod, count, r, spins=None):
"""[(entry, end, tags)] of up to count functions of mod, plus the Emitter that will emit them. spins: {spin wait
back-edge: head}, emitted with gates."""
starts, _ = gen.discover(mod)
w = mod.words
ends = np.append(starts[1:], len(w))
def per_fn(mask):
cs = np.concatenate([[0], np.cumsum(mask.astype(np.int64))])
return cs[ends] - cs[starts]
size = ends - starts
ok = (per_fn(forbidden_mask(w)) == 0) & (per_fn(cond_mask(w)) > 0) & (size >= MIN_INSNS) & (size <= MAX_INSNS)
has_br = per_fn((w & 0xFFFFFC1F) == 0xD61F0000) > 0
has_call = per_fn(call_mask(w)) > 0
cand = np.flatnonzero(ok)
with_br = [int(i) for i in cand if has_br[i]]
with_call = [int(i) for i in cand if has_call[i] and not has_br[i]]
rest = [int(i) for i in cand if not has_br[i] and not has_call[i]]
spins = spins or {}
gated = np.searchsorted(mod.lo + starts * 4, np.array(sorted(spins), dtype=np.int64), side='right') - 1
with_spin = sorted({int(i) for i in gated} & set(cand.tolist()))
for pool in (with_spin, with_br, with_call, rest):
r.shuffle(pool)
emitter = gen.Emitter(mod, set((mod.lo + starts * 4).tolist()), spins=spins)
picks, tags_seen = [], Counter()
# Functions with a spin wait first, then with a br (jump tables, fallbacks), at most a quarter of the picks each;
# then functions with calls, at most half.
for pool, quota in ((with_spin, count // 4), (with_br, count // 4), (with_call, count // 2), (rest, count)):
taken = 0
for i in pool:
if len(picks) >= count or taken >= quota:
break
entry, end = mod.lo + int(starts[i]) * 4, mod.lo + int(ends[i]) * 4
if any(e == entry for e, _, _ in picks):
continue
tags = check_function(emitter, entry, end)
if tags is None:
continue
if any(spins.get(x) is not None for x in range(entry, end, 4)):
tags.add('spin_gate')
picks.append((entry, end, tags))
taken += 1
picked = {e for e, _, _ in picks}
picks = [p for p in picks if not callees(emitter, p[0], p[1]) & picked]
for _, _, tags in picks:
tags_seen.update(tags)
print(f'{name}: {len(starts)} functions, {len(cand)} candidates, picked {len(picks)}: '
f'{dict(tags_seen)}', flush=True)
return picks, emitter
def call_sites(mod, entry, end):
"""Addresses of the BL and BLR instructions of [entry, end)."""
return [a for a in range(entry, end, 4) if mod.lift(a).kind in ('bl', 'blr')]
def callees(emitter, entry, end):
"""Function starts the emitted C of [entry, end) calls directly (`f_<addr>(c)`, not through rt_dispatch)."""
mod = emitter.mod
return {mod.lift(a).target for a in call_sites(mod, entry, end)
if mod.lift(a).kind == 'bl' and mod.lift(a).target in emitter.starts}
def check_function(emitter, entry, end):
"""Tags of a function the test can run in isolation, or None."""
mod = emitter.mod
insns = [mod.lift(a) for a in range(entry, end, 4)]
if insns[-1].kind not in gen.TERMINATORS or not any(i.kind == 'ret' for i in insns):
return None
tags = set()
for k, ins in enumerate(insns):
a = entry + 4 * k
if ins.kind in ('bl', 'blr'):
tags.add('calls')
if ins.kind in ('b', 'bcond'):
if not entry <= ins.target < end:
return None
if ins.target <= a:
tags.add('loop')
if ins.kind == 'br':
jt = gen.jumptables.resolve(mod, a, entry, end)
if jt:
tags.add('jump_table')
elif gen.jumptables.is_computed(mod, a, entry):
tags.add('all_label_fallback')
else:
return None
if simd_mask(mod.words[(entry - mod.lo) >> 2:(end - mod.lo) >> 2]).any():
tags.add('simd')
return tags
def build(work, emitted, jobs, cflags):
"""Compile the emitted functions with harness.c into work/func[.exe]; tests[k] is emitted[k]."""
cc = compiler()
env = dict(os.environ)
if os.name == 'nt' and MSYS_BIN.exists():
env['PATH'] = str(MSYS_BIN) + os.pathsep + env.get('PATH', '')
flags = ['-O1', '-fno-strict-aliasing', '-Wno-return-type', '-Wno-unused-label', '-I', str(ROOT / 'runtime')]
flags += cflags.split()
groups = {}
for name, entry, end, emitter in emitted:
groups.setdefault(name, []).append((entry, end))
srcs, n = [], max(1, jobs)
for name, fns in groups.items():
emitter = next(e for m, _, _, e in emitted if m == name)
per = (len(fns) + n - 1) // n
for j in range(0, len(fns), per):
path = work / f'{name}_{j // per:03d}.c'
path.write_text(gen.chunk_text(name, j // per, emitter, fns[j:j + per]))
srcs.append(path)
table = work / 'table.c'
called = set()
for _, entry, end, emitter in emitted:
called |= callees(emitter, entry, end)
table.write_text('#include "cpu.h"\n' + ''.join(f'void {gen.fname(e)}(Cpu *c);\n' for _, e, _, _ in emitted) +
'GuestFn tests[] = {' + ','.join(gen.fname(e) for _, e, _, _ in emitted) + '};\n' +
f'int ntests = {len(emitted)};\n' + 'void rt_stub_call(Cpu *c, uint64_t target);\n' +
''.join(f'void {gen.fname(t)}(Cpu *c) {{ rt_stub_call(c, {hex(t)}ull); }}\n'
for t in sorted(called)))
# fma.c: a --cflags=-march=x86-64-v2 run must exercise the software fma the v2 build links, not the C runtime's.
srcs += [table, ROOT / 'tests' / 'diff' / 'harness.c', ROOT / 'runtime' / 'fma.c']
objs, procs = [], []
for s in srcs:
o = work / (s.stem + '.o')
objs.append(o)
procs.append(subprocess.Popen([cc] + flags + ['-c', str(s), '-o', str(o)], env=env))
if any(p.wait() for p in procs):
raise SystemExit('compile failed')
exe = work / ('func.exe' if os.name == 'nt' else 'func')
subprocess.check_call([cc, '-fuse-ld=lld', '-o', str(exe)] + [str(o) for o in objs], env=env)
return exe
def make_buffer(r):
out = bytearray()
for _ in range(BUF_SIZE // 8):
k = r.random()
if k < 0.4:
v = r.randrange(PTR_LO, PTR_HI, 8)
elif k < 0.7:
v = r.randrange(0, 256)
else:
v = r.getrandbits(64)
out += v.to_bytes(8, 'little')
return bytes(out)
def make_case(r):
x = [rand_x(r) for _ in range(31)]
for i in range(8):
k = r.random()
if k < 0.7:
x[i] = r.randrange(PTR_LO, PTR_HI, 16)
elif k < 0.9:
x[i] = r.randrange(0, 64)
x[30] = SENTINEL
sp = STACK_TOP - r.randrange(0, 0x100, 16)
return rand_fpcr(r), x, sp, r.getrandbits(4) << 28, [rand_v(r) for _ in range(32)]
class Runner:
"""Unicorn with the module images read-only, a block hook that stops when execution leaves the function, and a
code hook on every call site of the picked functions that stops before the call, for the stub."""
MAX_CALLS = 1000
def __init__(self, images, buf0, sites):
self.emu = Emu(images)
mu = self.emu.mu
for base, data in images:
mu.mem_protect(base, (len(data) + 0xFFFF) & ~0xFFFF, uc.UC_PROT_READ | uc.UC_PROT_EXEC)
mu.mem_map(SENTINEL, 0x1000)
mu.mem_write(BUF, buf0)
self.buf0 = buf0
self.lo = self.hi = self.h = 0
self.left = False
self.at_call = None
mu.hook_add(uc.UC_HOOK_BLOCK, self._block)
# One hook per call site, all added before anything is translated, so no cached block lacks one.
for a in sorted(sites):
mu.hook_add(uc.UC_HOOK_CODE, self._call, begin=a, end=a)
def _block(self, mu, address, size, user):
if not self.lo <= address < self.hi and address != SENTINEL:
self.left = True
mu.emu_stop()
def _call(self, mu, address, size, user):
if self.lo <= address < self.hi:
self.at_call = address
mu.emu_stop()
def stub(self, a):
"""harness.c rt_stub for the call at a, which Unicorn has not executed yet. Returns the address to resume at."""
e, mu = self.emu, self.emu.mu
w = int.from_bytes(mu.mem_read(a, 4), 'little')
if (w & 0xFC000000) == 0x94000000: # BL
imm = w & 0x03FFFFFF
target = (a + ((imm - (1 << 26) if imm >> 25 else imm) << 2)) & M64
else: # BLR
target = mu.reg_read(e.xregs[(w >> 5) & 31])
mu.reg_write(A.UC_ARM64_REG_X30, a + 4)
h = ((self.h ^ target) * FNV) & M64
for val in list(mu.reg_read_batch(e.xregs)) + [mu.reg_read(A.UC_ARM64_REG_SP)]:
h = ((h ^ val) * FNV) & M64
for val in mu.reg_read_batch(e.qregs[:8]):
h = ((h ^ (val & M64)) * FNV) & M64
h = ((h ^ (val >> 64)) * FNV) & M64
self.h = h
regs = []
for i in range(19):
s = splitmix((h + i) & M64)
regs.append((e.xregs[i], PTR_LO + ((s >> 4) % ((PTR_HI - PTR_LO) // 16)) * 16 if i < 8 and s & 3 else s))
regs.append((A.UC_ARM64_REG_NZCV, splitmix((h + 19) & M64) >> 60 << 28))
for i in range(8):
regs.append((e.qregs[i], splitmix((h + 33 + 2 * i) & M64) << 64 | splitmix((h + 32 + 2 * i) & M64)))
mu.reg_write_batch(regs)
top = BUF + BUF_SIZE - 16
n = int.from_bytes(mu.mem_read(top, 8), 'little')
mu.mem_write(top, ((n + 1) & M64).to_bytes(8, 'little') + h.to_bytes(8, 'little'))
return a + 4
def run(self, entry, end, case, steps):
"""(x, sp, nzcv, pc, v, crc of the buffer) or None if the case is skipped."""
fpcr, x, sp, nzcv, v = case
e, mu = self.emu, self.emu.mu
self.lo, self.hi, self.left, self.h = entry, end, False, 0
mu.reg_write_batch(list(zip(e.xregs, x[:31])) + list(zip(e.qregs, v)) +
[(A.UC_ARM64_REG_SP, sp), (A.UC_ARM64_REG_NZCV, nzcv), (A.UC_ARM64_REG_FPCR, fpcr)])
pc, ok = entry, False
try:
for _ in range(self.MAX_CALLS):
self.at_call = None
mu.emu_start(pc, SENTINEL, count=steps)
if self.at_call is None:
ok = not self.left and mu.reg_read(A.UC_ARM64_REG_PC) == SENTINEL
break
pc = self.stub(self.at_call)
except uc.UcError:
ok = False
if not ok:
mu.mem_write(BUF, self.buf0)
return None
o = mu.reg_read_batch(e.out_regs)
crc = zlib.crc32(bytes(mu.mem_read(BUF, BUF_SIZE)))
mu.mem_write(BUF, self.buf0)
return list(o[:31]), o[31], o[32] & 0xF0000000, o[33], list(o[34:]), crc
def main():
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
ap.add_argument('--exefs', type=Path, default=ROOT / 'build' / 'extracted' / 'v121' / 'exefs')
ap.add_argument('--modules', default='main,subsdk0,sdk')
ap.add_argument('--functions', type=int, default=400, help='per module')
ap.add_argument('--cases', type=int, default=12, help='random states tried per function')
ap.add_argument('--steps', type=int, default=200000, help='Unicorn instruction cap per case')
ap.add_argument('--seed', type=int, default=1)
ap.add_argument('--jobs', type=int, default=os.cpu_count() or 4)
ap.add_argument('--show', type=int, default=20)
ap.add_argument('--patches', type=Path, help='patch table whose spin waits are emitted with their gates')
ap.add_argument('--cflags', default='-march=x86-64-v3' if platform.machine().lower() in ('amd64', 'x86_64') else '')
args = ap.parse_args()
r = random.Random(args.seed)
t0 = time.time()
want = args.modules.split(',')
work = ROOT / 'build' / 'test' / 'func'
work.mkdir(parents=True, exist_ok=True)
images, emitted, tags = [], [], []
mods = layout(args.exefs)
table = patches.load(args.patches, mods) if args.patches else patches.Table()
for name, base, _ in mods:
if name not in want:
continue
mod = Module(name, args.exefs / name, base, 'c')
img = work / f'{name}.img'
img.write_bytes(mod.image)
images.append((base, img, mod.image))
picks, emitter = pick_functions(name, mod, args.functions, r, table.spins_for(name, base))
for entry, end, t in picks:
emitted.append((name, entry, end, emitter))
tags.append(t)
print(f'picked {len(emitted)} functions ({time.time() - t0:.0f}s)', flush=True)
t1 = time.time()
exe = build(work, emitted, args.jobs, args.cflags)
print(f'compiled ({time.time() - t1:.0f}s)', flush=True)
buf0 = make_buffer(r)
sites = set()
for _, entry, end, emitter in emitted:
sites.update(call_sites(emitter.mod, entry, end))
runner = Runner([(base, data) for base, _, data in images], buf0, sites)
cases, expect, tested, skipped = [], [], set(), 0
t2 = time.time()
for k, (name, entry, end, _) in enumerate(emitted):
for _ in range(args.cases):
case = make_case(r)
want_state = runner.run(entry, end, case, args.steps)
if want_state is None:
skipped += 1
continue
fpcr, x, sp, nzcv, v = case
cases.append((k, fpcr, x, sp, nzcv, v, True))
expect.append(want_state)
tested.add(k)
print(f'unicorn: {len(cases)} cases on {len(tested)} functions, {skipped} skipped ({time.time() - t2:.0f}s)',
flush=True)
t3 = time.time()
recs, err = run_harness(exe, work, buf0, cases, [(base, img) for base, img, _ in images], stubs=True)
print(f'harness: {len(recs)} of {len(cases)} cases ({time.time() - t3:.0f}s)')
if err:
k = cases[len(recs)][0] if len(recs) < len(cases) else None
where = f', stopped in {emitted[k][0]} {emitted[k][1]:#x}' if k is not None else ''
print(f'harness failed: {err}{where}')
fails, nfail = {}, 0
for case, want_state, got in zip(cases, expect, recs):
# Generated functions keep NZCV in a local (recomp/flags.py) and leave c->nzcv as it was: flags only count
# where the function reads them, which the registers and memory show.
d = diff_state(got, want_state, check_pc=False, check_mem=True, check_nzcv=False)
if d:
nfail += 1
fails.setdefault(case[0], d[:4])
by_tag, fail_tag = Counter(), Counter()
for k in tested:
by_tag.update(tags[k] or {'plain'})
if k in fails:
fail_tag.update(tags[k] or {'plain'})
print(f'{len(tested)} functions, {len(cases)} cases: {nfail} failing cases, {len(fails)} failing functions '
f'({time.time() - t0:.0f}s total)')
print(f' tested by tag: {dict(by_tag)}; failing by tag: {dict(fail_tag)}')
for k, d in list(fails.items())[:args.show]:
name, entry, end, _ = emitted[k]
print(f' {name:8} {gen.fname(entry)} ({(end - entry) // 4} insns, {sorted(tags[k])}): {"; ".join(d)}')
return 1 if fails or err else 0
if __name__ == '__main__':
sys.exit(main())

162
tests/diff/harness.c Normal file
View File

@@ -0,0 +1,162 @@
// Runs lifted tests and writes the resulting CPU state per case.
// Shared by tests/diff/lift.py, simd.py, atomic.py and func.py; the expected state comes from Unicorn on the Python
// side.
//
// argv: buf.bin cases.bin out.bin image base [image base ...]
// Each image is mapped at its guest address (guest == host), the scratch buffer at BUF.
#include <stdio.h>
#include <stdlib.h>
#ifdef _WIN32
#include <windows.h>
#else
#include <sys/mman.h>
#endif
#include "cpu.h"
extern GuestFn tests[];
extern int ntests;
#define BUF 0x7200000000ull
#define BUF_SIZE 0x40000
#define PTR_LO (BUF + 0x8000)
#define PTR_SPAN 0x20000
static int g_stubs; // In.stubs of the running case
static uint64_t g_stub_h; // digest of every call's state so far
static uint64_t splitmix(uint64_t z) {
z += 0x9E3779B97F4A7C15ull;
z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ull;
z = (z ^ (z >> 27)) * 0x94D049BB133111EBull;
return z ^ (z >> 31);
}
static uint64_t mix(uint64_t h, uint64_t w) { return (h ^ w) * 0x100000001B3ull; }
// The callee of every call in a function under tests/diff/func.py; Runner.stub there is the same function on the
// Unicorn side. Folds the target, X0-X30, SP and V0-V7 into a digest, so a register the caller left unwritten at the
// call changes it, then sets X0-X18, NZCV and V0-V7 from the digest, so a caller reading a register it held across
// the call sees a value the callee could have left. X0-X7 mostly get pointers into the scratch buffer. The digest
// and the call count go to the top of the scratch buffer, which the memory check covers.
static void rt_stub(Cpu *c, uint64_t target) {
uint64_t h = mix(g_stub_h, target);
for (int i = 0; i < 31; i++) h = mix(h, c->x[i]);
h = mix(h, c->sp);
for (int i = 0; i < 8; i++) h = mix(mix(h, (uint64_t)c->v[i]), (uint64_t)(c->v[i] >> 64));
g_stub_h = h;
for (int i = 0; i < 19; i++) {
uint64_t s = splitmix(h + i);
c->x[i] = i < 8 && (s & 3) ? PTR_LO + ((s >> 4) % (PTR_SPAN / 16)) * 16 : s;
}
c->nzcv = splitmix(h + 19) >> 60 << 28;
for (int i = 0; i < 8; i++)
c->v[i] = (__uint128_t)splitmix(h + 33 + 2 * i) << 64 | splitmix(h + 32 + 2 * i);
uint64_t *slot = (uint64_t *)(uintptr_t)(BUF + BUF_SIZE - 16);
slot[0]++;
slot[1] = h;
}
void rt_dispatch(Cpu *c) {
if (g_stubs) rt_stub(c, c->pc);
}
void rt_stub_call(Cpu *c, uint64_t target) { rt_stub(c, target); }
void rt_spin_gate(Cpu *c, uint64_t head) { (void)c; (void)head; }
void rt_svc(Cpu *c, uint32_t imm, uint64_t pc) { (void)c; (void)imm; (void)pc; }
void rt_brk(Cpu *c, uint32_t imm, uint64_t pc) { (void)c; (void)imm; (void)pc; abort(); }
void rt_udf(Cpu *c, uint64_t pc) { (void)c; (void)pc; abort(); }
uint64_t rt_ticks(void) { return 0; }
volatile int g_fnprof;
void fnprof_hit(uint64_t addr, uint64_t lr, uint64_t x0) { (void)addr; (void)lr; (void)x0; }
static void *map_at(uint64_t addr, size_t len) {
len = (len + 0xffff) & ~(size_t)0xffff;
#ifdef _WIN32
void *p = VirtualAlloc((void *)addr, len, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
#else
void *p = mmap((void *)addr, len, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON | MAP_FIXED, -1, 0);
#endif
if (p != (void *)addr) {
fprintf(stderr, "map %llx (+%zx) failed\n", (unsigned long long)addr, len);
exit(1);
}
return p;
}
static void *load_at(const char *path, uint64_t addr) {
FILE *f = fopen(path, "rb");
if (!f) { perror(path); exit(1); }
fseek(f, 0, SEEK_END);
size_t size = (size_t)ftell(f);
fseek(f, 0, SEEK_SET);
void *p = map_at(addr, size);
if (fread(p, 1, size, f) != size) { perror(path); exit(1); }
fclose(f);
return p;
}
static uint32_t crc_tab[256];
static uint32_t crc32_buf(const uint8_t *p, size_t n) {
uint32_t c = 0xffffffffu;
for (size_t i = 0; i < n; i++) c = crc_tab[(c ^ p[i]) & 0xff] ^ (c >> 8);
return c ^ 0xffffffffu;
}
// Layouts match IN_FMT / OUT_FMT in tests/diff/common.py.
struct In { uint64_t k, fpcr, x[32], sp, nzcv; __uint128_t v[32]; uint64_t memchk, stubs; };
struct Out { uint64_t x[32], sp, nzcv, pc, mem; __uint128_t v[32]; };
int main(int argc, char **argv) {
if (argc < 6 || (argc - 4) % 2) {
fprintf(stderr, "usage: %s buf.bin cases.bin out.bin image base [image base ...]\n", argv[0]);
return 2;
}
for (uint32_t i = 0; i < 256; i++) {
uint32_t c = i;
for (int k = 0; k < 8; k++) c = c & 1 ? (c >> 1) ^ 0xEDB88320u : c >> 1;
crc_tab[i] = c;
}
for (int i = 4; i < argc; i += 2) load_at(argv[i], strtoull(argv[i + 1], 0, 16));
uint8_t *buf = map_at(BUF, BUF_SIZE);
FILE *bf = fopen(argv[1], "rb");
if (!bf || fread(buf, 1, BUF_SIZE, bf) != BUF_SIZE) { perror(argv[1]); return 1; }
fclose(bf);
uint8_t *pristine = malloc(BUF_SIZE);
memcpy(pristine, buf, BUF_SIZE);
FILE *f = fopen(argv[2], "rb"), *o = fopen(argv[3], "wb");
if (!f || !o) return 1;
static struct In in;
static struct Out out;
static Cpu c;
while (fread(&in, sizeof in, 1, f) == 1) {
if (in.k >= (uint64_t)ntests) { fprintf(stderr, "bad test index %llu\n", (unsigned long long)in.k); return 1; }
memset(&c, 0, sizeof c);
memcpy(c.x, in.x, sizeof c.x);
c.sp = in.sp;
c.nzcv = in.nzcv;
c.fpcr = in.fpcr;
rt_fpcr_sync(&c);
memcpy(c.v, in.v, sizeof c.v);
c.excl_addr = ~0ull;
g_stubs = (int)in.stubs;
g_stub_h = 0;
tests[in.k](&c);
memcpy(out.x, c.x, sizeof out.x);
out.sp = c.sp;
out.nzcv = c.nzcv & 0xf0000000ull;
out.pc = c.pc;
out.mem = 0;
if (in.memchk) {
out.mem = crc32_buf(buf, BUF_SIZE);
memcpy(buf, pristine, BUF_SIZE);
}
memcpy(out.v, c.v, sizeof out.v);
fwrite(&out, sizeof out, 1, o);
// Flushed per case so a crash leaves the completed cases readable.
fflush(o);
}
fclose(o);
return 0;
}

249
tests/diff/lift.py Normal file
View File

@@ -0,0 +1,249 @@
"""Differential test of the lifter on the game's own instructions: lifted C against Unicorn, one instruction at a time.
Samples distinct instruction words from the .text of each ExeFS module, one per opcode signature first, lifts each
with the portable FP/SIMD lowering, runs it from random register state, and compares X0-X30, SP, NZCV, PC (for
branches), V0-V31 and the scratch buffer (for loads and stores) with Unicorn as a Cortex-A57.
Skipped: system instructions and exceptions, whose single step cannot be compared, and exclusives and ARMv8.1
atomics, which atomic.py covers. Encodings the lifter rejects are counted per module and mnemonic; --coverage lifts
every distinct word instead of testing.
Adapted from LAS Recomp tools/test_lift.py @ da35b83 (MIT, Copyright (c) 2026 Peiikai).
Usage: lift.py [--exefs DIR] [--modules main,sdk,subsdk0] [--encodings N] [--cases K] [--seed S] [--jobs J]
lift.py --coverage
"""
import argparse
import os
import platform
import random
import sys
import time
import zlib
from collections import Counter
from pathlib import Path
import numpy as np
import unicorn as uc
from common import BUF, BUF_SIZE, ROOT, Emu, build_harness, diff_state, dis, mnemonic, rand_fpcr, rand_v, rand_x, \
run_harness
from lifter import Lifter, Unsupported, bits
from nso import Nso
BUF_MID = BUF + 0x10000
# Test load addresses, below 2^39 and clear of BUF.
BASES = {'main': 0x7000000000, 'subsdk0': 0x7008000000, 'sdk': 0x7010000000}
def is_mem(w):
return bits(w, 25, 1) == 0 and bits(w, 27, 1) == 1
def classify(w):
"""'skip' for instructions whose single-step behaviour cannot be compared."""
if (w & 0xFFC00000) == 0xD5000000: # system: hints, barriers, MRS/MSR, cache maintenance
return 'skip'
if (w & 0x3F000000) == 0x08000000 and (bits(w, 23, 1) == 0 or bits(w, 21, 1)):
return 'skip' # exclusives need a paired load; CAS is ARMv8.1; both are covered by atomic.py
if (w & 0x3F200C00) == 0x38200000: # LSE atomics, ARMv8.1: atomic.py
return 'skip'
if (w & 0xFF000000) == 0xD4000000: # SVC, BRK, HLT
return 'skip'
if (w & 0xFFFF0000) == 0: # UDF
return 'skip'
return 'ok'
def class_keys(words):
"""Opcode signature per word: register fields (Rd, Rn, Rm) cleared."""
return words & np.uint32(0xFFE0FC00)
def to_test_body(ins, pc):
if ins.kind == 'op':
return ins.code
nxt = f'0x{pc + 4:x}ull'
if ins.kind == 'b':
return f'c->pc = 0x{ins.target:x}ull;'
if ins.kind == 'bl':
return f'c->x[30] = {nxt}; c->pc = 0x{ins.target:x}ull;'
if ins.kind == 'bcond':
return f'c->pc = ({ins.cond}) ? 0x{ins.target:x}ull : {nxt};'
if ins.kind == 'br':
return f'c->pc = {ins.target};'
if ins.kind == 'blr':
return f'{{ uint64_t t = {ins.target}; c->x[30] = {nxt}; c->pc = t; }}'
if ins.kind == 'ret':
return 'c->pc = c->x[30];'
raise Unsupported(ins.kind)
def pick_module(name, nso, base, count, r, unsupported):
"""Distinct words of .text, one per opcode signature first, lifted. Returns [(word, pc, body)]."""
lo, size = nso.text
words = np.frombuffer(bytes(nso.image[lo:lo + size]), dtype='<u4')
uniq, first = np.unique(words, return_index=True)
keys = class_keys(uniq)
order = list(range(len(uniq)))
r.shuffle(order)
seen, primary, rest = set(), [], []
for i in order:
k = int(keys[i])
(rest if k in seen else primary).append(i)
seen.add(k)
lifter = Lifter(base, 'c')
picks = []
for i in primary + rest:
if len(picks) >= count:
break
w = int(uniq[i])
if classify(w) == 'skip':
continue
pc = base + lo + int(first[i]) * 4
try:
body = to_test_body(lifter.lift(w, pc), pc)
except Unsupported:
unsupported[(name, mnemonic(w))] += 1
continue
picks.append((w, pc, body))
print(f'{name}: {len(words)} words, {len(uniq)} distinct, {len(seen)} signatures, {len(picks)} picked')
return picks
def coverage(name, nso, base):
"""Lift every distinct word of .text. Returns {kind mnemonic: [distinct words, occurrences]} of words the lifter
rejects (Unsupported) or turns into an undefined-instruction trap (rt_udf)."""
lo, size = nso.text
words = np.frombuffer(bytes(nso.image[lo:lo + size]), dtype='<u4')
uniq, first, counts = np.unique(words, return_index=True, return_counts=True)
lifter = Lifter(base, 'c')
rejected = {}
for w, i, n in zip(uniq.tolist(), first.tolist(), counts.tolist()):
try:
ins = lifter.lift(w, base + lo + i * 4)
if ins.kind != 'trap' or not ins.code.startswith('rt_udf'):
continue
key = f'udf {mnemonic(w)}'
except Unsupported:
key = f'unsupported {mnemonic(w)}'
e = rejected.setdefault(key, [0, 0])
e[0] += 1
e[1] += n
total = sum(e[1] for e in rejected.values())
print(f'{name}: {len(words)} words, {len(uniq)} distinct, {total} rejected '
f'({sum(e[0] for e in rejected.values())} distinct)')
for mn, (d, n) in sorted(rejected.items(), key=lambda kv: -kv[1][1]):
print(f' {mn:24} {n:8} occurrences {d:6} distinct')
return rejected
def make_case(r, w):
if is_mem(w):
x = [BUF_MID + r.randrange(0, 0x100, 8) for _ in range(31)]
if (bits(w, 21, 1) and bits(w, 10, 2) == 2) or (w & 0xBFA00000) == 0x0C800000: # register offset
rm = bits(w, 16, 5)
if rm != bits(w, 5, 5) and rm < 31:
x[rm] = r.randrange(0, 0x100)
else:
x = [rand_x(r) for _ in range(31)]
sp = BUF_MID + r.randrange(0, 0x100, 16)
nzcv = r.getrandbits(4) << 28
v = [rand_v(r) for _ in range(32)]
fpcr = rand_fpcr(r)
return fpcr, x, sp, nzcv, v
def main():
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
ap.add_argument('--exefs', type=Path, default=ROOT / 'build' / 'extracted' / 'v121' / 'exefs')
ap.add_argument('--modules', default='main,sdk,subsdk0')
ap.add_argument('--encodings', type=int, default=20000, help='per module')
ap.add_argument('--cases', type=int, default=3, help='random states per encoding')
ap.add_argument('--seed', type=int, default=1)
ap.add_argument('--jobs', type=int, default=os.cpu_count() or 4)
ap.add_argument('--show', type=int, default=40)
ap.add_argument('--cflags', default='-march=x86-64-v3' if platform.machine().lower() in ('amd64', 'x86_64') else '')
ap.add_argument('--coverage', action='store_true',
help='only lift every distinct word and list the rejected ones (capstone "?" = undecodable data)')
args = ap.parse_args()
r = random.Random(args.seed)
t0 = time.time()
names = args.modules.split(',')
if args.coverage:
for name in names:
coverage(name, Nso(str(args.exefs / name)), BASES[name])
print(f'({time.time() - t0:.0f}s)')
return 0
work = ROOT / 'build' / 'test' / 'lift'
work.mkdir(parents=True, exist_ok=True)
images, picks, unsupported = [], [], Counter()
for name in names:
nso = Nso(str(args.exefs / name))
base = BASES[name]
img = work / f'{name}.img'
img.write_bytes(bytes(nso.image))
images.append((base, img, nso.image))
picks += [(name,) + p for p in pick_module(name, nso, base, args.encodings, r, unsupported)]
print(f'lifted {len(picks)} encodings ({time.time() - t0:.0f}s)')
t1 = time.time()
exe = build_harness(work, [p[3] for p in picks], args.jobs, args.cflags)
print(f'compiled ({time.time() - t1:.0f}s)')
emu = Emu([(base, data) for base, _, data in images])
mu = emu.mu
buf0 = bytes(r.getrandbits(8) for _ in range(4096)) * (BUF_SIZE // 4096)
mu.mem_write(BUF, buf0)
cases, expect = [], []
t2 = time.time()
for k, (name, w, pc, _) in enumerate(picks):
mem = is_mem(w)
for _ in range(args.cases):
fpcr, x, sp, nzcv, v = make_case(r, w)
try:
ox, osp, onz, opc, ov = emu.run(pc, fpcr, x, sp, nzcv, v)
except uc.UcError:
if mem:
mu.mem_write(BUF, buf0)
continue
mh = 0
if mem:
mh = zlib.crc32(bytes(mu.mem_read(BUF, BUF_SIZE)))
mu.mem_write(BUF, buf0)
cases.append((k, fpcr, x, sp, nzcv, v, mem))
expect.append((ox, osp, onz, opc, ov, mh))
print(f'unicorn: {len(cases)} cases ({time.time() - t2:.0f}s)')
t3 = time.time()
recs, err = run_harness(exe, work, buf0, cases, [(base, img) for base, img, _ in images])
print(f'harness: {len(recs)} of {len(cases)} cases ({time.time() - t3:.0f}s)')
if err:
k = cases[len(recs)][0] if len(recs) < len(cases) else None
where = f', stopped at {picks[k][1]:08x} {dis(picks[k][1])}' if k is not None else ''
print(f'harness failed: {err}{where}')
fails, nfail = {}, 0
for case, want, got in zip(cases, expect, recs):
k = case[0]
name, w, pc, _ = picks[k]
d = diff_state(got, want, check_pc=got[3] != 0, check_mem=case[6])
if d:
nfail += 1
fails.setdefault(w, (name, pc, d[:4]))
print('unsupported encodings (distinct words) by module and mnemonic:')
for (name, mn), n in sorted(unsupported.items(), key=lambda kv: (kv[0][0], -kv[1])):
print(f' {name:8} {mn:10} {n}')
print(f'{len(picks)} encodings, {len(cases)} cases: {nfail} failing cases, {len(fails)} failing encodings '
f'({time.time() - t0:.0f}s total)')
if fails:
print(' failing by mnemonic:', dict(Counter(mnemonic(w) for w in fails).most_common(40)))
for w, (name, pc, d) in list(fails.items())[:args.show]:
print(f' {name:8} {w:08x} {dis(w, pc):40} {"; ".join(d)}')
return 1 if fails or err else 0
if __name__ == '__main__':
sys.exit(main())

242
tests/diff/simd.py Normal file
View File

@@ -0,0 +1,242 @@
"""Differential test of the portable FP/SIMD lowering (lifter simd='c') against Unicorn as a Cortex-A57.
Needs no game files. Enumerates every opcode bit pattern of the FP/SIMD data-processing space (bits 27..25 = 111),
the FP/SIMD load/store space and CRC32 with random register fields, and compares which encodings the lifter and
Unicorn accept. Then a stratified sample of encodings both accept is compiled and run with random interesting
operands (NaNs, infinities, denormals, +-0, saturation edges, random FPCR.DN/FZ/RMode/AHP): X0-X30, SP, NZCV, V0-V31
and, for stores, memory must match. FPSR is not compared; the lowering does not model the cumulative exception bits
or QC.
Adapted from LAS Recomp tools/test_simd.py @ da35b83 (MIT, Copyright (c) 2026 Peiikai).
Usage: simd.py [--encodings N] [--cases K] [--seed S] [--jobs J] [--no-validity] [--filter HEX/MASK]
"""
import argparse
import functools
import os
import platform
import random
import struct
import sys
import time
import zlib
from collections import Counter, defaultdict
import unicorn as uc
from unicorn import arm64_const as A
from common import BUF, BUF_SIZE, ROOT, UC_INVALID, Emu, build_harness, diff_state, dis, mnemonic, rand_fpcr, \
rand_v, rand_x, run_harness
from lifter import Lifter, Unsupported
print = functools.partial(print, flush=True)
BASE_AREA = BUF + 0x8000
CODE, CODE_SIZE = 0x7210000000, 0x400000
def is_ldst(w):
return (w >> 25) & 1 == 0 and (w >> 27) & 1 == 1
def is_store(w):
if not is_ldst(w):
return False
if (w & 0x3B000000) == 0x18000000: # literal
return False
return not (w >> 22) & 1 # L bit, same position for structures, pairs and single registers
def is_literal(w):
return (w & 0x3F000000) == 0x1C000000
def class_key(w):
"""Opcode signature: register fields (Rd, Rn, Rm) cleared."""
if is_ldst(w):
return w & 0xFFE0FC00 if (w & 0xBE000000) == 0x0C000000 else w & 0xFFFFFC00
return w & 0xFFE0FC00
def candidates(r):
"""Every opcode bit pattern of the FP/SIMD data-processing and load/store spaces, random registers."""
out = []
for p in range(1 << 19):
hi, mid = p >> 15, p & 0x7FFF
rr = r.getrandbits(10)
out.append((hi << 28) | (0b111 << 25) | (mid << 10) | rr) # data processing
out.append((hi << 28) | (1 << 27) | (1 << 26) | (mid << 10) | rr) # loads/stores
for sf in (0, 1):
for C in (0, 1):
for sz in range(4):
for _ in range(4):
out.append((sf << 31) | (0b0011010110 << 21) | (r.getrandbits(5) << 16) | (0b010 << 13) |
(C << 12) | (sz << 10) | r.getrandbits(10))
return out
def validity(lifter, ws, emu):
"""Map word -> (lifter accepts, Unicorn accepts); words are placed at CODE + 4i."""
mu = emu.mu
res = {}
step = CODE_SIZE // 4
for base in range(0, len(ws), step):
chunk = ws[base:base + step]
mu.mem_write(CODE, struct.pack(f'<{len(chunk)}I', *chunk))
mu.ctl_remove_cache(CODE, CODE + CODE_SIZE)
state = list(zip(emu.xregs, [BASE_AREA] * 31)) + [(A.UC_ARM64_REG_SP, BASE_AREA)]
for i, w in enumerate(chunk):
pc = CODE + 4 * i
try:
lifter.lift(w, pc)
lok = True
except Unsupported:
lok = False
if i % 32 == 0:
mu.reg_write_batch(state)
try:
mu.emu_start(pc, pc + 4, count=1)
uok = True
except uc.UcError as e:
uok = e.errno not in UC_INVALID
res[w] = (lok, uok)
return res
def make_case(r, w):
fpcr = rand_fpcr(r)
v = [rand_v(r) for _ in range(32)]
nzcv = r.getrandbits(4) << 28
sp = BASE_AREA + r.randrange(0, 0x100, 16)
if is_ldst(w):
x = [BASE_AREA + r.randrange(0, 0x100) for _ in range(31)]
rm = (w >> 16) & 31
if rm < 31 and rm != (w >> 5) & 31:
reg_off = (w & 0x3B200C00) == 0x38200800 # register offset
post_reg = (w & 0xBF800000) in (0x0C800000, 0x0D800000)
if reg_off or post_reg:
x[rm] = r.randrange(0, 0x100)
else:
x = [rand_x(r) for _ in range(31)]
return fpcr, x, sp, nzcv, v
def main():
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
ap.add_argument('--encodings', type=int, default=20000)
ap.add_argument('--cases', type=int, default=8)
ap.add_argument('--seed', type=int, default=1)
ap.add_argument('--jobs', type=int, default=os.cpu_count() or 4)
ap.add_argument('--no-validity', action='store_true', help='skip the full validity enumeration')
ap.add_argument('--filter', help='only encodings with (w & MASK) == VAL, as VAL/MASK in hex')
ap.add_argument('--show', type=int, default=40)
ap.add_argument('--cflags', default='-march=x86-64-v3' if platform.machine().lower() in ('amd64', 'x86_64') else '')
args = ap.parse_args()
r = random.Random(args.seed)
t0 = time.time()
lifter = Lifter(0, 'c')
emu = Emu([(CODE, bytes(CODE_SIZE))])
cands = candidates(r)
if args.filter:
val, mask = (int(x, 16) for x in args.filter.split('/'))
cands = [w for w in cands if (w & mask) == val]
if args.no_validity:
val = validity(lifter, r.sample(cands, min(len(cands), args.encodings * 4)), emu)
else:
val = validity(lifter, cands, emu)
agree = Counter()
only = defaultdict(list)
for w, (lok, uok) in val.items():
agree[(lok, uok)] += 1
if lok != uok:
only['lifter' if lok else 'unicorn'].append(w)
print(f'validity: {len(val)} encodings, both valid {agree[(True, True)]}, both invalid {agree[(False, False)]}, '
f'lifter-only {agree[(True, False)]}, unicorn-only {agree[(False, True)]} ({time.time() - t0:.0f}s)')
for kind in ('lifter', 'unicorn'):
ws = only[kind]
if ws:
print(f' {kind}-only by mnemonic: {dict(Counter(mnemonic(w) for w in ws).most_common(30))}')
for w in ws[:args.show // 2]:
print(f' {w:08x} {dis(w)}')
# Stratified sample of valid encodings: one per opcode signature first.
valid = [w for w, (lok, uok) in val.items() if lok and uok]
r.shuffle(valid)
seen, pick = set(), []
for w in valid:
k = class_key(w)
if k not in seen:
seen.add(k)
pick.append(w)
picked = set(pick)
pick = (pick + [w for w in valid if w not in picked])[:args.encodings]
# Placed at CODE + 4i; literal loads get an offset into the code region.
for i, w in enumerate(pick):
pc = CODE + 4 * i
if is_literal(w):
tgt = CODE + r.randrange(0, (len(pick) * 4 + 0x1000) & ~3, 4)
pick[i] = (w & ~(0x7FFFF << 5)) | ((((tgt - pc) >> 2) & 0x7FFFF) << 5)
code_bytes = struct.pack(f'<{len(pick)}I', *pick) + bytes(0x1000 + 64)
bodies = [lifter.lift(w, CODE + 4 * i).code for i, w in enumerate(pick)]
work = ROOT / 'build' / 'test' / 'simd'
work.mkdir(parents=True, exist_ok=True)
t1 = time.time()
exe = build_harness(work, bodies, args.jobs, args.cflags)
print(f'compiled {len(bodies)} tests ({time.time() - t1:.0f}s)')
mu = emu.mu
mu.mem_write(CODE, code_bytes)
mu.ctl_remove_cache(CODE, CODE + CODE_SIZE)
buf0 = bytes(r.getrandbits(8) for _ in range(4096)) * (BUF_SIZE // 4096)
mu.mem_write(BUF, buf0)
code_path = work / 'code.bin'
code_path.write_bytes(code_bytes)
cases, expect = [], []
t2 = time.time()
for k, w in enumerate(pick):
pc = CODE + 4 * k
st = is_store(w)
for _ in range(args.cases):
fpcr, x, sp, nzcv, v = make_case(r, w)
try:
ox, osp, onz, opc, ov = emu.run(pc, fpcr, x, sp, nzcv, v)
except uc.UcError:
if st:
mu.mem_write(BUF, buf0)
continue
mh = 0
if st:
mh = zlib.crc32(bytes(mu.mem_read(BUF, BUF_SIZE)))
mu.mem_write(BUF, buf0)
cases.append((k, fpcr, x, sp, nzcv, v, st))
expect.append((ox, osp, onz, opc, ov, mh))
print(f'unicorn: {len(cases)} cases ({time.time() - t2:.0f}s)')
t3 = time.time()
recs, err = run_harness(exe, work, buf0, cases, [(CODE, code_path)])
print(f'harness: {len(recs)} of {len(cases)} cases ({time.time() - t3:.0f}s)')
if err:
k = cases[len(recs)][0] if len(recs) < len(cases) else None
print(f'harness failed: {err}' + (f', stopped at {pick[k]:08x} {dis(pick[k])}' if k is not None else ''))
fails, nfail = {}, 0
for case, want, got in zip(cases, expect, recs):
k, v, st = case[0], case[5], case[6]
d = diff_state(got, want, check_pc=False, check_mem=st)
if d:
nfail += 1
w = pick[k]
if w not in fails:
ins = [f'v{j}={v[j]:032x}' for j in sorted({(w >> 5) & 31, (w >> 16) & 31, w & 31})]
fails[w] = (case[1], d[:3], ins)
print(f'{len(pick)} encodings, {len(cases)} cases: {nfail} failing cases, {len(fails)} failing encodings '
f'({time.time() - t0:.0f}s total)')
if fails:
print(' failing by mnemonic:', dict(Counter(mnemonic(w) for w in fails).most_common(40)))
for w, (fpcr, d, ins) in list(fails.items())[:args.show]:
print(f' {w:08x} {dis(w):40} fpcr={fpcr:x} {"; ".join(d)}')
print(f' {" ".join(ins)}')
return 1 if fails or err else 0
if __name__ == '__main__':
sys.exit(main())

147
tests/excl_mt.py Normal file
View File

@@ -0,0 +1,147 @@
"""Multithreaded test of the exclusive monitor helpers in cpu.h (ldxN/stxN, ldxp128/stxp128).
- ABA: thread A loads X exclusively, thread B changes X->Y->X with its own exclusive pairs, then A's store exclusive
must fail. Checked for the 32-bit, 64-bit and 128-bit pair forms, plus a control where B touches nothing.
- Contention: N threads increment one counter with load/store-exclusive retry loops; the total must be exact.
Needs no game files. Usage: excl_mt.py [--runtime DIR] [--threads N] [--iters K]
"""
import argparse
import os
import subprocess
import sys
import tempfile
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent / 'diff'))
from common import MSYS_BIN, ROOT, compiler # noqa: E402
SRC = r'''
#include <pthread.h>
#include <stdio.h>
#include <stdlib.h>
#include "cpu.h"
static volatile int step;
static uint64_t cell[4] __attribute__((aligned(64)));
static int nthreads, iters;
static int fails;
static void wait_step(int s) { while (__atomic_load_n(&step, __ATOMIC_ACQUIRE) != s) {} }
static void set_step(int s) { __atomic_store_n(&step, s, __ATOMIC_RELEASE); }
// Thread B: X -> Y -> X with exclusive pairs of the given width (0 = do nothing, the control).
static int aba_width;
static void *aba_b(void *arg) {
(void)arg;
Cpu c = {0};
uint64_t a = (uint64_t)(uintptr_t)cell;
wait_step(1);
for (int k = 0; aba_width && k < 2; k++) {
for (;;) {
if (aba_width == 32) { uint32_t v = (uint32_t)ldx32(&c, a); if (!stx32(&c, a, v ^ 0x5a5a5a5au)) break; }
else if (aba_width == 64) { uint64_t v = ldx64(&c, a); if (!stx64(&c, a, v ^ 0x5a5a5a5a5a5a5a5aull)) break; }
else { __uint128_t v = ldxp128(&c, a); if (!stxp128(&c, a, (uint64_t)v ^ 1, (uint64_t)(v >> 64) ^ 2)) break; }
}
}
set_step(2);
return 0;
}
static void aba_case(int width, int expect_fail) {
pthread_t t;
Cpu c = {0};
uint64_t a = (uint64_t)(uintptr_t)cell;
cell[0] = 0x1111222233334444ull;
cell[1] = 0x5555666677778888ull;
aba_width = width;
set_step(0);
pthread_create(&t, 0, aba_b, 0);
uint32_t st;
if (width == 128) {
__uint128_t v = ldxp128(&c, a);
set_step(1);
wait_step(2);
st = stxp128(&c, a, (uint64_t)v + 7, (uint64_t)(v >> 64));
} else if (width == 32) {
uint32_t v = (uint32_t)ldx32(&c, a);
set_step(1);
wait_step(2);
st = stx32(&c, a, v + 7);
} else {
uint64_t v = ldx64(&c, a);
set_step(1);
wait_step(2);
st = stx64(&c, a, v + 7);
}
pthread_join(t, 0);
int want = expect_fail != 0;
printf("aba width %3d: stx status %u, want %d %s\n", width, st, want, st == (uint32_t)want ? "ok" : "FAIL");
if (st != (uint32_t)want) fails++;
}
static int inc_width;
static void *inc(void *arg) {
(void)arg;
Cpu c = {0};
uint64_t a = (uint64_t)(uintptr_t)cell;
for (int i = 0; i < iters; i++) {
for (;;) {
if (inc_width == 32) { uint32_t v = (uint32_t)ldx32(&c, a); if (!stx32(&c, a, v + 1)) break; }
else if (inc_width == 64) { uint64_t v = ldx64(&c, a); if (!stx64(&c, a, v + 1)) break; }
else { __uint128_t v = ldxp128(&c, a); if (!stxp128(&c, a, (uint64_t)v + 1, (uint64_t)(v >> 64) + 3)) break; }
}
}
return 0;
}
static void inc_case(int width) {
pthread_t t[64];
cell[0] = cell[1] = 0;
inc_width = width;
for (int i = 0; i < nthreads; i++) pthread_create(&t[i], 0, inc, 0);
for (int i = 0; i < nthreads; i++) pthread_join(t[i], 0);
uint64_t want = (uint64_t)nthreads * iters;
uint64_t got = width == 32 ? (uint32_t)cell[0] : cell[0];
int ok = got == (width == 32 ? (uint32_t)want : want) && (width != 128 || cell[1] == 3 * want);
printf("contention width %3d: %d threads x %d, counter %llu (hi %llu), want %llu %s\n", width, nthreads, iters,
(unsigned long long)got, (unsigned long long)cell[1], (unsigned long long)want, ok ? "ok" : "FAIL");
if (!ok) fails++;
}
int main(int argc, char **argv) {
nthreads = atoi(argv[1]);
iters = atoi(argv[2]);
aba_case(0, 0);
aba_case(32, 1);
aba_case(64, 1);
aba_case(128, 1);
inc_case(32);
inc_case(64);
inc_case(128);
printf("%s\n", fails ? "FAILED" : "PASSED");
return fails != 0;
}
'''
def main():
ap = argparse.ArgumentParser()
ap.add_argument('--runtime', type=Path, default=ROOT / 'runtime', help='directory holding cpu.h')
ap.add_argument('--threads', type=int, default=8)
ap.add_argument('--iters', type=int, default=200000)
args = ap.parse_args()
env = dict(os.environ)
if os.name == 'nt' and MSYS_BIN.exists():
env['PATH'] = str(MSYS_BIN) + os.pathsep + env.get('PATH', '')
with tempfile.TemporaryDirectory() as tmp:
src = Path(tmp) / 'excl_mt.c'
exe = Path(tmp) / ('excl_mt.exe' if os.name == 'nt' else 'excl_mt')
src.write_text(SRC)
subprocess.check_call([compiler(), '-O2', '-march=x86-64-v3', '-fno-strict-aliasing', '-I', str(args.runtime),
str(src), '-o', str(exe), '-lpthread'], env=env)
return subprocess.run([str(exe), str(args.threads), str(args.iters)], env=env).returncode
if __name__ == '__main__':
sys.exit(main())

111
tests/flags_check.c Normal file
View File

@@ -0,0 +1,111 @@
// The function-local flag helpers of runtime/cpu.h (Flags, fc_*, fnz_*, nz_*, adc*f) against the packed helpers
// (subs64() and friends, COND_xx), which tests/diff/lift.py checks against Unicorn. Built and run by
// tests/flags_check.py; prints the first mismatches and exits 1 on any.
#include <stdio.h>
#include "cpu.h"
static const uint64_t EDGE[] = {0, 1, 2, 0x7f, 0x80, 0xff, 0x7fff, 0x8000, 0xffff, 0x7fffffff, 0x80000000,
0x80000001, 0xfffffffe, 0xffffffff, 0x100000000ull, 0x7fffffffffffffffull,
0x8000000000000000ull, 0x8000000000000001ull, 0xfffffffffffffffeull,
0xffffffffffffffffull, 0x123456789abcdef0ull, 0xfedcba9876543210ull};
#define NEDGE (sizeof EDGE / sizeof EDGE[0])
static uint64_t rng = 0x9E3779B97F4A7C15ull;
static uint64_t next(void) {
rng ^= rng << 13, rng ^= rng >> 7, rng ^= rng << 17;
return rng;
}
static int fails;
static void bad(const char *what, int cc, uint64_t a, uint64_t b, uint64_t got, uint64_t want) {
if (fails++ < 20)
printf("%s cc %d a %llx b %llx: got %llx want %llx\n", what, cc, (unsigned long long)a, (unsigned long long)b,
(unsigned long long)got, (unsigned long long)want);
}
static int cond(const Cpu *c, int cc) {
switch (cc) {
case CC_EQ: return COND_EQ(c);
case CC_NE: return COND_NE(c);
case CC_CS: return COND_CS(c);
case CC_CC: return COND_CC(c);
case CC_MI: return COND_MI(c);
case CC_PL: return COND_PL(c);
case CC_VS: return COND_VS(c);
case CC_VC: return COND_VC(c);
case CC_HI: return COND_HI(c);
case CC_LS: return COND_LS(c);
case CC_GE: return COND_GE(c);
case CC_LT: return COND_LT(c);
case CC_GT: return COND_GT(c);
default: return COND_LE(c);
}
}
// The condition argument is not constant here, so this also covers the switch default paths.
#define CHECK(NAME, OLD, NEW, FC, FNZ) \
do { \
Cpu c = {0}; \
Flags f = {0}; \
uint64_t ro = OLD, rn = NEW; \
if (ro != rn) bad(NAME " result", -1, a, b, rn, ro); \
for (int cc = 0; cc < 14; cc++) \
if (!FC(&f, cc) != !cond(&c, cc)) bad(NAME, cc, a, b, FC(&f, cc), cond(&c, cc)); \
f.nz = 0; \
if (FNZ(&f) != c.nzcv) bad(NAME " packed", -1, a, b, FNZ(&f), c.nzcv); \
Flags g = {.nz = c.nzcv}; \
for (int cc = 0; cc < 14; cc++) \
if (!fc_nz(&g, cc) != !cond(&c, cc)) bad(NAME " nz", cc, a, b, fc_nz(&g, cc), cond(&c, cc)); \
} while (0)
static void one(uint64_t a, uint64_t b) {
uint32_t a32 = (uint32_t)a, b32 = (uint32_t)b;
CHECK("sub64", subs64(&c, a, b), fsub64(&f, a, b), fc_sub64, fnz_sub64);
CHECK("sub32", subs32(&c, a32, b32), fsub32(&f, a32, b32), fc_sub32, fnz_sub32);
CHECK("add64", adds64(&c, a, b), fadd64(&f, a, b), fc_add64, fnz_add64);
CHECK("add32", adds32(&c, a32, b32), fadd32(&f, a32, b32), fc_add32, fnz_add32);
CHECK("logic64", logic64(&c, a & b), flogic64(&f, a & b), fc_logic64, fnz_logic64);
CHECK("logic32", logic32(&c, a32 & b32), flogic32(&f, a32 & b32), fc_logic32, fnz_logic32);
{
Cpu c = {0};
if (nz_sub64(a, b) != (subs64(&c, a, b), c.nzcv)) bad("nz_sub64", -1, a, b, nz_sub64(a, b), c.nzcv);
if (nz_sub32(a32, b32) != (subs32(&c, a32, b32), c.nzcv)) bad("nz_sub32", -1, a, b, nz_sub32(a32, b32), c.nzcv);
if (nz_add64(a, b) != (adds64(&c, a, b), c.nzcv)) bad("nz_add64", -1, a, b, nz_add64(a, b), c.nzcv);
if (nz_add32(a32, b32) != (adds32(&c, a32, b32), c.nzcv)) bad("nz_add32", -1, a, b, nz_add32(a32, b32), c.nzcv);
}
for (int cin = 0; cin < 2; cin++) {
Cpu c = {.nzcv = (uint64_t)cin << 29};
Flags f = {0};
if (adc64f(cin, a, b) != adc64(&c, a, b)) bad("adc64", cin, a, b, adc64f(cin, a, b), adc64(&c, a, b));
if (sbc64f(cin, a, b) != sbc64(&c, a, b)) bad("sbc64", cin, a, b, sbc64f(cin, a, b), sbc64(&c, a, b));
if (adc32f(cin, a32, b32) != adc32(&c, a32, b32)) bad("adc32", cin, a, b, adc32f(cin, a32, b32), adc32(&c, a32, b32));
if (sbc32f(cin, a32, b32) != sbc32(&c, a32, b32)) bad("sbc32", cin, a, b, sbc32f(cin, a32, b32), sbc32(&c, a32, b32));
uint64_t r;
#define CARRY(NAME, NEW, OLD) \
c.nzcv = (uint64_t)cin << 29; \
r = NEW; \
if (r != OLD || f.nz != c.nzcv) bad(NAME, cin, a, b, f.nz, c.nzcv);
CARRY("adcs64", adcs64f(&f, cin, a, b), adcs64(&c, a, b))
CARRY("sbcs64", sbcs64f(&f, cin, a, b), sbcs64(&c, a, b))
CARRY("adcs32", adcs32f(&f, cin, a32, b32), adcs32(&c, a32, b32))
CARRY("sbcs32", sbcs32f(&f, cin, a32, b32), sbcs32(&c, a32, b32))
}
}
int main(void) {
long n = 0;
for (size_t i = 0; i < NEDGE; i++)
for (size_t j = 0; j < NEDGE; j++, n++) one(EDGE[i], EDGE[j]);
for (int k = 0; k < 2000000; k++, n++) {
uint64_t a = next(), b = next();
switch (k & 3) {
case 1: b = a; break; // equal operands
case 2: b = a + (next() & 3) - 1; break; // neighbours
case 3: a &= 0xff, b &= 0xff; break; // small values
}
one(a, b);
}
printf("%ld operand pairs, %d mismatches\n", n, fails);
return fails != 0;
}

32
tests/flags_check.py Normal file
View File

@@ -0,0 +1,32 @@
"""Build and run tests/flags_check.c: runtime/cpu.h's function-local flag helpers against its packed flag helpers.
Usage: flags_check.py
"""
import os
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests' / 'diff'))
from common import MSYS_BIN, compiler # noqa: E402
def main():
work = ROOT / 'build' / 'test' / 'flags'
work.mkdir(parents=True, exist_ok=True)
exe = work / ('flags_check.exe' if os.name == 'nt' else 'flags_check')
env = dict(os.environ)
if os.name == 'nt' and MSYS_BIN.exists():
env['PATH'] = str(MSYS_BIN) + os.pathsep + env.get('PATH', '')
for opt in ('-O0', '-O1'):
subprocess.check_call([compiler(), opt, '-fno-strict-aliasing', '-I', str(ROOT / 'runtime'),
str(ROOT / 'tests' / 'flags_check.c'), '-o', str(exe)], env=env)
print(opt, end=': ', flush=True)
if subprocess.run([str(exe)], env=env).returncode:
return 1
return 0
if __name__ == '__main__':
sys.exit(main())

79
tests/installer_e2e.py Normal file
View File

@@ -0,0 +1,79 @@
"""End-to-end run of the installer in dev mode with the game files named in config.local.toml.
Reads input.base, the 1.2.1 entry of input.updates (the one build/extracted/v121 was extracted from, else the only
one) and input.keys, and passes them to `python -m installer` started the way the release launcher starts it: the
embeddable Python (build/py314-embed) with this checkout on sys.path. The paths are never printed. Events go to
<events> (JSON lines); stdout gets each step's state and time, warnings and errors.
python tests/installer_e2e.py --dest build/install-test --lang en-US [--events FILE] [--no-inputs] [-- extra args]
--no-inputs leaves the game files out (update mode on an existing folder).
"""
import argparse
import json
import subprocess
import sys
import tomllib
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
EMBED = ROOT / 'build' / 'py314-embed' / 'py' / 'python.exe'
BOOT = ('import runpy, sys; sys.path.insert(0, sys.argv.pop(1)); '
"runpy.run_module('installer', run_name='__main__', alter_sys=True)")
def game_inputs():
config = tomllib.loads((ROOT / 'config.local.toml').read_text(encoding='utf-8'))['input']
updates = [Path(u) for u in config.get('updates', [])]
manifest = ROOT / 'build' / 'extracted' / 'v121' / 'extraction-manifest.json'
update = None
if manifest.is_file():
source = json.loads(manifest.read_text())['source']
update = next((u for u in updates if str(u.resolve()) == source), None)
if update is None and len(updates) == 1:
update = updates[0]
if update is None:
sys.exit('installer_e2e: cannot tell which input.updates entry is 1.2.1')
return ['--base', config['base'], '--update', str(update), '--keys', config['keys']]
def main():
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
ap.add_argument('--dest', type=Path, required=True)
ap.add_argument('--lang')
ap.add_argument('--events', type=Path)
ap.add_argument('--no-inputs', action='store_true')
ap.add_argument('extra', nargs='*')
a = ap.parse_args()
python = EMBED if EMBED.is_file() else Path(sys.executable)
command = [str(python), '-c', BOOT, str(ROOT), '--dest', str(a.dest.resolve())]
if a.lang:
command += ['--lang', a.lang]
if not a.no_inputs:
command += game_inputs()
command += a.extra
events = a.events or a.dest.resolve().parent / f'{a.dest.name}-events.jsonl'
with open(events, 'w', encoding='utf-8') as log:
proc = subprocess.Popen(command, stdout=subprocess.PIPE, text=True, encoding='utf-8', cwd=ROOT)
for line in proc.stdout:
log.write(line)
log.flush()
try:
e = json.loads(line)
except ValueError:
print('non-JSON line on stdout:', line.rstrip())
continue
if e['type'] == 'step' and e['state'] != 'started':
extra = f'{e["seconds"]:.1f} s' if 'seconds' in e else e.get('reason', '')
print(f'[{e["t"]:8.1f}] {e["step"]:8} {e["state"]} {extra}', flush=True)
elif e['type'] == 'step':
print(f'[{e["t"]:8.1f}] {e["step"]:8} started', flush=True)
elif e['type'] in ('warning', 'error', 'plan', 'end'):
print(f'[{e["t"]:8.1f}] {json.dumps({k: v for k, v in e.items() if k != "t"})}', flush=True)
elif e['type'] == 'start':
print(f'[{e["t"]:8.1f}] start mode={e["mode"]} release={e["release"]}', flush=True)
return proc.wait()
if __name__ == '__main__':
sys.exit(main())

451
tests/installer_gui_test.py Normal file
View File

@@ -0,0 +1,451 @@
"""The installer window: event model, form, catalogs, cancel, a self-closing Tk run and one end-to-end run.
.venv-py314\\Scripts\\python tests\\installer_gui_test.py
The end-to-end test runs the real CLI in --verify mode on a finished install (every step skips) and folds its events
through the same code the window uses (the fixture's step keys are recomputed for the checkout under test, because the keys fingerprint recomp/, tools/ and runtime/). The install is build/inst-gui/e2e-install: hard links to build/install-test, so
nothing there is written, except the state file and nx2navi.ini, which a run rewrites and which are copied. The run hashes
the 17 GB RomFS; it is skipped when the hash differs from the expected one in all three tries. On the developer machine
(RAM with a bad bit) a buffered read of a file sometimes returns flipped bits from the page cache while the file on
disk is intact (an unbuffered read of install-test's romfs.bin gives the expected hash), so such a skip is the machine,
not the installer. It is also skipped when build/install-test is absent.
"""
import io
import json
import os
import queue
import re
import shutil
import subprocess
import sys
import tempfile
import time
import unittest
from pathlib import Path
from types import SimpleNamespace
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT))
from installer import layout as layouts # noqa: E402
from installer import state as statefile # noqa: E402
from installer.events import Reporter # noqa: E402
from installer.runner import Context # noqa: E402
from installer.steps import Fonts, all_steps # noqa: E402
from installer.guilogic import (DONE, FAILED, PENDING, RUNNING, SKIPPED, UI_LANGUAGE_LABELS, # noqa: E402
UI_LANGUAGES, Catalog, Form, Model, Process, amount, default_game_language,
describe_event, parse_line, ui_language_for_langid, windows_ui_language)
INSTALLED = ROOT / 'build' / 'install-test'
FIXTURE = ROOT / 'build' / 'inst-gui' / 'e2e-install'
COPIED = ('nx2navi.ini', statefile.NAME) # written by a run, so never linked
START = {'type': 'start', 'release': 'r1', 'mode': 'install', 'dest': 'D:\\T', 'log': 'D:\\T.log',
'steps': ['check', 'fonts', 'extract', 'lift', 'compile', 'prepare', 'cleanup']}
def fold(*events):
m = Model()
for e in events:
m.apply(e)
return m
class ParseTest(unittest.TestCase):
def test_event_line(self):
self.assertEqual(parse_line('{"type": "step", "step": "check", "state": "started"}\n')['step'], 'check')
def test_noise_is_not_an_event(self):
for line in ('', 'Traceback (most recent call last):', '{broken', '[1, 2]', '{"no_type": 1}', '{"type": 3}'):
self.assertIsNone(parse_line(line), line)
def test_unicode_survives(self):
e = parse_line(json.dumps({'type': 'warning', 'id': 'x', 'params': {'p': 'Привет'}}, ensure_ascii=False))
self.assertEqual(e['params']['p'], 'Привет')
class ModelTest(unittest.TestCase):
def test_steps_follow_the_events(self):
m = fold(START, {'type': 'plan', 'run': ['check', 'extract'], 'skip': {'lift': 'current', 'compile': 'current'}},
{'type': 'step', 'step': 'check', 'state': 'started'})
self.assertEqual([m.states[s] for s in m.steps],
[RUNNING, PENDING, PENDING, SKIPPED, SKIPPED, PENDING, PENDING])
self.assertEqual(m.current, 'check')
m.apply({'type': 'step', 'step': 'check', 'state': 'done', 'seconds': 1.0})
self.assertEqual(m.states['check'], DONE)
self.assertIsNone(m.current)
m.apply({'type': 'step', 'step': 'extract', 'state': 'skipped', 'reason': 'current'})
self.assertEqual(m.states['extract'], SKIPPED)
def test_progress_fraction(self):
m = fold(START, {'type': 'step', 'step': 'extract', 'state': 'started'},
{'type': 'progress', 'step': 'extract', 'phase': 'copy', 'done': 5, 'total': 20, 'unit': 'bytes'})
self.assertEqual((m.current, m.phase, m.fraction()), ('extract', 'copy', 0.25))
m.apply({'type': 'progress', 'step': 'extract', 'phase': 'merge'})
self.assertIsNone(m.fraction()) # a phase without size is indeterminate
m.apply({'type': 'progress', 'step': 'extract', 'phase': 'copy', 'done': 30, 'total': 20})
self.assertEqual(m.fraction(), 1.0)
def test_a_new_step_resets_progress(self):
m = fold(START, {'type': 'progress', 'step': 'check', 'phase': 'inputs', 'done': 1, 'total': 2},
{'type': 'step', 'step': 'extract', 'state': 'started'})
self.assertIsNone(m.phase)
self.assertIsNone(m.fraction())
def test_error_marks_the_step(self):
m = fold(START, {'type': 'step', 'step': 'check', 'state': 'started'},
{'type': 'error', 'id': 'check.disk_space', 'step': 'check', 'params': {'path': 'D:\\T'}},
{'type': 'end', 'result': 'failed', 'seconds': 2})
self.assertEqual(m.states['check'], FAILED)
self.assertEqual(m.error[0], 'check.disk_space')
self.assertEqual(m.result, 'failed')
def test_space_text(self):
m = fold({'type': 'space', 'dest': 'D:\\T', 'need': 36_400_000_000, 'free': 100_000_000_000, 'temp': 'C:\\t',
'temp_need': 0, 'temp_free': 5, 'same_disk': True})
text = m.need_text(Catalog('en'))
self.assertIn('36.4', text)
self.assertIn('100.0', text)
self.assertEqual(Model().need_text(Catalog('en')), '')
m = fold({'type': 'space', 'dest': 'D:\\T', 'need': 20_000_000_000, 'free': 1, 'temp': 'C:\\t',
'temp_need': 2_000_000_000, 'temp_free': 3, 'same_disk': False})
self.assertIn('C:\\t', m.need_text(Catalog('en')))
def test_warnings_collect(self):
m = fold(START, {'type': 'warning', 'id': 'warn.lang_needs_fonts', 'params': {'lang': 'ko'}})
self.assertEqual(m.warnings, [('warn.lang_needs_fonts', {'lang': 'ko'})])
def test_finish_without_end_event(self):
m = fold(START, {'type': 'step', 'step': 'lift', 'state': 'started'})
m.finish(-1073741510)
self.assertEqual(m.result, 'failed')
self.assertEqual(m.states['lift'], FAILED)
self.assertEqual(m.error[0], 'ui.process_died')
def test_finish_after_cancel(self):
m = fold(START, {'type': 'step', 'step': 'lift', 'state': 'started'}, {'type': 'end', 'result': 'cancelled'})
m.finish(3)
self.assertEqual(m.result, 'cancelled')
self.assertIsNone(m.error)
self.assertEqual(m.states['lift'], PENDING) # not finished; the next run resumes here
def test_exit_code_3_alone_is_cancelled(self):
m = Model()
m.finish(3)
self.assertEqual(m.result, 'cancelled')
def test_unknown_event_is_ignored(self):
m = fold(START, {'type': 'future_thing', 'x': 1})
self.assertEqual(m.states['check'], PENDING)
class RunStatesTest(unittest.TestCase):
OK = [START, {'type': 'plan', 'run': ['check', 'extract'], 'skip': {'fonts': 'current', 'lift': 'current',
'compile': 'current', 'prepare': 'current',
'cleanup': 'not_needed'}},
{'type': 'step', 'step': 'check', 'state': 'started'}, {'type': 'step', 'step': 'check', 'state': 'done'},
{'type': 'step', 'step': 'extract', 'state': 'started'}, {'type': 'step', 'step': 'extract', 'state': 'done'},
{'type': 'step', 'step': 'fonts', 'state': 'skipped'},
{'type': 'step', 'step': 'lift', 'state': 'skipped'}, {'type': 'step', 'step': 'compile', 'state': 'skipped'},
{'type': 'step', 'step': 'prepare', 'state': 'skipped'}, {'type': 'step', 'step': 'cleanup', 'state': 'done'},
{'type': 'end', 'result': 'ok', 'seconds': 1}]
FAILED = [START, {'type': 'step', 'step': 'check', 'state': 'started'}, {'type': 'step', 'step': 'check', 'state': 'done'},
{'type': 'step', 'step': 'extract', 'state': 'started'},
{'type': 'error', 'id': 'extract.romfs_mismatch', 'step': 'extract', 'params': {}},
{'type': 'warning', 'id': 'warn.lang_needs_fonts', 'params': {'lang': 'ko'}},
{'type': 'end', 'result': 'failed', 'seconds': 1}]
def test_ok_run_has_only_done_and_skipped_steps(self):
m = fold(*self.OK)
m.finish(0)
self.assertEqual(m.result, 'ok')
self.assertIsNone(m.error)
self.assertEqual({m.states[s] for s in m.steps}, {DONE, SKIPPED})
def test_a_new_run_keeps_nothing_of_the_failed_one(self):
m = fold(*self.FAILED)
m.finish(1)
self.assertEqual(m.states['extract'], FAILED)
for e in self.OK: # the same model object fed a second run (the window makes a new one, but even then)
m.apply(e)
m.finish(0)
self.assertEqual({m.states[s] for s in m.steps}, {DONE, SKIPPED})
self.assertIsNone(m.error)
self.assertEqual(m.warnings, [])
self.assertEqual(m.result, 'ok')
class FormTest(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.dir = Path(self.tmp.name)
for name in ('base.nsp', 'upd.nsp', 'prod.keys'):
(self.dir / name).write_bytes(b'x')
def tearDown(self):
self.tmp.cleanup()
def form(self, **kw):
values = dict(base=str(self.dir / 'base.nsp'), update=str(self.dir / 'upd.nsp'), keys=str(self.dir / 'prod.keys'),
lang='en-US', dest=str(self.dir / 'out'))
values.update(kw)
return Form(**values)
def test_complete_form(self):
self.assertEqual(self.form().problems(), [])
def test_empty_form(self):
self.assertEqual(Form().problems(), ['ui.need_dest', 'ui.need_base', 'ui.need_update', 'ui.need_keys',
'ui.need_lang'])
def test_missing_files(self):
self.assertEqual(self.form(base=str(self.dir / 'no.nsp'), keys=str(self.dir / 'no.keys')).problems(),
['ui.missing_base', 'ui.missing_keys'])
def test_unknown_language(self):
self.assertEqual(self.form(lang='xx').problems(), ['ui.need_lang'])
def test_firmware_folder(self):
firmware = self.dir / 'firmware'
firmware.mkdir()
self.assertEqual(self.form(firmware=str(firmware)).problems(), [])
self.assertEqual(self.form(firmware=str(self.dir / 'no-firmware')).problems(), ['ui.missing_firmware'])
def test_command(self):
firmware = self.dir / 'firmware'
firmware.mkdir()
cmd = self.form(firmware=str(firmware)).command('py.exe', root=ROOT, temp=self.dir / 't', verify_hashes=True)
self.assertEqual(cmd[:3], ['py.exe', '-m', 'installer'])
for flag in ('--base', '--update', '--keys', '--firmware', '--lang', '--dest', '--root', '--temp', '--verify'):
self.assertIn(flag, cmd)
self.assertEqual(cmd[cmd.index('--firmware') + 1], str(firmware))
self.assertEqual(cmd[cmd.index('--lang') + 1], 'en-US')
def test_installed_folder_needs_only_itself(self):
out = self.dir / 'out'
statefile.save(self._mkdir(out), statefile.empty('r0') | {'lang': 'ru'})
form = Form(dest=str(out))
self.assertTrue(form.update_mode())
self.assertEqual(form.problems(), [])
cmd = form.command('py.exe')
self.assertEqual(cmd, ['py.exe', '-m', 'installer', '--dest', str(out)])
def test_installed_folder_still_checks_given_files(self):
out = self.dir / 'out'
statefile.save(self._mkdir(out), statefile.empty('r0'))
self.assertEqual(Form(dest=str(out), base=str(self.dir / 'no.nsp')).problems(), ['ui.missing_base'])
def test_broken_state_is_a_new_install(self):
out = self._mkdir(self.dir / 'out')
(out / statefile.NAME).write_text('{nope', encoding='utf-8')
self.assertFalse(Form(dest=str(out)).update_mode())
@staticmethod
def _mkdir(path):
path.mkdir(parents=True, exist_ok=True)
return path
class CatalogTest(unittest.TestCase):
def test_every_language_covers_english_with_the_same_placeholders(self):
en = Catalog('en').english
for lang in UI_LANGUAGES[1:]:
with self.subTest(lang=lang):
other = Catalog(lang).texts
self.assertEqual(sorted(set(en) - set(other)), [])
self.assertEqual(sorted(set(other) - set(en)), [])
holes = [k for k in en
if sorted(re.findall(r'\{\w+\}', en[k])) != sorted(re.findall(r'\{\w+\}', other[k]))]
self.assertEqual(holes, [])
def test_language_names_tell_the_codes_apart(self):
# gui.App.selected_lang maps the shown name back to its code across every window language
names = {}
for lang in UI_LANGUAGES:
cat = Catalog(lang)
for id in cat.english:
if id.startswith('lang.'):
names.setdefault(cat(id), set()).add(id)
self.assertEqual({n: ids for n, ids in names.items() if len(ids) > 1}, {})
self.assertEqual(sorted(UI_LANGUAGE_LABELS), sorted(UI_LANGUAGES))
def test_every_id_the_window_uses_exists(self):
cat = Catalog('en')
ids = ['ui.' + n for n in ('title', 'start', 'cancel', 'open_folder', 'notice', 'space_one', 'space_two',
'process_died', 'result_ok', 'result_failed', 'result_cancelled')]
ids += [f'step.{s}' for s in ('check', 'fonts', 'extract', 'lift', 'compile', 'prepare', 'cleanup')]
ids += ['ui.state_' + n for n in ('started', 'done', 'skipped', 'pending', 'running', 'failed')]
for lang in UI_LANGUAGES:
cat.set_language(lang)
self.assertEqual(cat.lang, lang)
self.assertEqual([i for i in ids if not cat.has(i)], [])
def test_codes_become_names(self):
for lang, korean in (('en', 'Korean'), ('ru', 'Корейский'), ('pt-BR', 'Coreano'), ('zh-Hans', '韩语')):
cat = Catalog(lang)
self.assertIn(korean, cat('warn.lang_needs_fonts', lang='ko'))
self.assertNotIn(' ko ', cat('warn.lang_needs_fonts', lang='ko'))
self.assertNotIn('extract', cat('state.output_changed', step='extract'))
for word in ('base', 'update', 'keys'):
self.assertIn(cat('param.input.' + word), cat('check.keys_wrong', input=word))
self.assertNotIn('en-US', cat('check.lang_unknown', lang='xx', known='ja, en-US'))
self.assertEqual(Catalog('en')('check.lang_unknown', lang='xx', known='ja, en-US'),
"Unknown language 'xx'. Known: Japanese, English (US).")
def test_fallback_and_params(self):
cat = Catalog('ru')
self.assertIn('36.4', cat('ui.log_space', need='36.4', free='1.0'))
self.assertEqual(cat('no.such.id'), 'no.such.id')
self.assertIn('a, b', cat('check.release_incomplete', paths=['a', 'b']))
cat.set_language('de') # unknown language falls back to English
self.assertEqual(cat.lang, 'en')
def test_describe_and_amount(self):
cat = Catalog('en')
self.assertEqual(amount(cat, 6e9, 16e9, 'bytes'), '6.0 / 16.0 GB')
self.assertEqual(amount(cat, 1, 9, 'shaders'), '1 / 9 shaders')
self.assertEqual(amount(cat, 1, 9, 'widgets'), '1 / 9 widgets')
line = describe_event(cat, {'type': 'step', 'step': 'check', 'state': 'done', 'seconds': 1.5})
self.assertIn('1.5 s', line)
self.assertEqual(describe_event(cat, {'type': 'start'}), '')
def test_windows_language_is_a_known_value(self):
self.assertIn(windows_ui_language(), UI_LANGUAGES)
def test_windows_langid_picks_the_window_language(self):
for langid, lang in ((0x0419, 'ru'), (0x0819, 'ru'), (0x0416, 'pt-BR'), (0x0816, 'en'),
(0x0804, 'zh-Hans'), (0x1004, 'zh-Hans'),
(0x0404, 'en'), (0x0c04, 'en'), (0x1404, 'en'), (0x0409, 'en'), (0x0411, 'en')):
self.assertEqual(ui_language_for_langid(langid), lang, hex(langid))
def test_chinese_window_starts_with_a_game_language_that_needs_no_shared_fonts(self):
self.assertEqual(default_game_language('zh-Hans'), 'en-US')
self.assertEqual(default_game_language('ru'), 'ru')
BREAK_CHILD = 'import time\nprint("ready", flush=True)\ntime.sleep(60)\n'
class CancelTest(unittest.TestCase):
def test_ctrl_break_ends_the_process_group(self):
p = Process([sys.executable, '-c', BREAK_CHILD])
try:
self.assertEqual(p.lines.get(timeout=20).strip(), 'ready')
t0 = time.monotonic()
p.cancel()
code = None
while code is None and time.monotonic() - t0 < 20:
code = p.poll()
time.sleep(0.05)
self.assertIsNotNone(code, 'the child ignored Ctrl+Break')
self.assertNotEqual(code, 0)
finally:
p.kill()
class SmokeTest(unittest.TestCase):
def test_every_screen_in_both_languages(self):
try:
import tkinter # noqa: F401
except ImportError:
self.skipTest('no tkinter')
r = subprocess.run([sys.executable, '-m', 'installer.gui', '--selftest'], cwd=ROOT, timeout=120,
capture_output=True, text=True)
self.assertEqual(r.returncode, 0, r.stderr[-2000:])
def make_fixture():
"""build/inst-gui/e2e-install: links to install-test (see the module docstring), with the step keys of the state
set to the ones this checkout computes, so the install counts as current for it."""
if not (FIXTURE / statefile.NAME).is_file():
shutil.rmtree(FIXTURE, ignore_errors=True)
for folder, _dirs, files in os.walk(INSTALLED):
target = FIXTURE / Path(folder).relative_to(INSTALLED)
target.mkdir(parents=True, exist_ok=True)
for name in files:
src, dst = Path(folder) / name, target / name
if name in COPIED and target == FIXTURE:
shutil.copy2(src, dst)
else:
os.link(src, dst)
layout = layouts.find(ROOT)
state = statefile.load(FIXTURE)
options = SimpleNamespace(base=None, update=None, keys=None, lang=None)
ctx = Context(layout, FIXTURE, ROOT / 'build' / 'inst-gui' / 'e2e-temp', Reporter(out=io.StringIO()), options, state=state)
ctx.steps = all_steps()
ctx.plan()
# An install made before the fonts step has no record for it: record it as current when its output is (the
# languages that need the fonts have them in shared-fonts/).
if Fonts().current(ctx, {}):
state['steps'].setdefault('fonts', {'release': state.get('release', ''), 'key': ctx.keys['fonts']})
for step_id, record in state['steps'].items():
record['key'] = ctx.keys[step_id]
statefile.save(FIXTURE, state)
return FIXTURE
@unittest.skipUnless((INSTALLED / statefile.NAME).is_file(), 'build/install-test is absent')
class EndToEndTest(unittest.TestCase):
@staticmethod
def run_cli(form, temp, tries=1):
"""Runs the CLI in --verify mode through Process, up to `tries` times while the RomFS hash reads wrong."""
for attempt in range(tries):
p = Process(form.command(sys.executable, root=ROOT, temp=temp, verify_hashes=True), cwd=ROOT)
model, stray = Model(), []
t0, code = time.monotonic(), None
while code is None and time.monotonic() - t0 < 600:
try:
line = p.lines.get(timeout=0.2)
except queue.Empty:
line = ''
if line:
event = parse_line(line)
if event:
model.apply(event)
elif line.strip():
stray.append(line)
code = p.poll()
p.kill()
model.finish(code)
if not any(w[0] == 'state.output_changed' for w in model.warnings):
break
return code, model, stray, time.monotonic() - t0
def test_verify_run_skips_every_step(self):
form = Form(dest=str(make_fixture()))
self.assertTrue(form.update_mode())
code, model, stray, elapsed = self.run_cli(form, ROOT / 'build' / 'inst-gui' / 'e2e-temp', tries=3)
print(f'\nend-to-end: exit {code}, result {model.result}, {elapsed:.1f} s, states {model.states}')
if any(w[0] == 'state.output_changed' for w in model.warnings):
self.skipTest('the RomFS hash read wrong three times (page cache of the faulty RAM, see the module docstring)')
self.assertEqual(code, 0, stray[-5:] or model.error)
self.assertEqual(model.result, 'ok')
self.assertIn(model.mode, ('resume', 'update'))
self.assertEqual({s: model.states[s] for s in model.steps if s not in ('check', 'cleanup')},
{s: SKIPPED for s in model.steps if s not in ('check', 'cleanup')})
self.assertIsNotNone(model.space)
self.assertEqual(stray, [])
def test_cancel_during_the_check(self):
cmd = Form(dest=str(make_fixture())).command(sys.executable, root=ROOT, temp=ROOT / 'build' / 'inst-gui' / 'e2e-temp',
verify_hashes=True)
p = Process(cmd, cwd=ROOT)
model, t0, code, cancelled = Model(), time.monotonic(), None, False
while code is None and time.monotonic() - t0 < 120:
try:
event = parse_line(p.lines.get(timeout=0.2))
except queue.Empty:
event = None
if event:
model.apply(event)
if event.get('phase') == 'verify_romfs' and not cancelled:
cancelled = True
p.cancel()
code = p.poll()
p.kill()
model.finish(code)
self.assertTrue(cancelled)
self.assertEqual((code, model.result), (3, 'cancelled'))
self.assertIsNone(model.error)
if __name__ == '__main__':
unittest.main(verbosity=2)

655
tests/installer_test.py Normal file
View File

@@ -0,0 +1,655 @@
"""installer/ bookkeeping with fake steps: fresh install, resume after a failure or cancel, transient steps, update
with a newer release, nx2navi.ini and the event stream. No game files, no tools; stdlib only.
Run: python tests/installer_test.py
"""
import http.client
import io
import json
import re
import sys
import tempfile
import unittest
from pathlib import Path
from types import SimpleNamespace
from unittest import mock
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT))
from installer import ini, runner, state, steps # noqa: E402
from installer.events import Reporter # noqa: E402
from installer.layout import Layout # noqa: E402
class Fake(runner.Step):
"""A step whose output is <dir>/<id>.out holding its key; dir is the temp folder for a transient step."""
def __init__(self, id, transient=False, always=False, previous=None):
self.id, self.transient, self.always, self.previous = id, transient, always, previous
def out(self, ctx):
return (ctx.temp if self.transient else ctx.dest) / f'{self.id}.out'
def key(self, ctx):
upstream = ctx.keys[self.previous] if self.previous else ''
return f'{self.id}:{ctx.options.inputs.get(self.id, "v1")}:{upstream}'
def current(self, ctx, record):
p = self.out(ctx)
return p.is_file() and p.read_text() == ctx.keys[self.id]
def verify(self, ctx, record):
ctx.options.verified.append(self.id)
return ctx.options.verify_ok.get(self.id, True)
def run(self, ctx):
o = ctx.options
o.ran.append(self.id)
if self.id in ctx.forced:
o.forced.append(self.id)
if self.id == 'check':
ctx.verify_kept(self.id)
failure = o.fail.pop(self.id, None)
if failure == 'error':
raise runner.InstallError('fake.failed', step=self.id)
if failure == 'cancel':
raise KeyboardInterrupt
if self.id == 'cleanup':
for p in ctx.temp.glob('*.out'):
p.unlink()
return {}
p = self.out(ctx)
p.parent.mkdir(parents=True, exist_ok=True)
p.write_text(ctx.keys[self.id])
return {'note': self.id}
def fake_steps():
return [Fake('check', always=True), Fake('extract'), Fake('lift', transient=True, previous='extract'),
Fake('compile', previous='lift'), Fake('prepare', previous='compile'), Fake('cleanup', always=True)]
class Bed:
"""A dest and temp folder plus one run of the fake steps per call."""
def __init__(self, base):
self.dest, self.temp = base / 'dest', base / 'temp'
self.temp.mkdir()
def run(self, release='1.0', inputs=None, fail=None, verify_ok=None, verify=False):
layout = Layout(root=Path('.'), dev=True, release=release, python=Path('python'), tools=Path('tools'),
recomp=Path('recomp'), llvm=Path('llvm'),
deps=Path('deps'), runtime=Path('rt'), runtime_lib=Path('lib'), ninja=Path('ninja'),
hactool=Path('hactool'), xlate=Path('xlate'))
options = SimpleNamespace(inputs=inputs or {}, fail=dict(fail or {}), verify_ok=verify_ok or {}, ran=[],
verified=[], forced=[])
out = io.StringIO()
ctx = runner.Context(layout, self.dest, self.temp, Reporter(out=out), options, state=state.load(self.dest),
verify=verify)
mode = ctx.mode
result = runner.run_steps(fake_steps(), ctx)
events = [json.loads(line) for line in out.getvalue().splitlines()]
return SimpleNamespace(result=result, ran=options.ran, verified=options.verified, forced=options.forced,
events=events, mode=mode,
state=state.load(self.dest))
@staticmethod
def skipped(r):
return {e['step']: e['reason'] for e in r.events if e['type'] == 'step' and e['state'] == 'skipped'}
ALL = ['check', 'extract', 'lift', 'compile', 'prepare', 'cleanup']
class RunnerTest(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.bed = Bed(Path(self.tmp.name))
def tearDown(self):
self.tmp.cleanup()
def test_fresh_install_runs_everything_and_records_it(self):
r = self.bed.run()
self.assertEqual((r.result, r.mode, r.ran), ('ok', 'install', ALL))
self.assertEqual(set(r.state['steps']), {'extract', 'lift', 'compile', 'prepare'})
self.assertEqual(r.state['steps']['compile']['note'], 'compile')
self.assertEqual(r.state['release'], '1.0')
started = [e['step'] for e in r.events if e['type'] == 'step' and e['state'] == 'started']
done = [e['step'] for e in r.events if e['type'] == 'step' and e['state'] == 'done']
self.assertEqual((started, done), (ALL, ALL))
def test_rerun_skips_finished_steps_and_unneeded_transient(self):
self.bed.run()
r = self.bed.run()
self.assertEqual((r.result, r.mode, r.ran), ('ok', 'resume', ['check', 'cleanup']))
self.assertEqual(self.bed.skipped(r), {'extract': 'current', 'lift': 'not_needed', 'compile': 'current',
'prepare': 'current'})
self.assertEqual(r.verified, []) # same release: no full checks
def test_resume_after_failure(self):
r = self.bed.run(fail={'compile': 'error'})
self.assertEqual((r.result, r.ran), ('failed', ['check', 'extract', 'lift', 'compile']))
error = [e for e in r.events if e['type'] == 'error'][0]
self.assertEqual((error['id'], error['step'], error['params']), ('fake.failed', 'compile', {'step': 'compile'}))
self.assertEqual(set(r.state['steps']), {'extract', 'lift'})
r = self.bed.run()
# lift's output is still in the temp folder (no cleanup ran), so only compile and later run.
self.assertEqual((r.result, r.mode, r.ran), ('ok', 'resume', ['check', 'compile', 'prepare', 'cleanup']))
def test_resume_after_cancel_with_transient_output_gone(self):
r = self.bed.run(fail={'prepare': 'cancel'})
self.assertEqual(r.result, 'cancelled')
self.assertNotIn('prepare', r.state['steps'])
for p in self.bed.temp.glob('*.out'):
p.unlink()
r = self.bed.run()
self.assertEqual(r.ran, ['check', 'prepare', 'cleanup'])
self.assertEqual(self.bed.skipped(r)['lift'], 'not_needed')
def test_failed_check_writes_nothing(self):
r = self.bed.run(fail={'check': 'error'})
self.assertEqual(r.result, 'failed')
self.assertFalse(self.bed.dest.exists())
def test_removed_output_runs_again(self):
self.bed.run()
(self.bed.dest / 'compile.out').unlink()
r = self.bed.run()
self.assertEqual(r.ran, ['check', 'lift', 'compile', 'cleanup'])
def test_update_with_new_lifter_keeps_extraction(self):
self.bed.run(release='1.0')
r = self.bed.run(release='1.1', inputs={'lift': 'v2'})
self.assertEqual((r.result, r.mode), ('ok', 'update'))
self.assertEqual(r.ran, ['check', 'lift', 'compile', 'prepare', 'cleanup'])
self.assertEqual(r.verified, ['extract']) # kept from another release: full check
self.assertEqual(r.state['steps']['extract']['release'], '1.0')
self.assertEqual(r.state['steps']['compile']['release'], '1.1')
self.assertEqual(r.state['release'], '1.1')
def test_update_that_changes_nothing_skips_the_rebuild(self):
self.bed.run(release='1.0')
r = self.bed.run(release='1.1')
self.assertEqual(r.ran, ['check', 'cleanup'])
self.assertEqual(r.verified, ['extract', 'compile', 'prepare'])
def test_update_with_changed_kept_files_extracts_again(self):
self.bed.run(release='1.0')
r = self.bed.run(release='1.1', verify_ok={'extract': False})
self.assertEqual(r.ran, ['check', 'extract', 'cleanup'])
self.assertEqual(r.forced, ['extract']) # the step knows not to trust what it left behind
warnings = [e for e in r.events if e['type'] == 'warning']
self.assertEqual([(w['id'], w['params']) for w in warnings], [('state.output_changed', {'step': 'extract'})])
def test_verify_option_checks_same_release(self):
self.bed.run()
r = self.bed.run(verify=True)
self.assertEqual(r.verified, ['extract', 'compile', 'prepare'])
def test_state_errors(self):
self.bed.dest.mkdir()
(self.bed.dest / state.NAME).write_text('{not json')
with self.assertRaises(state.StateError) as e:
state.load(self.bed.dest)
self.assertEqual(e.exception.id, 'state.unreadable')
(self.bed.dest / state.NAME).write_text(json.dumps({'format': 99, 'steps': {}}))
with self.assertRaises(state.StateError) as e:
state.load(self.bed.dest)
self.assertEqual(e.exception.id, 'state.newer_format')
class ReleaseFilesTest(unittest.TestCase):
def test_release_ships_every_tool_the_steps_run(self):
sys.path.insert(0, str(ROOT / 'tools'))
import make_release
text = (ROOT / 'installer' / 'steps.py').read_text(encoding='utf-8')
used = set(re.findall(r"layout\.tools / '(\w+\.py)'", text))
for names in re.findall(r"layout\.tools / n for n in \((.*?)\)", text, re.S):
used |= set(re.findall(r"'(\w+\.py)'", names))
self.assertIn('cheats.py', used)
self.assertEqual(used - set(make_release.TOOLS_FILES), set())
class RenamedVersionTest(unittest.TestCase):
"""An install the setup before 0.3.1 recorded as 1.1.1 is the 1.1.0 update (#332)."""
def setUp(self):
tmp = tempfile.TemporaryDirectory()
self.addCleanup(tmp.cleanup)
root = Path(__file__).resolve().parents[1]
layout = Layout(root=root, dev=True, release='9.9', python=Path('python'), tools=root / 'tools',
recomp=root / 'recomp', llvm=Path('llvm'), deps=Path('deps'), runtime=Path('rt'),
runtime_lib=Path('lib'), ninja=Path('ninja'), hactool=Path('hactool'), xlate=Path('xlate'))
state = {'format': 1, 'release': '0.2.8', 'version': '1.1.1', 'inputs': {},
'steps': {'extract': {'key': 'recorded by 0.2.8', 'release': '0.2.8'}}}
out = io.StringIO()
self.ctx = runner.Context(layout, Path(tmp.name), Path(tmp.name), Reporter(out=out), SimpleNamespace(),
state=state)
def test_old_name_is_read_as_1_1_0(self):
steps.migrate_state(self.ctx)
self.assertEqual(steps.game_version(self.ctx), '1.1.0')
self.assertEqual(steps.version_tag(steps.game_version(self.ctx)), 'v110')
self.assertEqual(steps.expected_update(self.ctx)['display_version'], '1.1.0')
self.assertTrue(self.ctx.layout.patches(steps.game_version(self.ctx)).is_file())
def test_extraction_is_kept_under_the_new_name(self):
steps.migrate_state(self.ctx)
self.assertEqual(self.ctx.state['steps']['extract']['key'], steps.Extract().key(self.ctx))
self.assertEqual(self.ctx.state['steps']['extract']['release'], '0.2.8') # full check of kept output runs
def test_other_versions_are_untouched(self):
self.ctx.state['version'] = '1.4.3'
steps.migrate_state(self.ctx)
self.assertEqual(self.ctx.state['version'], '1.4.3')
self.assertEqual(self.ctx.state['steps']['extract']['key'], 'recorded by 0.2.8')
class LiftKeyTest(unittest.TestCase):
def setUp(self):
tmp = tempfile.TemporaryDirectory()
self.addCleanup(tmp.cleanup)
self.dest = Path(tmp.name)
layout = SimpleNamespace(recomp=self.dest / 'no-recomp', tools=self.dest / 'no-tools',
patches=lambda version: self.dest / 'no-patches')
self.ctx = SimpleNamespace(dest=self.dest, layout=layout, keys={'extract': 'x'}, state={'version': '1.4.3'})
def key(self):
return steps.Lift().key(self.ctx)
def add_cheats(self, name, text, mod='Cheats', where='cheats'):
folder = self.dest / 'mods' / mod / where
folder.mkdir(parents=True, exist_ok=True)
(folder / name).write_text(text)
def test_extracting_main_keeps_the_key(self):
before = self.key()
(self.dest / 'exefs').mkdir()
(self.dest / 'exefs' / 'main').write_bytes(bytes(0x48))
self.assertEqual(self.key(), before)
def test_cheat_files_change_the_key(self):
before = self.key()
self.add_cheats('277178B7DBA1B6D4.txt', '[a]')
added = self.key()
self.assertNotEqual(added, before)
self.add_cheats('277178B7DBA1B6D4.txt', '[b]')
edited = self.key()
self.assertNotEqual(edited, added)
self.add_cheats('277178B7DBA1B6D4.txt', '[c]', mod='Other', where='atmosphere/contents/0100F2C0115B6000/cheats')
self.assertNotEqual(self.key(), edited)
def test_exefs_patch_files_change_the_key(self):
before = self.key()
self.add_cheats('dynamic-fps.pchtxt', '@nsobid-277178B7DBA1B6D4', mod='Fps', where='exefs')
added = self.key()
self.assertNotEqual(added, before)
self.add_cheats('277178B7DBA1B6D4.ips', 'PATCHEOF', mod='Ips', where='atmosphere/exefs_patches/x')
both = self.key()
self.assertNotEqual(both, added)
self.add_cheats('main.npdm', 'not a patch', mod='Fps', where='exefs')
self.assertEqual(self.key(), both)
def test_renaming_a_mod_folder_changes_the_key(self):
# tools/cheats.py and tools/exefs_patches.py name a patch set after "<mod folder>/<cheat or file>": a renamed
# folder needs a lift even though its files are the same
self.add_cheats('277178B7DBA1B6D4.txt', '[a]', mod='MyNXCheats')
self.add_cheats('dynamic-fps.pchtxt', '@nsobid-277178B7DBA1B6D4', mod='Fps', where='exefs')
before = self.key()
mods = self.dest / 'mods'
(mods / 'MyNXCheats').rename(mods / 'MyNXCheats 1.4.3')
cheats_renamed = self.key()
self.assertNotEqual(cheats_renamed, before)
(mods / 'Fps').rename(mods / 'Fps 2')
self.assertNotEqual(self.key(), cheats_renamed)
def test_status_and_other_files_keep_the_key(self):
self.add_cheats('277178B7DBA1B6D4.txt', '[a]')
before = self.key()
(self.dest / 'build-cache').mkdir()
(self.dest / 'build-cache' / 'cheats.status').write_text('written by lift')
self.add_cheats('readme.md', 'not a cheat file')
(self.dest / 'mods' / 'Cheats' / 'romfs').mkdir()
(self.dest / 'mods' / 'Cheats' / 'romfs' / 'a.txt').write_text('a RomFS file')
self.assertEqual(self.key(), before)
class IniTest(unittest.TestCase):
EXAMPLE = '[display]\n# vk_present = mailbox\n\n[language]\n# locale = en-US\n# lang = en-US\n# x = 1\n'
def test_example_line_is_activated(self):
out = ini.set_language(self.EXAMPLE, 'ru')
self.assertIn('\nlang = ru\n', out)
self.assertNotIn('# lang', out)
self.assertIn('# locale = en-US', out)
def test_active_line_is_rewritten_and_the_rest_kept(self):
text = '[display]\nvk_present = fifo\n[language]\n LANG = ja \nfps60 = 0\n'
out = ini.set_language(text, 'de')
self.assertEqual(out, '[display]\nvk_present = fifo\n[language]\n lang = de\nfps60 = 0\n')
self.assertEqual(ini.get_language(out), 'de')
def test_missing_section_is_added(self):
self.assertEqual(ini.set_language('fps60 = 0\n', 'fr'), 'fps60 = 0\n\n[language]\nlang = fr\n')
self.assertEqual(ini.set_language('[language]\nlocale = de-DE\n', 'fr'),
'[language]\nlang = fr\nlocale = de-DE\n')
def test_old_chinese_codes_are_written_as_the_canonical_ones(self):
self.assertEqual(ini.canonical_language('zh-CN'), 'zh-Hans')
self.assertEqual(ini.canonical_language('zh-TW'), 'zh-Hant')
self.assertEqual(ini.canonical_language('ja'), 'ja')
self.assertNotIn('zh-CN', ini.LANGUAGES)
self.assertNotIn('zh-TW', ini.LANGUAGES)
out = ini.set_language(self.EXAMPLE, 'zh-CN')
self.assertEqual(ini.get_language(out), 'zh-Hans')
self.assertEqual(ini.get_language(ini.set_language(out, 'zh-TW')), 'zh-Hant')
def test_write_keeps_user_lines(self):
with tempfile.TemporaryDirectory() as d:
d = Path(d)
example = d / 'nx2navi.ini.example'
example.write_text(self.EXAMPLE)
path = ini.write(d / '', 'en-GB', example)
self.assertEqual(ini.get_language(path.read_text()), 'en-GB')
path.write_text(path.read_text().replace('# vk_present = mailbox', 'vk_present = fifo'))
ini.write(d, 'ja', example)
text = path.read_text()
self.assertIn('vk_present = fifo', text)
self.assertEqual(ini.get_language(text), 'ja')
class CleanupTest(unittest.TestCase):
"""The real Cleanup step on a finished install whose romfs/ is intact, emptied or gone (#107)."""
def setUp(self):
self.steps = steps
self.tmp = tempfile.TemporaryDirectory()
base = Path(self.tmp.name)
self.dest, temp = base / 'dest', base / 'temp'
(self.dest / 'romfs').mkdir(parents=True)
temp.mkdir()
image = b'romfs image'
(self.dest / 'romfs' / 'romfs.bin').write_bytes(image)
digest = steps.extract.sha256(self.dest / 'romfs' / 'romfs.bin')
(self.dest / 'romfs' / 'index.json').write_text(json.dumps({'romfs_size': len(image), 'romfs_sha256': digest}))
self.expected = {'romfs': {'size': len(image), 'sha256': digest}}
self.ctx = SimpleNamespace(reporter=Reporter(out=io.StringIO(), log_path=base / 'x.log'), dest=self.dest,
temp=temp, lang='en-US', state={},
layout=SimpleNamespace(ini_example=ROOT / 'nx2navi.ini.example', dev=False,
python=base / 'python' / 'python.exe', root=base / 'release'))
self.saved = steps.expected_update
steps.expected_update = lambda ctx: self.expected
def tearDown(self):
self.steps.expected_update = self.saved
self.ctx.reporter.close()
self.tmp.cleanup()
def run_cleanup(self):
try:
self.steps.Cleanup().run(self.ctx)
return 'ok'
except self.steps.InstallError as e:
return e.id
def test_intact_romfs_passes(self):
self.assertEqual(self.run_cleanup(), 'ok')
def test_keeps_build_and_writes_rebuild_command(self):
for name in ('gen', 'build', 'check'):
(self.ctx.temp / name).mkdir()
self.assertEqual(self.run_cleanup(), 'ok')
self.assertTrue((self.ctx.temp / 'gen').is_dir() and (self.ctx.temp / 'build').is_dir())
self.assertFalse((self.ctx.temp / 'check').exists())
lines = (self.dest / 'build-cache' / 'rebuild.txt').read_text(encoding='utf-8').splitlines()
self.assertEqual(lines[1:], ['-c', steps.REBUILD_BOOT, str(self.ctx.layout.root), '--root',
str(self.ctx.layout.root), '--rebuild', str(self.dest)])
def test_emptied_romfs_fails(self):
for p in (self.dest / 'romfs').iterdir():
p.unlink()
self.assertEqual(self.run_cleanup(), 'cleanup.romfs_unreadable')
def test_missing_romfs_fails(self):
self.steps.remove(self.dest / 'romfs')
self.assertEqual(self.run_cleanup(), 'cleanup.romfs_unreadable')
def test_romfs_of_another_size_fails(self):
(self.dest / 'romfs' / 'romfs.bin').write_bytes(b'short')
self.assertEqual(self.run_cleanup(), 'cleanup.romfs_unreadable')
class BaseGameCheckTest(unittest.TestCase):
"""Issue #261: an update NSP chosen as the base game was reported as "not Ocarina of Time"."""
BASE, UPDATE = '0100F2C0115B6000', '0100F2C0115B6800'
@staticmethod
def nca(content_type, title_id):
return ({'name': 'x.nca', 'nca_offset': 0, 'size': 0xc00, 'content_type': content_type, 'title_id': title_id}, b'')
def test_update_package_names_the_update(self):
# An update NSP keeps the base title id on its Program NCA; only its Meta NCA carries the update id.
records = [self.nca(0, self.BASE), self.nca(1, self.UPDATE)]
with self.assertRaises(ValueError):
steps.program_nca(records)
self.assertEqual(steps.base_misuse(records), 'check.base_is_update')
def test_combined_package_is_named(self):
records = [self.nca(0, self.BASE), self.nca(1, self.BASE), self.nca(1, self.UPDATE)]
self.assertEqual(steps.base_misuse(records), 'check.base_combined')
def test_plain_base_game_passes(self):
records = [self.nca(0, self.BASE), self.nca(1, self.BASE), self.nca(3, self.BASE)]
self.assertIsNone(steps.base_misuse(records))
self.assertEqual(steps.program_nca(records)[0]['title_id'], self.BASE)
class FontsTest(unittest.TestCase):
"""The fonts step: keyed by language and firmware, a no-op without firmware, writes dest/shared-fonts."""
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
base = Path(self.tmp.name)
self.dest, temp, self.firmware = base / 'dest', base / 'temp', base / 'firmware'
temp.mkdir()
self.firmware.mkdir()
(self.firmware / 'a.nca').write_bytes(b'nca')
self.keys = base / 'prod.keys'
self.keys.write_text('header_key = 00\n')
self.ctx = SimpleNamespace(dest=self.dest, temp=temp, lang='zh-Hans', state={},
layout=SimpleNamespace(tools=ROOT / 'tools', root=ROOT),
reporter=Reporter(out=io.StringIO()),
options=SimpleNamespace(lang='zh-Hans', firmware=self.firmware, keys=str(self.keys)))
self.saved_run_tool = steps.run_tool
def tearDown(self):
steps.run_tool = self.saved_run_tool
self.ctx.reporter.close()
self.tmp.cleanup()
def fake_tool(self, files=None, code=0):
"""Stand-in for extract_shared_fonts.py: writes the requested files where --output points."""
def run(ctx, step, script, args):
out = Path(args[args.index('--output') + 1])
out.mkdir(parents=True)
for name in steps.SHARED_FONT_FILES if files is None else files:
(out / name).write_bytes(b'bfttf')
(out / 'manifest.json').write_text('{}')
log = ctx.temp / 'fonts.log'
log.write_text('error line\n')
return code, log
steps.run_tool = run
def test_all_steps_order(self):
self.assertEqual([s.id for s in steps.all_steps()],
['check', 'fonts', 'extract', 'lift', 'compile', 'prepare', 'dlss', 'cleanup'])
def test_extracts_into_the_install_folder(self):
self.fake_tool()
self.assertEqual(steps.Fonts().run(self.ctx), {'files': 6})
self.assertTrue(steps.shared_fonts_ready(self.dest))
self.assertEqual(sorted(p.name for p in (self.dest / 'shared-fonts').iterdir()),
sorted((*steps.SHARED_FONT_FILES, 'manifest.json')))
def test_noop_without_firmware(self):
self.ctx.options.firmware = None
self.fake_tool()
self.assertEqual(steps.Fonts().run(self.ctx), {})
self.assertFalse((self.dest / 'shared-fonts').exists())
def test_noop_for_other_languages(self):
self.ctx.lang, self.ctx.options.lang = 'en-US', 'en-US'
self.fake_tool()
self.assertEqual(steps.Fonts().run(self.ctx), {})
self.assertFalse((self.dest / 'shared-fonts').exists())
def test_failed_extraction_keeps_the_old_folder(self):
self.fake_tool()
steps.Fonts().run(self.ctx)
self.fake_tool(code=1)
with self.assertRaises(steps.InstallError) as e:
steps.Fonts().run(self.ctx)
self.assertEqual(e.exception.id, 'fonts.failed')
self.assertTrue(steps.shared_fonts_ready(self.dest))
def test_incomplete_output_is_an_error(self):
self.fake_tool(files=steps.SHARED_FONT_FILES[:1])
with self.assertRaises(steps.InstallError) as e:
steps.Fonts().run(self.ctx)
self.assertEqual(e.exception.id, 'fonts.incomplete')
def test_current_tracks_language_and_output(self):
fonts = steps.Fonts()
self.assertFalse(fonts.current(self.ctx, {}))
self.fake_tool()
fonts.run(self.ctx)
self.assertTrue(fonts.current(self.ctx, {}))
self.ctx.options.lang = 'en-US'
self.assertTrue(fonts.current(self.ctx, {}))
self.ctx.options.lang = 'zh-Hans'
(self.dest / 'shared-fonts' / '0-standard.bfttf').unlink()
self.assertFalse(fonts.current(self.ctx, {}))
def test_key_changes_with_language_and_firmware(self):
fonts = steps.Fonts()
zh = fonts.key(self.ctx)
self.ctx.options.lang = 'en-US'
self.assertNotEqual(zh, fonts.key(self.ctx))
self.ctx.options.lang = 'zh-Hans'
(self.firmware / 'b.nca').write_bytes(b'other')
self.assertNotEqual(zh, fonts.key(self.ctx))
def test_registered_child_is_used(self):
nested = self.firmware / 'registered'
nested.mkdir()
(nested / 'c.nca').write_bytes(b'nca')
self.assertEqual(steps.firmware_dir(self.firmware), nested)
class EventsTest(unittest.TestCase):
def test_lines_are_json_and_the_log_is_english(self):
with tempfile.TemporaryDirectory() as d:
out = io.StringIO()
rep = Reporter(out=out, log_path=Path(d) / 'x.log')
rep.step('check', 'started')
rep.progress('extract', 'copy', 0, 10, 'bytes')
rep.progress('extract', 'copy', 5, 10, 'bytes') # throttled
rep.progress('extract', 'copy', 10, 10, 'bytes') # last one always goes out
rep.error('check.disk_space', {'path': 'D:\\x', 'need_gb': 40, 'free_gb': 3}, step='check')
rep.close()
events = [json.loads(line) for line in out.getvalue().splitlines()]
self.assertEqual([e['type'] for e in events], ['step', 'progress', 'progress', 'error'])
self.assertEqual([e.get('done') for e in events[1:3]], [0, 10])
self.assertTrue(all('t' in e for e in events))
log = (Path(d) / 'x.log').read_text(encoding='utf-8')
self.assertIn('40 GB needed, 3 GB free', log)
self.assertIn('[check.disk_space]', log)
def test_catalog_has_every_id_the_code_uses(self):
catalog = json.loads((ROOT / 'installer' / 'text' / 'en.json').read_text(encoding='utf-8'))
used = set()
for path in (ROOT / 'installer').glob('*.py'):
text = path.read_text(encoding='utf-8')
used |= set(re.findall(r"InstallError\('([a-z_.]+)'", text))
used |= set(re.findall(r"warning\('([a-z_.]+)'", text))
used |= {f'phase.{s}.{p}' for s, p in re.findall(r"progress\('(\w+)', '(\w+)'[,)]", text)}
self.assertEqual(sorted(i for i in used if i not in catalog), [])
@unittest.skipUnless(steps.os.name == 'nt', 'DLSS is Windows-only')
class DlssTest(unittest.TestCase):
"""The DLSS step: off without --dlss, needs an RTX GPU, keeps a player's library, verifies the download."""
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
base = Path(self.tmp.name)
self.dest, temp = base / 'dest', base / 'temp'
self.dest.mkdir()
temp.mkdir()
self.src = base / 'nvngx_dlss.dll'
self.src.write_bytes(b'dlss' * 1000)
self.saved = steps.DLSS_URL, steps.DLSS_SIZE, steps.DLSS_SHA256
steps.DLSS_URL = self.src.as_uri() # urllib reads file: URLs as it reads https: ones
steps.DLSS_SIZE, steps.DLSS_SHA256 = 4000, steps.file_sha256(self.src)
rtx = {'devices': [{'name': 'NVIDIA GeForce RTX 4080 SUPER', 'api': [1, 4, 0], 'type': 2}]}
self.ctx = SimpleNamespace(dest=self.dest, temp=temp, host=rtx, reporter=Reporter(out=io.StringIO()),
options=SimpleNamespace(dlss=True))
def tearDown(self):
steps.DLSS_URL, steps.DLSS_SIZE, steps.DLSS_SHA256 = self.saved
self.ctx.reporter.close()
self.tmp.cleanup()
def warnings(self):
return [json.loads(line)['id'] for line in self.ctx.reporter.out.getvalue().splitlines()
if json.loads(line)['type'] == 'warning']
def test_off_without_the_option(self):
self.ctx.options.dlss = False
self.assertEqual(steps.Dlss().run(self.ctx), {})
self.assertFalse((self.dest / 'nvngx_dlss.dll').exists())
self.assertNotEqual(steps.Dlss().key(self.ctx), steps.Dlss().key(SimpleNamespace(options=SimpleNamespace(dlss=True))))
def test_downloads_and_checks_the_library(self):
rec = steps.Dlss().run(self.ctx)
self.assertEqual(rec, {'installed': steps.DLSS_VERSION})
self.assertEqual((self.dest / 'nvngx_dlss.dll').read_bytes(), self.src.read_bytes())
self.assertFalse(list(self.ctx.temp.iterdir()))
self.assertTrue(steps.Dlss().current(self.ctx, rec))
(self.dest / 'nvngx_dlss.dll').unlink()
self.assertFalse(steps.Dlss().current(self.ctx, rec))
def test_a_wrong_file_is_not_installed_and_tried_again(self):
steps.DLSS_SHA256 = '0' * 64
rec = steps.Dlss().run(self.ctx)
self.assertEqual(rec, {'failed': True})
self.assertFalse((self.dest / 'nvngx_dlss.dll').exists())
self.assertEqual(self.warnings(), ['warn.dlss_download_failed'])
self.assertFalse(steps.Dlss().current(self.ctx, rec))
def test_a_cut_connection_warns(self):
class Cut(io.BytesIO): # a response whose connection drops mid-file
def read(self, n=-1):
raise http.client.IncompleteRead(b'dl', 3998)
with mock.patch.object(steps.urllib.request, 'urlopen', return_value=Cut()):
self.assertEqual(steps.Dlss().run(self.ctx), {'failed': True})
self.assertFalse((self.dest / 'nvngx_dlss.dll').exists())
self.assertFalse(list(self.ctx.temp.iterdir()))
self.assertEqual(self.warnings(), ['warn.dlss_download_failed'])
def test_keeps_the_players_library(self):
(self.dest / 'nvngx_dlss.dll').write_bytes(b'newer')
self.assertEqual(steps.Dlss().run(self.ctx), {'kept': True})
self.assertEqual((self.dest / 'nvngx_dlss.dll').read_bytes(), b'newer')
self.assertEqual(self.warnings(), ['warn.dlss_kept'])
def test_skipped_without_an_rtx_gpu(self):
self.ctx.host = {'devices': [{'name': 'AMD Radeon RX 7800 XT', 'api': [1, 3, 0], 'type': 2}]}
self.assertEqual(steps.Dlss().run(self.ctx), {'skipped': 'no_rtx'})
self.assertEqual(self.warnings(), [])
if __name__ == '__main__':
unittest.main()

809
tests/jumptables_check.py Normal file
View File

@@ -0,0 +1,809 @@
"""Check of recomp/jumptables.py on every `br` in the game's modules against an emulation oracle.
The oracle is LAS Recomp's jump-table method: find `cmp #N; b.hi/b.hs` before the br (or a `b.ls/b.lo` to the
dispatch block), replay ADRP/ADR/ADD-imm from the function entry, then single-step Unicorn from the bounds check to
the br once per index. Functions are [seed, next seed) over the module's seeds plus BL targets.
Reports per module: br sites, resolved by the resolver / by the oracle, agreement, `is_computed` sites left
unresolved, tables whose targets leave the function, an independent check of hoisted table bases (hoisted_base),
PLT stubs (`br x17` after the GOT load) that must stay a plain call, and resolver time. A disagreement is counted
as 'oracle wrong' only with independent evidence (oracle_wrong). Samples of every disagreement, unverified base and
unresolved computed site are printed with disassembly. Output also goes to build/logs/jumptables-check.log, and one
row per table-like br to build/logs/jumptables-<module>.tsv.
The oracle is adapted from LAS Recomp tools/gen.py @ da35b83 (MIT, Copyright (c) 2026 Peiikai).
Exits with 1 if any disagreement is unexplained, a hoisted base is not verified, or a PLT stub counts as computed.
Usage: jumptables_check.py [--exefs DIR] [--modules main,sdk,subsdk0] [--samples N] [--no-oracle]
"""
import argparse
import bisect
import sys
import time
from collections import Counter
from pathlib import Path
import capstone
import numpy as np
import unicorn as uc
from unicorn import arm64_const as A
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'recomp'))
import jumptables # noqa: E402
import layout # noqa: E402
from lifter import bits, sext # noqa: E402
from module import Module # noqa: E402
SCRATCH_MEM = 0x7F00000000
LOG = ROOT / 'build' / 'logs' / 'jumptables-check.log'
_md = capstone.Cs(capstone.CS_ARCH_ARM64, capstone.CS_MODE_ARM)
class Oracle:
"""LAS Recomp's Unicorn jump-table resolver (`Module.jump_table` and helpers in tools/gen.py)."""
def __init__(self, mod):
self.m = mod
mu = uc.Uc(uc.UC_ARCH_ARM64, uc.UC_MODE_ARM)
mu.mem_map(mod.base, len(mod.image))
mu.mem_write(mod.base, mod.image)
mu.mem_map(SCRATCH_MEM, 0x100000)
mu.reg_write(A.UC_ARM64_REG_CPACR_EL1, 0x300000)
self.mu = mu
self.set_at = self.regs = None
self.start = self.cmp = None # the first instruction of the bounds-check block, the bounds cmp
self.guard = None # _taken: the b.ls/b.lo into the dispatch block (set_at)
self.count = None # indices the bounds check lets through
self.exclude = frozenset() # indices the emulation skips (oracle_wrong: excluded on every way in)
self.watch = self.seen = None # (address, register): record the register when the emulation reaches address
def word(self, a):
return self.m.word(a)
def kind(self, a):
return self.m.lift(a).kind
def jump_table(self, br_pc, entry, end):
"""Targets, or None. Leaves the index set point and the registers set there in self.set_at, self.regs."""
self.set_at = self.regs = self.start = self.cmp = self.guard = self.count = None
r = self._fallthrough(br_pc, entry, end)
if r is None:
r = self._taken(br_pc, entry, end)
return r
def _index_regs(self, start, at, idx):
def copy_src(w):
if (w & 0x7F800000) == 0x12000000: # AND (imm)
return bits(w, 5, 5)
if (w & 0x7FE0FFE0) == 0x2A0003E0: # MOV (ORR with zr)
return bits(w, 16, 5)
if (w & 0x7F800000) in (0x53000000, 0x13000000): # UBFM/SBFM
return bits(w, 5, 5)
return None
regs = {idx}
changed = True
while changed:
changed = False
for a in range(start, at, 4):
w = self.word(a)
rd, src = w & 31, copy_src(w)
if src is None or src >= 31 or rd >= 31:
continue
for x, y in ((rd, src), (src, rd)):
if x in regs and y not in regs:
regs.add(y)
changed = True
return sorted(regs)
def _reset(self, entry, start):
mu = self.mu
for r in range(29):
mu.reg_write(A.UC_ARM64_REG_X0 + r, SCRATCH_MEM + 0x80000)
mu.reg_write(A.UC_ARM64_REG_SP, SCRATCH_MEM + 0x80000)
a = entry
while a < start:
w = self.word(a)
if (w & 0x1F000000) == 0x10000000 or (w & 0xFF800000) == 0x91000000:
mu.emu_start(a, 0, count=1)
a += 4
def _step_to(self, frm, to):
mu, pc = self.mu, frm
for _ in range(64):
if self.watch and pc == self.watch[0] and self.seen is None:
self.seen = self._read(self.watch[1])
if pc == to:
return True
mu.emu_start(pc, 0, count=1)
pc = mu.reg_read(A.UC_ARM64_REG_PC)
return False
def _read(self, rn):
return self.mu.reg_read(A.UC_ARM64_REG_X0 + rn if rn < 29 else
(A.UC_ARM64_REG_X29 if rn == 29 else A.UC_ARM64_REG_X30))
def _emulate(self, br_pc, entry, end, start, regs, count, set_at):
rn = bits(self.word(br_pc), 5, 5)
targets = []
try:
for i in range(count):
if i in self.exclude:
continue
self._reset(entry, start)
if set_at != start and not self._step_to(start, set_at):
return None
for r in regs:
self.mu.reg_write(A.UC_ARM64_REG_X0 + r if r < 29 else A.UC_ARM64_REG_X29, i)
if not self._step_to(set_at, br_pc):
return None
t = self._read(rn)
if not (entry <= t < end) or t % 4:
return None
targets.append(t)
except uc.UcError:
return None
return sorted(set(targets))
def _taken(self, br_pc, entry, end):
s = br_pc
while s - 4 >= entry and br_pc - (s - 4) <= 4 * 12 and self.kind(s - 4) in ('op', 'asm'):
s -= 4
for g in range(entry, end, 4):
w = self.word(g)
if (w & 0xFF000010) != 0x54000000 or bits(w, 0, 4) not in (9, 3):
continue
if g + sext(bits(w, 5, 19), 19) * 4 != s:
continue
cw = self.word(g - 4) if g - 4 >= entry else 0
if (cw & 0x7F80001F) != 0x7100001F or bits(cw, 22, 1):
continue
limit = bits(cw, 10, 12)
count = limit + 1 if bits(w, 0, 4) == 9 else limit
if count <= 0 or count > 4096:
continue
self.set_at, self.regs, self.guard, self.cmp, self.count = s, [bits(cw, 5, 5)], g, g - 4, count
b = g - 4
while b - 4 >= entry and g - 4 - (b - 4) <= 4 * 12 and self.kind(b - 4) in ('op', 'asm'):
b -= 4
self.start = b
return self._emulate(br_pc, entry, end, s, [bits(cw, 5, 5)], count, s)
return None
def _fallthrough(self, br_pc, entry, end):
a = br_pc - 4
guard = cmp = None
while a >= entry and br_pc - a <= 4 * 16:
w = self.word(a)
if (w & 0xFF000010) == 0x54000000 and bits(w, 0, 4) in (8, 2):
guard = a
break
if self.kind(a) in ('b', 'bl', 'blr', 'br', 'ret', 'trap'):
return None
a -= 4
if guard is None:
return None
a = guard - 4
while a >= entry and guard - a <= 4 * 8:
if (self.word(a) & 0x7F80001F) == 0x7100001F:
cmp = a
break
if self.kind(a) != 'op':
return None
a -= 4
if cmp is None:
return None
w = self.word(cmp)
if bits(w, 22, 1):
return None
limit = bits(w, 10, 12)
count = limit + 1 if bits(self.word(guard), 0, 4) == 8 else limit
if count <= 0 or count > 4096:
return None
start = cmp
while start - 4 >= entry and cmp - (start - 4) <= 4 * 12 and self.kind(start - 4) in ('op', 'asm'):
start -= 4
regs = self._index_regs(start, cmp, bits(w, 5, 5))
self.set_at, self.regs, self.start, self.cmp, self.count = cmp, regs, start, cmp, count
rn = bits(self.word(br_pc), 5, 5)
targets = []
try:
for i in range(count):
if i in self.exclude:
continue
self._reset(entry, start)
if not self._step_to(start, cmp):
return None
for r in regs:
self.mu.reg_write(A.UC_ARM64_REG_X0 + r if r < 29 else A.UC_ARM64_REG_X29, i)
if not self._step_to(cmp, br_pc):
return None
t = self._read(rn)
if not (entry <= t < end) or t % 4:
return None
targets.append(t)
except uc.UcError:
return None
return sorted(set(targets))
_mdd = capstone.Cs(capstone.CS_ARCH_ARM64, capstone.CS_MODE_ARM)
_mdd.detail = True
CALL_CLOBBERED = set(range(19)) | {30}
def _insn(mod, a):
return next(_mdd.disasm(mod.image[a - mod.base:a - mod.base + 4], a), None)
def _regno(name):
"""X register number of a capstone register name (w and x views), 31 for sp/zr, None for others."""
if name in ('fp', 'x29', 'w29'):
return 29
if name in ('lr', 'x30', 'w30'):
return 30
if name in ('sp', 'wsp', 'xzr', 'wzr'):
return 31
return int(name[1:]) if name[0] in 'xw' and name[1:].isdigit() else None
def _writes(ins):
"""X register numbers an instruction writes, by capstone (w and x views folded), plus the call clobbers."""
if ins is None:
return set(range(31))
if ins.mnemonic in ('cmp', 'cmn', 'tst', 'ccmp', 'ccmn'):
return set() # capstone lists the first operand of these aliases as written
try:
_, w = ins.regs_access()
except capstone.CsError:
return set(range(31))
out = set()
for r in w:
n = _regno(ins.reg_name(r))
if n is not None and n < 31:
out.add(n)
if ins.mnemonic in ('bl', 'blr'):
out |= CALL_CLOBBERED
return out
class Reaching:
"""Reaching definitions of single registers over the resolver's blocks, with capstone's write sets."""
def __init__(self, mod, cfg):
self.mod, self.cfg, self.cache = mod, cfg, {}
self.writes = {}
for b, stop in zip(cfg.leaders, cfg.stops):
for x in range(b, stop, 4):
self.writes[x] = _writes(_insn(mod, x))
def at_blocks(self, reg):
"""{block start: definitions of reg reaching it}; 'entry' for a value from outside the function."""
if reg in self.cache:
return self.cache[reg]
cfg = self.cfg
gen = {}
for b, stop in zip(cfg.leaders, cfg.stops):
gen[b] = next((x for x in range(stop - 4, b - 4, -4) if reg in self.writes[x]), None)
ins = {b: {'entry'} if b in cfg.unknown else set() for b in cfg.leaders}
changed = True
while changed:
changed = False
for b in cfg.leaders:
new = set(ins[b])
for src, _ in cfg.preds[b]:
p = cfg.block(src)
new |= {gen[p]} if gen[p] is not None else ins[p]
if new != ins[b]:
ins[b], changed = new, True
self.cache[reg] = ins
return ins
def at(self, reg, a):
"""Definitions of reg reaching instruction a."""
b = self.cfg.block(a)
last = next((x for x in range(a - 4, b - 4, -4) if reg in self.writes[x]), None)
return {last} if last is not None else self.at_blocks(reg)[b]
def values(self, reg, a, active=None):
"""Constants reg holds at a, following adrp/add/adr and register copies; None if one is not constant.
A definition already being evaluated (a cycle of copies) adds no value of its own.
"""
active = set() if active is None else active
out = set()
for d in self.at(reg, a):
if d == 'entry':
return None
if (reg, d) in active:
continue
active.add((reg, d))
ins = _insn(self.mod, d)
ops = ins.operands
if ins.mnemonic in ('adrp', 'adr'):
out.add(ops[1].imm)
elif ins.mnemonic == 'add' and len(ops) == 3 and ops[2].type == capstone.arm64.ARM64_OP_IMM \
and _regno(ins.reg_name(ops[1].reg)) == reg:
prev = self.values(reg, d, active)
if prev is None:
return None
imm = ops[2].imm << (ops[2].shift.value if ops[2].shift.type else 0)
out |= {v + imm for v in prev}
elif ins.mnemonic == 'mov' and len(ops) == 2 and ops[1].type == capstone.arm64.ARM64_OP_REG \
and ins.reg_name(ops[1].reg)[0] in 'xlf' and (_regno(ins.reg_name(ops[1].reg)) or 0) < 31:
prev = self.values(_regno(ins.reg_name(ops[1].reg)), d, active)
if prev is None:
return None
out |= prev
else:
return None
return out
class FunctionCheck:
"""The resolver's final tables and blocks for one function, with reaching definitions on demand."""
def __init__(self, mod, entry, end):
self.mod, self.entry, self.end = mod, entry, end
self.tables = {a: t for a, t in jumptables._function(mod, entry, end).items() if t is not None}
self.cfg = jumptables._Cfg(mod, entry, end, self.tables, {})
self._reaching = self._facts = None
@property
def reaching(self):
if self._reaching is None:
self._reaching = Reaching(self.mod, self.cfg)
return self._reaching
@property
def facts(self):
if self._facts is None:
self._facts = jumptables._facts(self.cfg) or {}
return self._facts
def table_load(mod, s, br):
"""Last register-offset load in [s, br): the table read of a dispatch block."""
a = br
while a > s:
a -= 4
ins = _insn(mod, a)
if ins and ins.mnemonic.startswith('ldr') and len(ins.operands) == 2 \
and ins.operands[1].type == capstone.arm64.ARM64_OP_MEM and ins.operands[1].mem.index:
return ins
return None
def hoisted_base(fc, br):
"""(base register, its single constant at the dispatch block or None, verdict) of the table read before br.
The verdict is 'local' when the dispatch block writes the base, 'verified' when every definition reaching the
block (over the resolver's blocks and table edges, with capstone's write sets, through register copies) is an
adrp/add or adr of one and the same constant and the resolver's fact does not name another, else a reason.
"""
mod, s = fc.mod, fc.cfg.block(br)
load = table_load(mod, s, br)
if load is None:
return None, None, 'no table load'
reg = _regno(load.reg_name(load.operands[1].mem.base))
if reg is None or reg == 31 or any(reg in _writes(_insn(mod, x)) for x in range(s, load.address, 4)):
return reg, None, 'local'
values = fc.reaching.values(reg, s)
if values is None or len(values) != 1:
return reg, None, 'not one constant' if values else 'not constant'
value = values.pop()
fact = fc.facts.get(s, {}).get(reg)
if fact is not None and fact != 'z' and fact[0] == 'c' and fact[1] != value:
return reg, value, 'facts disagree'
return reg, value, 'verified'
def _reads(ins):
try:
r, _ = ins.regs_access()
except capstone.CsError:
return set(range(32))
return {n for n in (_regno(ins.reg_name(x)) for x in r) if n is not None}
def _branch_target(ins):
"""Target of a direct b, b.cond, cbz/cbnz or tbz/tbnz, else None."""
if ins is None or not ins.operands or not (ins.mnemonic in ('b', 'cbz', 'cbnz', 'tbz', 'tbnz')
or ins.mnemonic.startswith('b.')):
return None
op = ins.operands[-1]
return op.imm if op.type == capstone.arm64.ARM64_OP_IMM else None
def _ways_in(fc, lo, hi, tables):
"""Direct branches of the function into [lo, hi], or None when the range is entered otherwise too: by falling
through into lo, at the function entry, or as a target of another table."""
mod = fc.mod
if lo <= fc.entry or any(lo <= t <= hi for ts in tables.values() if ts != 'all' for t in ts):
return None
prev = _insn(mod, lo - 4)
if prev is None or prev.mnemonic not in ('b', 'br', 'ret'):
return None
out = []
for a in range(fc.entry, fc.end, 4):
ins = _insn(mod, a)
t = _branch_target(ins)
if t is not None and lo <= t <= hi:
out.append((a, ins, t))
return out
# Instructions an index slice may hold: integer arithmetic that reads only general registers and writes the flags at
# most (the flags-reading forms adc/sbc/csel/ccmp/... stay out).
_FLAG_SETTERS = {'cmp', 'cmn', 'tst', 'ands', 'bics', 'adds', 'subs', 'negs'}
_PURE_ALU = {'mov', 'movz', 'movn', 'movk', 'add', 'sub', 'and', 'orr', 'eor', 'bic', 'orn', 'eon', 'lsl', 'lsr',
'asr', 'ror', 'ubfx', 'sbfx', 'ubfiz', 'sbfiz', 'bfi', 'bfxil', 'uxtb', 'uxth', 'uxtw', 'sxtb', 'sxth',
'sxtw', 'neg', 'mvn', 'mul', 'madd', 'msub', 'udiv', 'sdiv', 'ubfm', 'sbfm', 'bfm'} | _FLAG_SETTERS
_DEPTH = 4 # predecessor blocks above the dispatch region
def _ends_block(ins):
return ins is None or _branch_target(ins) is not None or ins.mnemonic in ('br', 'ret', 'brk', 'udf')
class _Ways:
"""Direct branches and fall-throughs of one function, with the addresses control may enter from elsewhere."""
def __init__(self, fc, tables):
mod = self.mod = fc.mod
self.entry, self.into = fc.entry, {}
for a in range(fc.entry, fc.end, 4):
ins = _insn(mod, a)
t = _branch_target(ins)
if t is not None and fc.entry <= t < fc.end:
self.into.setdefault(t, []).append((a, ins))
self.roots = {fc.entry} | set(fc.cfg.unknown)
self.roots |= {t for ts in tables.values() if ts != 'all' for t in ts}
self.leaders = self.roots | set(self.into)
def block_start(self, a):
s = a
while s > self.entry and s not in self.leaders and not _ends_block(_insn(self.mod, s - 4)):
s -= 4
return s
def preds(self, s):
"""(branch or last instruction, insn, taken) for every way into s."""
out = [(a, ins, True) for a, ins in self.into.get(s, ())]
if s > self.entry:
p = _insn(self.mod, s - 4)
if p is not None and p.mnemonic not in ('b', 'br', 'ret', 'brk', 'udf'):
out.append((s - 4, p, False))
return out
def _edge_holds(oracle, ways, start, src, ins, taken, reg, k):
"""Whether the way out of the block [start, src] over the branch at src can be taken with x<reg> = k at src:
False only when the branch condition is a function of x<reg> alone (its slice, back to the last write of x<reg>,
is plain arithmetic and flag setting on x<reg>), evaluated by Unicorn; True otherwise."""
if _branch_target(ins) is None or ins.mnemonic == 'b':
return True
mod = ways.mod
def gp_reads(i): # general registers read, zr dropped; None when sp or a non-general register is read
try:
r, _ = i.regs_access()
except capstone.CsError:
return None
if i.mnemonic in ('cmp', 'cmn', 'tst'): # capstone lists their first operand as written, not read
r = list(r) + [op.reg for op in i.operands if op.type == capstone.arm64.ARM64_OP_REG]
out = set()
for x in r:
name = i.reg_name(x)
if name in ('xzr', 'wzr', 'nzcv'):
continue
n = _regno(name)
if n is None or n == 31:
return None
out.add(n)
return out
def sets_flags(i):
try:
return i.mnemonic in _FLAG_SETTERS or 'nzcv' in (i.reg_name(r) for r in i.regs_access()[1])
except capstone.CsError:
return True
if ins.mnemonic.startswith('b.'):
setter = next((x for x in range(src - 4, start - 4, -4) if sets_flags(_insn(mod, x))), None)
if setter is None or _insn(mod, setter).mnemonic not in _FLAG_SETTERS or any(reg in _writes(_insn(mod, x)) for x in range(setter, src, 4)):
return True
need, top, sl = gp_reads(_insn(mod, setter)), setter, [setter]
else:
need, top, sl = gp_reads(ins), src, []
if need is None:
return True
for x in range(top - 4, start - 4, -4):
i = _insn(mod, x)
w = _writes(i)
if reg in w:
break # x<reg> is k from here on
if not w & need:
continue
r = gp_reads(i)
if i.mnemonic not in _PURE_ALU or r is None:
return True
sl.append(x)
need = (need - w) | r
if need - {reg}:
return True
mu = oracle.mu
try:
for r in range(29):
mu.reg_write(A.UC_ARM64_REG_X0 + r, SCRATCH_MEM + 0x80000)
mu.reg_write(A.UC_ARM64_REG_X0 + reg if reg < 29 else A.UC_ARM64_REG_X29, k)
for x in reversed(sl):
mu.emu_start(x, 0, count=1)
mu.emu_start(src, 0, count=1)
went = mu.reg_read(A.UC_ARM64_REG_PC) != src + 4
except uc.UcError:
return True
return went == taken
def _way_live(oracle, ways, src, ins, taken, reg, k, depth, seen):
"""Whether the way over src can carry x<reg> = k: its edge holds (_edge_holds) and its block either writes x<reg>
or is entered with k (_reaches)."""
start = ways.block_start(src)
if not _edge_holds(oracle, ways, start, src, ins, taken, reg, k):
return False
if any(reg in _writes(_insn(ways.mod, x)) for x in range(start, src + 4, 4)):
return True
return _reaches(oracle, ways, start, reg, k, depth, seen)
def _reaches(oracle, ways, s, reg, k, depth, seen):
"""Whether control can enter s with x<reg> = k, over at most `depth` predecessor blocks."""
if s in ways.roots or depth == 0 or s in seen:
return True
seen = seen | {s}
return any(_way_live(oracle, ways, src, ins, taken, reg, k, depth - 1, seen)
for src, ins, taken in ways.preds(s))
def excluded_indices(fc, br, oracle, tables, count):
"""Indices < count that cannot reach br: no way into the dispatch region [oracle.start, br] (its first instruction
and every branch target inside it) can carry them. The index is the register of the oracle's bounds cmp, which no
instruction from such an entry to br may write. Ways in are the module's direct branches and fall-throughs by
capstone; the function entry, exception landing pads and every resolver table target count as entries from
anywhere. Branch conditions are evaluated only where they depend on the index alone (_edge_holds). When the
oracle enters the dispatch block by the taken bounds branch (_taken), that branch must be its only way in."""
mod, lo = fc.mod, oracle.start
if lo is None:
return frozenset()
if oracle.guard is not None:
ways = _ways_in(fc, oracle.set_at, br, tables)
if ways is None or [a for a, _, _ in ways] != [oracle.guard]:
return frozenset()
reg = bits(oracle.word(oracle.cmp), 5, 5)
ways = _Ways(fc, tables)
# The region falls through from lo to br (_fallthrough), or is the bounds-check block whose b.ls/b.lo is the only
# way into the dispatch block (_taken).
hi = br if oracle.guard is None else oracle.cmp
if oracle.guard is not None and any(reg in _writes(_insn(mod, x)) for x in range(oracle.set_at, br, 4)):
return frozenset()
entries = [lo] + [t for t in range(lo + 4, hi + 4, 4) if t in ways.leaders]
for e in entries:
if e in ways.roots or any(reg in _writes(_insn(mod, x)) for x in range(e, hi, 4)):
return frozenset()
out = set()
for k in range(count):
live = False
for e in entries:
for src, ins, taken in ways.preds(e):
if lo <= src < hi:
continue # inside the region: its own conditions are ignored
if _way_live(oracle, ways, src, ins, taken, reg, k, _DEPTH, frozenset()):
live = True
break
if live:
break
if not live:
out.add(k)
return frozenset(out)
def oracle_wrong(fc, br, oracle, ours=None):
"""Why the oracle's targets for br are not the table's, from evidence independent of the resolver; or None.
- the index the table is read with does not depend on the registers the oracle sets (it comes from another
register or is reloaded from memory, which the oracle leaves at its scratch value);
- the table base the oracle uses (its replay of ADRP/ADR/ADD-imm from the entry) is not the one constant that
every reaching definition gives;
- the oracle enumerates an index that no way into the dispatch region lets through (excluded_indices: a case
peeled off before the table by `cmp wI, #k; b.ne`, `cbnz wI` or a range check on wI, possibly some blocks up),
and without it the oracle gives the resolver's targets.
"""
mod = fc.mod
s = fc.cfg.block(br)
load = table_load(mod, s, br)
if load is None or oracle.set_at is None:
return None
live = {_regno(load.reg_name(load.operands[1].mem.index))}
a = load.address
while a > oracle.set_at:
a -= 4
ins = _insn(mod, a)
w = _writes(ins)
if live & w:
live = (live - w) | _reads(ins)
if not live & set(oracle.regs):
return 'index not set by the oracle (live-in ' + ', '.join(f'x{r}' for r in sorted(live)) + ')'
reg, value, verdict = hoisted_base(fc, br)
if verdict == 'verified':
oracle.watch = (load.address, reg)
oracle.jump_table(br, fc.entry, fc.end)
seen, oracle.watch, oracle.seen = oracle.seen, None, None
if seen is not None and seen != value:
return f'oracle base x{reg} = {seen:#x}, every reaching definition gives {value:#x}'
if ours is not None:
oracle.jump_table(br, fc.entry, fc.end)
ex = excluded_indices(fc, br, oracle, fc.tables, oracle.count) if oracle.count else frozenset()
if ex:
oracle.exclude = ex
try:
again = oracle.jump_table(br, fc.entry, fc.end)
finally:
oracle.exclude = frozenset()
if again == ours:
return 'index ' + ', '.join(str(k) for k in sorted(ex)) + ' unreachable on every way in'
return None
def plt_stubs(mod):
"""Addresses of `br x17` in `adrp x16; ldr x17, [x16, #o]; add x16, x16, #o; br x17` stubs."""
w = mod.words
m = ((w[3:] == 0xD61F0220) & ((w[:-3] & 0x9F00001F) == 0x90000010) & ((w[1:-2] & 0xFFC003FF) == 0xF9400211)
& ((w[2:-1] & 0xFFC003FF) == 0x91000210))
return (mod.lo + 4 * (np.nonzero(m)[0] + 3)).tolist()
def function_starts(mod):
"""Seeds plus BL targets in .text."""
w = mod.words.astype(np.int64)
pc = mod.lo + 4 * np.arange(len(w), dtype=np.int64)
bl = (w & 0xFC000000) == 0x94000000
t = pc[bl] + 4 * (((w[bl] & 0x3FFFFFF) ^ 0x2000000) - 0x2000000)
starts = set(mod.seeds()) | {a for a in t.tolist() if mod.in_text(a)}
return sorted(starts)
def disasm(mod, lo, hi, marks=()):
out = []
for ins in _md.disasm(mod.image[lo - mod.base:hi - mod.base], lo):
tag = '>' if ins.address in marks else ' '
out.append(f' {tag}{ins.address - mod.base:8x}: {ins.mnemonic} {ins.op_str}')
return out
def window(mod, br, entry, n=24):
"""Code before the br, and the bounds check that branches into its block if it sits elsewhere."""
lo = max(entry, br - 4 * n)
lines = disasm(mod, lo, br + 4, {br})
labels = jumptables.branch_sources(mod, lo, br + 4)
far = sorted({s for d, ss in labels.items() for s in ss if not lo <= s <= br})
for s in far[:3]:
lines.append(f' ... reached from {s - mod.base:x}:')
lines += disasm(mod, max(entry, s - 16), s + 4, {s})
return lines
def main():
ap = argparse.ArgumentParser()
ap.add_argument('--exefs', default=str(ROOT / 'build' / 'extracted' / 'v121' / 'exefs'))
ap.add_argument('--modules', default='main,sdk,subsdk0')
ap.add_argument('--samples', type=int, default=8)
ap.add_argument('--no-oracle', action='store_true')
args = ap.parse_args()
LOG.parent.mkdir(parents=True, exist_ok=True)
log = open(LOG, 'w', encoding='utf-8')
def out(s=''):
print(s)
log.write(s + '\n')
bases = {n: b for n, b, _ in layout.layout(args.exefs)}
failed = 0
for name in args.modules.split(','):
mod = Module(name, Path(args.exefs) / name, bases[name])
starts = function_starts(mod)
brs = (mod.lo + 4 * np.nonzero((mod.words & 0xFFFFFC1F) == 0xD61F0000)[0]).tolist()
oracle = None if args.no_oracle else Oracle(mod)
stats, t_res, t_orc = Counter(), 0.0, 0.0
samples, fc = {}, None
rows = open(LOG.with_name(f'jumptables-{name}.tsv'), 'w', encoding='utf-8')
rows.write('br\tentry\tend\tcomputed\tresolver\toracle\tcategory\n')
for br in brs:
i = bisect.bisect_right(starts, br) - 1
entry, end = starts[i], starts[i + 1] if i + 1 < len(starts) else mod.hi
t0 = time.perf_counter()
raw = jumptables.table_targets(mod, br, entry, end)
ours = jumptables.resolve(mod, br, entry, end)
comp = jumptables.is_computed(mod, br, entry)
t1 = time.perf_counter()
orc = oracle.jump_table(br, entry, end) if oracle else None
t_res += t1 - t0
t_orc += time.perf_counter() - t1
stats['br'] += 1
stats['computed'] += comp
if raw is not None and ours is None:
stats['targets outside function'] += 1
samples.setdefault('targets outside function', []).append((br, entry, end, raw, orc))
if (ours is not None or orc is not None) and (fc is None or (fc.entry, fc.end) != (entry, end)):
fc = FunctionCheck(mod, entry, end)
why = None
if ours is not None:
stats['resolved'] += 1
stats['entries'] += len(ours)
reg, _, verdict = hoisted_base(fc, br)
stats[f'base {verdict}' if verdict in ('local', 'verified') else 'base not verified'] += 1
if verdict not in ('local', 'verified'):
samples.setdefault('base not verified', []).append((br, entry, end, ours, orc))
rows.write(f'{br - mod.base:x}\t{entry - mod.base:x}\t{end - mod.base:x}\t-\t-\t-\t'
f'base x{reg}: {verdict}\n')
if not comp:
stats['resolved, not computed'] += 1
samples.setdefault('resolved, not computed', []).append((br, entry, end, ours, orc))
if orc is not None:
stats['oracle resolved'] += 1
if ours is not None and orc is not None:
key = 'agree' if ours == orc else 'disagree'
elif ours is not None:
key = 'only resolver'
elif orc is not None:
key = 'only oracle'
else:
key = 'computed, unresolved' if comp else None
if key in ('disagree', 'only oracle'):
why = oracle_wrong(fc, br, oracle, ours)
if why:
key = 'oracle wrong' if key == 'disagree' else 'only oracle, oracle wrong'
if key or raw is not None:
rows.write(f'{br - mod.base:x}\t{entry - mod.base:x}\t{end - mod.base:x}\t{int(comp)}\t'
f'{"-" if raw is None else len(raw)}\t{"-" if orc is None else len(orc)}\t{key}'
f'{": " + why if why else ""}\n')
if key:
stats[key] += 1
if key != 'agree':
samples.setdefault(key, []).append((br, entry, end, ours if ours is not None else raw, orc, why))
rows.close()
plt = plt_stubs(mod)
stats['PLT stubs'] = len(plt)
stats['PLT stubs computed or resolved'] = sum(
jumptables.is_computed(mod, a, a - 12) or jumptables.table_targets(mod, a, a - 12, a + 4) is not None
for a in plt)
out(f'== {name}: {len(starts)} functions')
for k in ('br', 'computed', 'resolved', 'entries', 'oracle resolved', 'agree', 'disagree', 'oracle wrong',
'only resolver', 'only oracle', 'only oracle, oracle wrong', 'computed, unresolved',
'resolved, not computed', 'targets outside function', 'base local', 'base verified',
'base not verified', 'PLT stubs', 'PLT stubs computed or resolved'):
out(f' {k:32s} {stats[k]}')
failed += stats['disagree'] + stats['base not verified'] + stats['PLT stubs computed or resolved']
out(f' {"resolver time":32s} {t_res:.2f} s')
if oracle:
out(f' {"oracle time":32s} {t_orc:.2f} s')
for key in ('disagree', 'only oracle', 'base not verified', 'oracle wrong', 'only oracle, oracle wrong',
'computed, unresolved', 'resolved, not computed', 'targets outside function', 'only resolver'):
for br, entry, end, ours, orc, *why in samples.get(key, [])[:args.samples]:
out(f'-- {key}: br {br - mod.base:x} in [{entry - mod.base:x}, {end - mod.base:x})'
+ (f': {why[0]}' if why and why[0] else ''))
fmt = lambda t: 'None' if t is None else f'{len(t)} [' + ' '.join(
f'{a - mod.base:x}' for a in t[:12]) + (' ...' if len(t) > 12 else '') + ']'
out(f' resolver {fmt(ours)}')
out(f' oracle {fmt(orc)}')
for line in window(mod, br, entry):
out(line)
log.close()
return 1 if failed else 0
if __name__ == '__main__':
sys.exit(main())

178
tests/mods_romfs_check.py Normal file
View File

@@ -0,0 +1,178 @@
"""The mods folder served as RomFS overlays (runtime/romfs.c, runtime/mods.c): load order, added files, case, Unicode.
.venv\\Scripts\\python tests\\mods_romfs_check.py --exe build\\v121-<tag>\\nx2navi.exe --version v121
Builds a mods folder in a temp directory with files of its own bytes at real RomFS paths (from
build/extracted/<version>/romfs/index.json), runs `nx2navi.exe --probe-fs` on it and reads the `mods:` lines of the log.
--probe-fs compares the bytes it reads through the RomFS with every overlay file, so a wrong winner fails it too.
A second folder checks the merged resource size table the start writes to the profile (runtime/romfs.c
restbl_overlay): a grown file's entry, an added file's, a shipped table's and another version's table's.
Needs the extracted RomFS of the version.
"""
import argparse
import json
import os
import struct
import subprocess
import sys
import tempfile
import zlib
from pathlib import Path
import zstandard
ROOT = Path(__file__).resolve().parents[1]
UNI = 'Ж мод'
def pick_paths(version):
index = json.loads((ROOT / 'build/extracted' / version / 'romfs/index.json').read_text())
paths = [f['path'] for f in index['files'] if 1024 <= f['size'] <= 65536]
have = set(f['path'] for f in index['files'])
p1 = next(p for p in paths if any(c.isalpha() for c in p.rsplit('/', 1)[-1]))
p2 = next(p for p in paths if p != p1)
i = next(k for k, c in enumerate(p1) if c.isalpha() and k > p1.rfind('/'))
p1case = p1[:i] + p1[i].swapcase() + p1[i + 1:]
assert p1case not in have
return p1, p2, p1case
def put(root, mod, rel, data, part='romfs', times=64):
f = Path(root) / mod / part / rel
f.parent.mkdir(parents=True, exist_ok=True)
f.write_bytes(data * times)
def run(exe, mods, order, disabled=''):
"""--probe-fs with a profile of its own beside mods (the merged resource size table is written there)."""
env = dict(os.environ, NX2NAVI_MODS_DIR=str(mods), NX2NAVI_MODS_ORDER=order, NX2NAVI_MODS_DISABLED=disabled)
profile = Path(mods).parent / 'profile'
r = subprocess.run([str(exe), '--profile', str(profile), '--probe-fs'], env=env, capture_output=True, timeout=600)
log = (r.stdout + r.stderr).decode('utf-8', 'replace')
return r.returncode, log
# ---- the resource size table (RESTBL: "RESTBL", u32 version, u32 name length, u32 hashed count, u32 named count,
# {u32 crc32, u32 size} sorted, then {char[name length], u32 size})
def restbl_read(data):
raw = zstandard.ZstdDecompressor().decompress(data)
assert raw[:6] == b'RESTBL'
_, name_len, hashed, named = struct.unpack_from('<4I', raw, 6)
out = dict(struct.unpack_from('<II', raw, 22 + i * 8) for i in range(hashed))
hashes = [struct.unpack_from('<I', raw, 22 + i * 8)[0] for i in range(hashed)]
assert hashes == sorted(hashes), 'hashed entries out of order'
at = 22 + hashed * 8
for _ in range(named):
out[raw[at:at + name_len].split(b'\0')[0].decode()] = struct.unpack_from('<I', raw, at + name_len)[0]
at += name_len + 4
return out, raw
def restbl_pack(entries, raw):
"""raw (a table) with the sizes of its hashed entries set from entries (crc32 -> size, existing hashes only)."""
out = bytearray(raw)
hashed = struct.unpack_from('<I', raw, 14)[0]
for i in range(hashed):
h = struct.unpack_from('<I', raw, 22 + i * 8)[0]
if h in entries:
struct.pack_into('<I', out, 26 + i * 8, entries[h])
return zstandard.ZstdCompressor(write_content_size=True).compress(bytes(out))
def key(path):
return zlib.crc32((path[:-3] if path.endswith(('.zs', '.mc')) else path).encode())
def restbl_case(exe, version, check, tmp):
romfs = ROOT / 'build/extracted' / version / 'romfs'
index = json.loads((romfs / 'index.json').read_text())
files = {f['path']: f for f in index['files']}
table_path = next(p for p in files if p.startswith('System/Resource/ResourceSizeTable.') and
p.endswith('.rsizetable.zs'))
with (romfs / 'romfs.bin').open('rb') as image:
def read(path, n=None):
image.seek(files[path]['offset'])
return image.read(n or files[path]['size'])
game, game_raw = restbl_read(read(table_path))
packs = [p for p in files if p.startswith('Pack/Actor/') and p.endswith('.pack.zs') and key(p) in game]
grown, shipped_key, old_key, old_other = packs[0], packs[1], packs[2], packs[3]
was = zstandard.get_frame_parameters(read(grown, 18)).content_size
zs = zstandard.ZstdCompressor(write_content_size=True)
mods = Path(tmp) / 'restbl' / 'mods'
put(mods, 'Grow', grown, zs.compress(bytes(was + 100000)), times=1)
put(mods, 'Grow', 'Pack/Actor/ModTest_Added.pack.zs', zs.compress(bytes(50000)), times=1)
put(mods, 'Ships', table_path, restbl_pack({key(shipped_key): game[key(shipped_key)] + 12345}, game_raw), times=1)
put(mods, 'Old', old_key, b'old', times=1) # smaller than the original: only its table raises it
put(mods, 'Old', table_path.replace('.Product.', '.Product.999', 1) if '.Product.' in table_path else
'System/Resource/ResourceSizeTable.Product.999.Nin_NX_NVN.rsizetable.zs',
restbl_pack({key(old_key): game[key(old_key)] + 7777, key(old_other): game[key(old_other)] + 5555},
game_raw), times=1)
code, log = run(exe, mods, 'Grow|Ships|Old')
check(code == 0, 'probe-fs passes with the resource size table mods', log)
merged_file = mods.parent / 'profile' / 'mods' / table_path.rsplit('/', 1)[1]
check(merged_file.is_file(), f'the merged table is written to the profile ({merged_file.name})', log)
if not merged_file.is_file():
return
merged, _ = restbl_read(merged_file.read_bytes())
g = game[key(grown)]
check(merged[key(grown)] >= g + 100000, f'a file grown by 100000 bytes: entry {g} -> {merged[key(grown)]}')
added = merged.get(key('Pack/Actor/ModTest_Added.pack.zs'), 0)
check(added >= 50000 + 256, f'an added 50000-byte pack gets an entry ({added})')
check(merged[key(shipped_key)] == game[key(shipped_key)] + 12345, 'a shipped table\'s larger entry is merged')
check(merged[key(old_key)] == game[key(old_key)] + 7777, 'another version\'s table raises its own mod\'s file')
check(merged[key(old_other)] == game[key(old_other)], 'another version\'s table leaves other files alone')
lower = [h for h, s in game.items() if merged.get(h, 0) < s]
check(not lower, f'no entry of the game\'s table is lowered ({len(lower)})')
def main():
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
ap.add_argument('--exe', type=Path, required=True)
ap.add_argument('--version', choices=('v121', 'v143'), required=True)
args = ap.parse_args()
p1, p2, p1case = pick_paths(args.version)
fails = 0
def check(ok, what, log=None):
nonlocal fails
print(('PASS ' if ok else 'FAIL ') + what)
if not ok:
fails += 1
if log:
print('\n'.join(l for l in log.splitlines() if 'mods' in l or 'overlay' in l.lower())[-3000:])
with tempfile.TemporaryDirectory() as tmp:
mods = Path(tmp) / 'mods'
put(mods, 'Top', p1, b'mod-A')
put(mods, 'Low', p1, b'mod-B')
put(mods, 'Low', p2, b'mod-C')
put(mods, 'Top', 'New/added.bin', b'mod-D')
put(mods, 'Low', 'New/added.bin', b'mod-E')
put(mods, 'Case', p1case, b'mod-F')
put(mods, UNI, p2, b'mod-G')
# a file of another game version's (no RomFS has a Product.999 table): counted, never served
put(mods, 'Low', 'System/Resource/ResourceSizeTable.Product.999.rsizetable.zs', b'mod-H')
put(mods, 'Lite', 'Lite/added.bin', b'mod-I', part='romfslite') # a mod manager's romfslite folder
code, log = run(args.exe, mods, f'Top|Low|{UNI}|Case')
check(code == 0, 'probe-fs passes with five mods (served bytes equal the winners\' files)', log)
for line in ('mods: Top: 2 files served (1 replaced, 1 added), 0 overridden',
'mods: Low: 1 files served (1 replaced, 0 added), 2 overridden by Top, 1 for another game version',
'mods: Case: 0 files served (0 replaced, 0 added), 1 overridden by Top',
f'mods: {UNI}: 0 files served (0 replaced, 0 added), 1 overridden by Low',
'mods: Lite: 1 files served (0 replaced, 1 added), 0 overridden'):
check(line in log, line, log)
check(p1case not in log, 'the case-flipped path is served as the RomFS path, not added', log)
code, log = run(args.exe, mods, f'Top|Low|{UNI}|Case', disabled='Top')
check(code == 0, 'probe-fs passes with Top switched off', log)
check('mods: Low: 3 files served (2 replaced, 1 added), 0 overridden' in log, 'Low serves P1 and the added file', log)
check('mods: Top: ' not in log.replace('mods: Top: off', ''), 'a switched-off mod serves nothing', log)
restbl_case(args.exe, args.version, check, tmp)
print('mods_romfs_check:', 'FAIL' if fails else 'ok')
return 1 if fails else 0
if __name__ == '__main__':
sys.exit(main())

311
tests/mouse_direct_check.py Normal file
View File

@@ -0,0 +1,311 @@
"""Check the direct camera hook's register contract and raw-count routing without game data.
Usage: python tests/mouse_direct_check.py --cc <clang.exe>
"""
import argparse
import os
from pathlib import Path
import subprocess
import tempfile
ROOT = Path(__file__).resolve().parents[1]
HARNESS = r'''
#include <assert.h>
#include <stdio.h>
#include "mouse_direct.c"
static int raw_x, raw_y, takes, captured=1, foreground=1, allowed=1, mode=1, patched=1, selecting;
static float ratio=0.5f;
static uint64_t now=1000000000;
static GuestFn hook, activity_hook, begin_hook, angles_hook, hand_hook;
const char *cfg_get(const char *key) { (void)key; return mode ? "direct" : "stick"; }
const char *cfg_value(const char *key) { return cfg_get(key); }
double cfg_double(const char *key) { (void)key; return 1; }
int rt_patch_set(const char *s) { (void)s; return 0; }
int rt_patch_enabled(int s) { (void)s; return patched; }
int ikb_captured(void) { return captured; }
int ikb_foreground(void) { return foreground; }
int input_cursor_allowed(void) { return allowed; }
int input_direct_mouse_selector_active(void) { return selecting; }
void ikb_mouse_take(int32_t *x, int32_t *y) { ++takes; *x=raw_x; *y=raw_y; raw_x=raw_y=0; }
void input_direct_mouse_sample(int enabled, int32_t *x, int32_t *y) {
*x=*y=0;
if(enabled && captured && foreground && allowed) ikb_mouse_take(x,y);
}
uint64_t plat_now_ns(void) { return now; }
int frame_step_realtime(void) { return 1; }
double frame_step_limit_hz(void) { return 60; }
void *rt_patch_page(void) { return &ratio; }
void rt_register_host(uint64_t addr, GuestFn fn) {
if(addr==HOST_MOUSE_DIRECT) hook=fn; else if(addr==HOST_MOUSE_ACTIVITY) activity_hook=fn;
else if(addr==HOST_MOUSE_ORBIT_BEGIN) begin_hook=fn; else if(addr==HOST_MOUSE_ORBIT_ANGLES) angles_hook=fn;
else if(addr==HOST_MOUSE_HAND) hand_hook=fn; else abort();
}
void plat_mutex_lock(PlatMutex *m) { (void)m; }
void plat_mutex_unlock(PlatMutex *m) { (void)m; }
void rt_log(const char *fmt, ...) { (void)fmt; }
static float scalar(__uint128_t v) { float f; memcpy(&f,&v,4); return f; }
int main(void) {
assert(hook && activity_hook);
Cpu c, expected;
unsigned char target[0x500] = {0};
memset(&c, 0x5a, sizeof c);
c.x[20]=(uintptr_t)target; c.x[21]=0; c.v[9]=c.v[10]=0;
expected=c;
raw_x=100; raw_y=-50;
hook(&c);
assert(fabsf(scalar(c.v[9])-2)<1e-6f);
assert(fabsf(scalar(c.v[10])-1)<1e-6f);
float stored; memcpy(&stored,target+0x420,4); assert(stored==scalar(c.v[9]));
expected.v[9]=c.v[9]; expected.v[10]=c.v[10]; assert(!memcmp(&expected,&c,sizeof c));
assert(input_direct_mouse_active());
// The next read sees no repeated movement. Gamepad/keyboard input is preserved.
expected=c; hook(&c); assert(!memcmp(&expected,&c,sizeof c));
// Equal mouse distance produces equal integrated input at 30, 60 and 144 Hz; there is no unit-circle clamp.
float rates[]={30,60,144};
for(int i=0;i<3;i++) {
ratio=30/rates[i]; raw_x=10000; raw_y=10000; c.v[9]=c.v[10]=0; hook(&c);
assert(fabsf(scalar(c.v[9])*ratio-100)<0.001f);
assert(fabsf(scalar(c.v[10])*ratio+100)<0.001f);
}
// One count is not discarded by a dead zone.
ratio=0.5f; raw_x=1; raw_y=0; c.v[9]=c.v[10]=0; hook(&c); assert(scalar(c.v[9])>0);
uint64_t stack[4]={0,0x123456789abcdef0ull,0x456789abcdef0123ull,0}; c.sp=(uintptr_t)stack;
// The flags are in w8 at the site: the game's next instruction stores w8 to +0x418, so memory is not touched.
uint32_t inactive=0x28, mem=0x77; memcpy(target+0x418,&mem,4);
c.x[8]=inactive; activity_hook(&c);
assert((uint32_t)c.x[8]==0x11);
uint32_t after; memcpy(&after,target+0x418,4); assert(after==mem);
assert(c.v[9]==stack[1] && c.v[8]==stack[2]);
// A controller-only update keeps the game's activity bits.
c.x[8]=inactive; activity_hook(&c); assert((uint32_t)c.x[8]==inactive);
// Disabled game camera input leaves raw counts for the menu's legacy mouse-to-stick path.
c.x[21]=1; raw_x=42; int before=takes; hook(&c);
assert(takes==before && raw_x==42 && !input_direct_mouse_active()); c.x[21]=0;
// Focus loss and host menus do not move the camera.
captured=0; before=takes; hook(&c); assert(takes==before); captured=1;
foreground=0; hook(&c); assert(takes==before); foreground=1;
allowed=0; hook(&c); assert(takes==before); allowed=1;
mode=0; hook(&c); assert(takes==before && !input_direct_mouse_active()); mode=1;
patched=0; hook(&c); assert(takes==before && !input_direct_mouse_active()); patched=1;
now+=150000001; assert(!input_direct_mouse_active());
// Orbit input removes the mouse from the stick path and writes equal degrees per count at the final angles.
unsigned char camera[0x300]={0}, frame[0x100]={0}, hand[0x300]={0};
uint64_t input_address=(uintptr_t)target; memcpy(camera+0x20,&input_address,8);
for(int rate=0;rate<3;rate++) {
memset(orbit_seen,0,sizeof orbit_seen);
ratio=30/rates[rate]; c.x[20]=(uintptr_t)target; c.x[21]=0; c.v[9]=c.v[10]=0;
raw_x=100; raw_y=-50; hook(&c);
guest_float_store((uintptr_t)camera+0x108,30); guest_float_store((uintptr_t)camera+0x120,40);
guest_float_store((uintptr_t)frame+0x44,30); guest_float_store((uintptr_t)frame+0x48,40);
c.x[0]=(uintptr_t)camera; begin_hook(&c);
assert(guest_float((uintptr_t)target+0x420)==0 && guest_float((uintptr_t)target+0x424)==0);
c.x[20]=(uintptr_t)camera; c.sp=(uintptr_t)frame; angles_hook(&c);
assert(fabsf(guest_float((uintptr_t)frame+0x44)-26)<0.0001f);
assert(fabsf(guest_float((uintptr_t)frame+0x48)-32)<0.0001f);
assert(fabsf(guest_float((uintptr_t)camera+0x15c)-26)<0.0001f);
c.x[0]=(uintptr_t)camera; begin_hook(&c); angles_hook(&c);
assert(fabsf(guest_float((uintptr_t)frame+0x48)-32)<0.0001f); // one sample cannot repeat next update
// The next input frame is stationary and the game's parameter update rewrites the old default target.
// Restoring the mouse's desired target makes the camera filter stay at the new pitch, not spring to zero.
c.x[20]=(uintptr_t)target; c.v[9]=c.v[10]=0; raw_x=raw_y=0; hook(&c);
guest_float_store((uintptr_t)camera+0x15c,0);
c.x[0]=(uintptr_t)camera; begin_hook(&c);
float current=guest_float((uintptr_t)camera+0x108), goal=guest_float((uintptr_t)camera+0x15c);
assert(fabsf(goal-26)<0.0001f && fabsf((current+goal)*0.5f-26)<0.0001f);
// A later game filter overwrites the pitch after the begin hook. Neither a stationary frame nor the next
// tiny mouse delta may integrate from that recentered result.
guest_float_store((uintptr_t)camera+0x108,0);
guest_float_store((uintptr_t)camera+0x11c,5);
guest_float_store((uintptr_t)camera+0x10c,20);
guest_float_store((uintptr_t)camera+0x88,0.3f);
guest_float_store((uintptr_t)frame+0x44,0);
c.x[20]=(uintptr_t)camera; angles_hook(&c);
assert(fabsf(guest_float((uintptr_t)frame+0x44)-26)<0.0001f);
assert(fabsf(guest_float((uintptr_t)camera+0x15c)-7)<0.0001f);
assert(fabsf(guest_float((uintptr_t)camera+0x108)-12)<0.0001f);
for(int i=0;i<100;i++) {
guest_float_store((uintptr_t)frame+0x44,0); angles_hook(&c);
assert(fabsf(guest_float((uintptr_t)camera+0x108)-12)<0.0001f);
}
c.x[20]=(uintptr_t)target; c.v[9]=c.v[10]=0; raw_y=1; hook(&c);
c.x[0]=(uintptr_t)camera; begin_hook(&c);
guest_float_store((uintptr_t)frame+0x44,0);
c.x[20]=(uintptr_t)camera; angles_hook(&c);
assert(fabsf(guest_float((uintptr_t)frame+0x44)-26.08f)<0.0001f);
// An explicit controller recenter still hands pitch back to the game.
uint64_t recenter=0x20; memcpy(target+0x148,&recenter,8);
c.x[20]=(uintptr_t)target; c.v[9]=c.v[10]=0; hook(&c);
c.x[0]=(uintptr_t)camera; begin_hook(&c);
guest_float_store((uintptr_t)frame+0x44,5);
c.x[20]=(uintptr_t)camera; angles_hook(&c);
assert(guest_float((uintptr_t)frame+0x44)==5);
memset(target+0x148,0,8);
memset(camera+0x88,0,4); memset(camera+0x10c,0,4); memset(camera+0x11c,0,4);
}
// A gamepad-only step (no mouse motion, no pitch hold) leaves the camera input and the angles as the game made them.
memset(orbit_seen,0,sizeof orbit_seen);
c.x[20]=(uintptr_t)target; c.x[21]=0; raw_x=raw_y=0; c.v[9]=c.v[10]=0;
{ float px=0.25f, py=-0.5f; memcpy(&c.v[9],&px,4); memcpy(&c.v[10],&py,4); }
hook(&c);
guest_float_store((uintptr_t)target+0x420,0.25f); guest_float_store((uintptr_t)target+0x424,0.75f);
guest_float_store((uintptr_t)frame+0x44,11); guest_float_store((uintptr_t)frame+0x48,22);
guest_float_store((uintptr_t)camera+0x15c,3);
c.x[0]=(uintptr_t)camera; begin_hook(&c);
assert(guest_float((uintptr_t)target+0x420)==0.25f && guest_float((uintptr_t)target+0x424)==0.75f);
c.x[20]=(uintptr_t)camera; c.sp=(uintptr_t)frame; angles_hook(&c);
assert(guest_float((uintptr_t)frame+0x44)==11 && guest_float((uintptr_t)frame+0x48)==22);
assert(guest_float((uintptr_t)camera+0x15c)==3);
guest_float_store((uintptr_t)camera+0x108,30); guest_float_store((uintptr_t)camera+0x120,40);
for(int count=0;count<100;count++) {
c.x[20]=(uintptr_t)target; c.x[21]=0; c.v[9]=c.v[10]=0; raw_x=1; raw_y=0; hook(&c);
c.x[0]=(uintptr_t)camera; begin_hook(&c); c.x[20]=(uintptr_t)camera;
guest_float_store((uintptr_t)frame+0x44,guest_float((uintptr_t)camera+0x108));
guest_float_store((uintptr_t)frame+0x48,guest_float((uintptr_t)camera+0x120));
angles_hook(&c);
}
assert(fabsf(guest_float((uintptr_t)camera+0x120)-32)<0.001f);
// A held object receives mouse deltas after its own curves, once per sample, including with camera input off.
c.x[20]=(uintptr_t)target; c.v[9]=c.v[10]=0; raw_x=100; raw_y=50; ratio=0.5f; hook(&c);
c.x[19]=(uintptr_t)hand; c.x[8]=0x12345678; c.v[15]=0;
guest_float_store((uintptr_t)frame+0x60,0); hand_hook(&c);
assert(fabsf(guest_float((uintptr_t)frame+0x60)-2)<0.00001f);
assert(fabsf(scalar(c.v[15])+1)<0.00001f);
hand_hook(&c); assert(fabsf(scalar(c.v[15])+1)<0.00001f);
c.x[21]=1; raw_x=5; raw_y=6; hook(&c); assert(raw_x==0 && raw_y==0 && input_direct_mouse_active());
// A radial selector retains its virtual cursor between mouse packets, including repeated HID polls.
float cursor[2]={0}; int32_t selector[2], held_selector[2];
input_direct_mouse_radial(100,50,1,cursor,selector);
held_selector[0]=selector[0]; held_selector[1]=selector[1];
for(int i=0;i<100;i++) {
input_direct_mouse_radial(0,0,1,cursor,selector);
assert(selector[0]==held_selector[0] && selector[1]==held_selector[1]);
}
input_direct_mouse_radial(-100,-50,1,cursor,selector); assert(!selector[0] && !selector[1]);
input_direct_mouse_radial(100000,100000,1,cursor,selector);
assert((int64_t)selector[0]*selector[0]+(int64_t)selector[1]*selector[1]<=32768ll*32768);
// Quick lists must not latch a wheel direction or automatically drain a large flick after the mouse stops.
InputMouseRow row={0}; uint64_t row_time=0;
input_direct_mouse_row(10000,1,row_time,&row,selector); assert(selector[0]==32767 && !selector[1]);
input_direct_mouse_row(0,1,row_time+10000000,&row,selector); assert(selector[0]==32767);
for(int i=1;i<=100;i++) {
input_direct_mouse_row(0,1,row_time+i*100000000ull,&row,selector); assert(!selector[0] && !selector[1]);
}
// Repeated HID polls do not rearm a pulse; a frame that skips its deadline must still observe neutral first.
row=(InputMouseRow){0};
for(int i=0;i<31;i++) { input_direct_mouse_row(1,1,0,&row,selector); assert(!selector[0]); }
input_direct_mouse_row(1,1,0,&row,selector); assert(selector[0]==32767);
input_direct_mouse_row(1000,1,100000000,&row,selector); assert(!selector[0]);
input_direct_mouse_row(1000,1,110000000,&row,selector); assert(!selector[0]);
input_direct_mouse_row(1000,1,140000000,&row,selector); assert(selector[0]==32767);
input_direct_mouse_row(-1000,1,141000000,&row,selector); assert(!selector[0]);
input_direct_mouse_row(-1,1,180000000,&row,selector); assert(selector[0]==-32767);
selecting=1; c.x[20]=(uintptr_t)target; raw_x=42; raw_y=0; before=takes; hook(&c);
assert(takes==before && raw_x==42); selecting=0; raw_x=0;
puts("direct mouse: register contract, routing, linearity, frame ratio and capture checks passed");
puts("direct mouse: final orbit angles, single-count motion, equal axes and held-object routing passed");
puts("direct mouse: persistent pitch target, radial cursor hold and gamepad-only steps untouched passed");
puts("direct mouse: final-pitch recenter rejection, explicit handoff and bounded quick-list steps passed");
}
'''
INPUT_HARNESS = r'''
#include <assert.h>
#include "input.c"
GfxStats gfx_counters;
InputMap input_map;
// Other input entry points are linked by PE unwind metadata; reaching them is outside this script-only test.
void ikb_cursor_update(void) { abort(); }
void ikb_menu_closed(void) { abort(); }
int cfg_bool(const char *key) { (void)key; abort(); }
int input_menu_gate(int *h,int open,uint64_t *b,int32_t s[4]) { (void)h;(void)open;(void)b;(void)s;abort(); }
int input_overlay_gate(int *h,uint64_t *b) { (void)h;(void)b;abort(); }
int menu_is_open(void) { abort(); }
void menu_toggle(void) { abort(); }
void menu_perf_cycle(void) { abort(); }
const char *input_action_name(int action) { (void)action;abort(); }
void input_map_configure(void) { abort(); }
int input_direct_mouse_active(void) { return 1; }
int input_direct_mouse_enabled(void) { return 1; }
static int row_calls, radial_calls, live_mouse;
void input_direct_mouse_row(int32_t dx,float sensitivity,uint64_t now,InputMouseRow *row,int32_t out[2]) {
(void)dx;(void)sensitivity;(void)now; row_calls++; out[0]=123;out[1]=0;row->direction=1;
}
void input_direct_mouse_radial(int32_t dx,int32_t dy,float sensitivity,float cursor[2],int32_t out[2]) {
(void)dx;(void)dy;(void)sensitivity;radial_calls++;cursor[0]=1;out[0]=456;out[1]=789;
}
static const char *script_path;
static uint64_t test_tick;
static int64_t test_world=-1;
const char *cfg_get(const char *key) {
if (!strcmp(key,"NX2NAVI_INPUT_SCRIPT")) return script_path;
if (!strcmp(key,"NX2NAVI_INPUT")) return "none";
return NULL;
}
unsigned cfg_version(void) { return 0; }
double cfg_double(const char *key) { (void)key; return 1; }
uint64_t frame_step_ticks(void) { return test_tick; }
int64_t frame_step_world_us(void) { return test_world; }
void rt_log(const char *fmt, ...) { (void)fmt; }
void plat_mutex_lock(PlatMutex *m) { (void)m; }
void plat_mutex_unlock(PlatMutex *m) { (void)m; }
int ikb_captured(void) { return live_mouse; }
int ikb_foreground(void) { return 0; }
void ikb_mouse_take(int32_t *x,int32_t *y) { *x=*y=0; }
uint64_t plat_now_ns(void) { return 1000000000ull; }
int main(int argc,char **argv) {
assert(argc==2); script_path=argv[1];
uint64_t buttons=0; int32_t left[2]={0},right[2]={0},dx,dy;
test_tick=10;
for(int i=0;i<3;i++) script_apply(&buttons,left,right);
input_direct_mouse_sample(1,&dx,&dy);
assert(dx==123 && dy==-45 && !buttons && !right[0] && !right[1]);
input_direct_mouse_sample(1,&dx,&dy); assert(!dx && !dy);
test_tick=20; script_apply(&buttons,left,right);
g_menu=1; input_direct_mouse_sample(1,&dx,&dy); assert(!dx && !dy);
g_menu=0; input_direct_mouse_sample(1,&dx,&dy); assert(!dx && !dy);
test_world=100000;
for(int i=0;i<3;i++) script_apply(&buttons,left,right);
rec=tmpfile(); assert(rec); rec_state=1;
input_direct_mouse_sample(1,&dx,&dy); assert(dx==-100 && dy==50);
rewind(rec); char line[128]; assert(fgets(line,sizeof line,rec));
assert(strstr(line,"MOUSE -100 50")); assert(!fgets(line,sizeof line,rec));
fclose(rec); rec_state=-1;
live_mouse=1;
mouse_camera(right,1); assert(radial_calls==1 && !row_calls && right[0]==456 && right[1]==789);
assert(input_direct_mouse_selector_active());
mouse_camera(right,2); assert(row_calls==1 && radial_calls==1 && right[0]==123 && !right[1]);
mouse_camera(right,0); assert(!input_direct_mouse_selector_active() && !right[0] && !right[1]);
mouse_camera(right,3); assert(row_calls==2 && right[0]==123 && !right[1]);
live_mouse=0; mouse_camera(right,3);
assert(!input_direct_mouse_selector_active() && !right[0] && !right[1]);
puts("direct mouse: one-shot script replay, host-menu blocking and raw-count recording passed");
puts("direct mouse: separate wheel/list routing, selector ownership and release passed");
}
'''
def main():
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument('--cc', required=True)
ap.add_argument('--deps', type=Path, help='dependency include directory (SDL3/Vulkan), to also check input scripts')
args = ap.parse_args()
with tempfile.TemporaryDirectory(prefix='nx2navi-mouse-') as tmp:
folder = Path(tmp)
(folder / 'patches.h').write_text('#define RT_PATCH_PAGE 0x7003adf000ull\n')
source = folder / 'check.c'
source.write_text(HARNESS)
exe = folder / ('check.exe' if os.name == 'nt' else 'check')
link = [] if os.name == 'nt' else ['-lm']
subprocess.run([args.cc, '-O2', '-Wall', '-Wextra', '-I', str(folder), '-I', str(ROOT/'runtime'),
str(source), *link, '-o', str(exe)], check=True)
subprocess.run([str(exe)], check=True)
if args.deps:
source.write_text(INPUT_HARNESS)
script = folder / 'input.txt'
script.write_text('10 MOUSE 123 -45\n20 MOUSE 1000 0\n@100 MOUSE -100 50\n')
subprocess.run([args.cc, '-O2', '-flto', '-ffunction-sections', '-fdata-sections', '-Wl,--gc-sections', *(['-fuse-ld=lld', '-Wl,--exclude-all-symbols'] if os.name == 'nt' else []),
'-I', str(ROOT/'runtime'), '-I', str(args.deps), '-I', str(args.deps/'vma'),
str(source), *link, '-o', str(exe)], check=True)
subprocess.run([str(exe), str(script)], check=True)
if __name__ == '__main__':
main()

173
tests/ncz_test.py Normal file
View File

@@ -0,0 +1,173 @@
"""tools/ncz.py and the installer's unpacking of NSZ input (installer/steps.py, Check.unpack) on small made-up files:
an NCA with an AES-CTR section and a plain one, stored as an NCZ with one zstd stream or as NCZBLOCK blocks.
Run: .venv-py314\\Scripts\\python tests/ncz_test.py
"""
import hashlib
import io
import os
import struct
import sys
import tempfile
import unittest
from compression import zstd
from pathlib import Path
from types import SimpleNamespace
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT))
sys.path.insert(0, str(ROOT / 'tools'))
import ncz # noqa: E402
from installer import steps # noqa: E402
from installer.events import Reporter # noqa: E402
KEY, COUNTER = bytes(range(16)), bytes(range(16, 32))
# (offset, size, crypto type): an AES-CTR section, then a plain one
SECTIONS = ((0x4000, 0x9000, 3), (0xD000, 0x3200, 1))
def pfs0(files):
"""An NSP of (name, data) pairs."""
strings = b''.join(name.encode() + b'\0' for name, _ in files)
strings += bytes(-(16 + 0x18 * len(files) + len(strings)) % 0x20)
head = b'PFS0' + struct.pack('<III', len(files), len(strings), 0)
offset = name_at = 0
for name, data in files:
head += struct.pack('<QQII', offset, len(data), name_at, 0)
offset += len(data)
name_at += len(name) + 1
return head + strings + b''.join(data for _, data in files)
def made_up_nca():
"""(the NCA as stored on the console, its body after the header with the sections decrypted)."""
rand = os.urandom
plain = rand(0x2000) + bytes(0x7000) # half random, half compressible
rest = bytes(0x1200) + rand(0x2000)
counter = COUNTER[:8] + struct.pack('>Q', 0x4000 >> 4)
secret = Cipher(algorithms.AES(KEY), modes.CTR(counter)).encryptor().update(plain)
header = rand(0x4000)
return header + secret + rest, plain + rest
def ncz_file(nca, body, blocks=False):
out = nca[:0x4000] + b'NCZSECTN' + struct.pack('<Q', len(SECTIONS))
for offset, size, crypto in SECTIONS:
out += struct.pack('<QQQQ', offset, size, crypto, 0) + KEY + COUNTER
if not blocks:
return out + zstd.compress(body)
exponent = 14
pieces = [body[i:i + (1 << exponent)] for i in range(0, len(body), 1 << exponent)]
stored = [zstd.compress(p) if p.count(0) > len(p) // 2 else p for p in pieces] # random blocks stay as they are
out += b'NCZBLOCK' + bytes([2, 1, 0, exponent]) + struct.pack('<IQ', len(pieces), len(body))
out += struct.pack(f'<{len(stored)}I', *map(len, stored))
return out + b''.join(stored)
class UnpackTest(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.dir = Path(self.tmp.name)
self.nca, self.body = made_up_nca()
self.name = hashlib.sha256(self.nca).hexdigest()[:32]
self.tik = b'ticket' * 40
def tearDown(self):
self.tmp.cleanup()
def nsz(self, blocks=False, damage=None):
data = ncz_file(self.nca, self.body, blocks)
if damage:
data = damage(bytearray(data))
path = self.dir / 'game.nsz'
path.write_bytes(pfs0([(self.name + '.ncz', bytes(data)), ('a.tik', self.tik)]))
return path
def expected(self):
return pfs0([(self.name + '.nca', self.nca), ('a.tik', self.tik)])
def test_stream_comes_back_byte_for_byte(self):
src = self.nsz()
self.assertEqual(ncz.compressed(src)[0], 'nsz')
self.assertEqual(ncz.unpacked_size(src), len(self.expected()))
seen = []
self.assertEqual(ncz.unpack(src, self.dir / 'out.nsp', lambda d, t: seen.append((d, t))), 'nsz')
self.assertEqual((self.dir / 'out.nsp').read_bytes(), self.expected())
self.assertEqual(seen[-1], (len(self.expected()), len(self.expected())))
self.assertFalse((self.dir / 'out.nsp.part').exists())
def test_blocks_come_back_byte_for_byte(self):
ncz.unpack(self.nsz(blocks=True), self.dir / 'out.nsp')
self.assertEqual((self.dir / 'out.nsp').read_bytes(), self.expected())
def test_plain_nsp_is_not_compressed(self):
path = self.dir / 'plain.nsp'
path.write_bytes(self.expected())
self.assertIsNone(ncz.compressed(path))
def test_changed_byte_fails_the_hash(self):
def flip(data):
data[0x4000 + 16 + 0x40 * len(SECTIONS) + 0x100] ^= 1 # inside a stored (random) block
return data
with self.assertRaises(ncz.DamagedFile) as e:
ncz.unpack(self.nsz(blocks=True, damage=flip), self.dir / 'out.nsp')
self.assertIn('hash mismatch', e.exception.detail)
def test_cut_stream_is_damaged(self):
with self.assertRaises(ValueError):
ncz.unpack(self.nsz(damage=lambda d: d[:-40]), self.dir / 'out.nsp')
self.assertFalse((self.dir / 'out.nsp').exists())
class CheckUnpackTest(unittest.TestCase):
"""Check.unpack: an NSZ base becomes <temp>/unpacked/base.nsp, a plain NSP update is read in place, a second run
reuses the NSP."""
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
base = Path(self.tmp.name)
nca, body = made_up_nca()
name = hashlib.sha256(nca).hexdigest()[:32]
self.base, self.update = base / 'base.nsz', base / 'update.nsp'
self.base.write_bytes(pfs0([(name + '.ncz', ncz_file(nca, body))]))
self.update.write_bytes(pfs0([('a.tik', b'x' * 64)]))
self.expected = pfs0([(name + '.nca', nca)])
self.out = io.StringIO()
self.ctx = SimpleNamespace(options=SimpleNamespace(base=str(self.base), update=str(self.update)),
temp=base / 'temp', reporter=Reporter(out=self.out, log_path=base / 'x.log'))
def tearDown(self):
self.ctx.reporter.close()
self.tmp.cleanup()
def test_unpacks_the_base_and_reuses_it(self):
steps.Check().unpack(self.ctx)
target = self.ctx.temp / 'unpacked' / 'base.nsp'
self.assertEqual(target.read_bytes(), self.expected)
self.assertEqual(steps.game_file(self.ctx, 'base'), target)
self.assertEqual(steps.game_file(self.ctx, 'update'), self.update)
self.assertIn('"phase": "unpack_base"', self.out.getvalue())
mtime = target.stat().st_mtime_ns
steps.Check().unpack(self.ctx)
self.assertEqual(target.stat().st_mtime_ns, mtime)
def test_content_decides_not_the_name(self):
renamed = self.base.with_name('base.nsp')
self.base.rename(renamed)
self.ctx.options.base = str(renamed)
steps.Check().unpack(self.ctx)
self.assertEqual((self.ctx.temp / 'unpacked' / 'base.nsp').read_bytes(), self.expected)
def test_damaged_base_names_the_chosen_file(self):
data = bytearray(self.base.read_bytes())
data[-100] ^= 0xFF
self.base.write_bytes(bytes(data))
with self.assertRaises(steps.InstallError) as e:
steps.Check().unpack(self.ctx)
self.assertIn(e.exception.id, ('check.input_damaged', 'check.unpack_failed'))
if __name__ == '__main__':
unittest.main()

View File

@@ -0,0 +1,111 @@
"""tools/pack_shader_cache.py on a fake shader cache: the pack layout gfx_shader.c reads (sorted index, bounds, data),
loose files folded in and deleted, a second run merging new loose files, a loose file winning over its packed copy,
malformed files skipped and --keep. No game files; stdlib only.
Run: python tests/pack_shader_cache_test.py
"""
import hashlib
import struct
import sys
import tempfile
import unittest
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT / 'tools'))
import pack_shader_cache as psc # noqa: E402
USE = struct.pack('<5I', 0x4d554b54, 1, 3, 0, 1) # gfx_shader.c use_store
def name(seed):
return hashlib.sha256(seed.encode()).hexdigest()
def read_pack(path):
"""{(kind, name hex): bytes}, checking what gfx_shader.c pack_open checks."""
b = path.read_bytes()
assert b[:8] == psc.MAGIC
count = struct.unpack_from('<I', b, 8)[0]
out, last = {}, None
for i in range(count):
nm, kind, size, offset = struct.unpack_from('<32sIIQ', b, 16 + 48 * i)
assert offset + size <= len(b) and offset >= 16 + 48 * count
assert last is None or (last[0], last[1]) < (kind, nm), 'index not sorted by (kind, name)'
last = (kind, nm)
out[(kind, nm.hex())] = b[offset:offset + size]
return out
class PackTest(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.cache = Path(self.tmp.name) / 'shader-cache'
(self.cache / 'in').mkdir(parents=True)
def tearDown(self):
self.tmp.cleanup()
def shader(self, seed, spv=b'\x03\x02\x23\x07spirv', res=b'{"x":1}'):
k = name(seed)
(self.cache / f'{k}.spv').write_bytes(spv + seed.encode())
(self.cache / f'{k}.res.json').write_bytes(res)
(self.cache / f'{name("use" + seed)}.use').write_bytes(USE)
(self.cache / 'in').mkdir(exist_ok=True) # in_store makes it again after a pack removed it
(self.cache / 'in' / f'{name("in" + seed)}.in').write_bytes(b'TKSI' + seed.encode())
return k
def test_pack_holds_every_file_and_deletes_the_loose_ones(self):
keys = [self.shader(str(i)) for i in range(50)]
(self.cache / 'notes.txt').write_text('kept')
(self.cache / f'{keys[0]}.code').write_bytes(b'kept too')
n, new, skipped, _ = psc.pack(self.cache)
self.assertEqual((n, new, skipped), (200, 200, 0))
got = read_pack(self.cache / psc.PACK)
self.assertEqual(got[(1, keys[3])], b'\x03\x02\x23\x07spirv3')
self.assertEqual(got[(2, keys[3])], b'{"x":1}')
self.assertEqual(got[(3, name('use3'))], USE)
self.assertEqual(got[(4, name('in3'))], b'TKSI3')
left = sorted(p.name for p in self.cache.iterdir())
self.assertEqual(left, sorted([psc.PACK, 'notes.txt', f'{keys[0]}.code'])) # in/ removed when empty
def test_second_run_merges_new_loose_files_and_loose_wins(self):
a = self.shader('a')
psc.pack(self.cache)
b = self.shader('b')
(self.cache / f'{a}.res.json').write_bytes(b'{"newer":1}')
n, new, _, _ = psc.pack(self.cache)
got = read_pack(self.cache / psc.PACK)
self.assertEqual(n, 8)
self.assertEqual(new, 5)
self.assertEqual(got[(2, a)], b'{"newer":1}')
self.assertEqual(got[(1, b)], b'\x03\x02\x23\x07spirvb')
def test_malformed_files_are_skipped_or_replaced_by_the_packed_copy(self):
a = self.shader('a')
psc.pack(self.cache)
(self.cache / f'{name("usea")}.use').write_bytes(b'short') # bad loose copy: the packed one stays
(self.cache / f'{name("bad")}.use').write_bytes(b'short') # bad and nowhere else: skipped
(self.cache / f'{name("empty")}.spv').write_bytes(b'')
n, _, skipped, _ = psc.pack(self.cache)
got = read_pack(self.cache / psc.PACK)
self.assertEqual(got[(3, name('usea'))], USE)
self.assertNotIn((3, name('bad')), got)
self.assertEqual((n, skipped), (4, 2))
self.assertTrue((self.cache / f'{name("bad")}.use').exists()) # never deleted unless packed
def test_keep_leaves_the_loose_files(self):
k = self.shader('a')
psc.pack(self.cache, keep=True)
self.assertTrue((self.cache / f'{k}.spv').exists())
self.assertEqual(len(read_pack(self.cache / psc.PACK)), 4)
def test_a_broken_pack_is_rebuilt_from_the_loose_files(self):
self.shader('a')
(self.cache / psc.PACK).write_bytes(b'garbage')
n, _, _, _ = psc.pack(self.cache)
self.assertEqual(n, 4)
if __name__ == '__main__':
unittest.main()

590
tests/romfs_prompts_test.py Normal file
View File

@@ -0,0 +1,590 @@
"""tools/make_romfs_patches.py --prompts and tools/button_icons.py on a RomFS made here, with no game files: each prompt
names the label of the Xbox button that the pad layout presses, the shoulder buttons, triggers, Plus, Minus and the
d-pad are drawn as on an Xbox controller, the PC prompts draw every button as its key or mouse button (the defaults or
a nx2navi.ini's bind_ lines), the resource size table grows with the fonts, every other file and member stays byte for
byte, the same input gives the same output, and a RomFS the tool does not recognise fails before it writes anything. Both game versions: 1.2.1 (Common.Product.110, 72x72 button guide images) and 1.4.3
(Common.Product.140, 108x108).
Run: .venv-py314\\Scripts\\python tests\\romfs_prompts_test.py
"""
import compression.zstd as zstd
import hashlib
import importlib.util
import io
import json
import random
import struct
import subprocess
import sys
import tempfile
import unittest
import zlib
from pathlib import Path
from fontTools.fontBuilder import FontBuilder
from fontTools.pens.recordingPen import RecordingPen
from fontTools.pens.t2CharStringPen import T2CharStringPen
from fontTools.ttLib import TTFont
ROOT = Path(__file__).resolve().parent.parent
TOOL = ROOT / 'tools' / 'make_romfs_patches.py'
spec = importlib.util.spec_from_file_location('button_icons', ROOT / 'tools' / 'button_icons.py')
icons = importlib.util.module_from_spec(spec)
spec.loader.exec_module(icons)
# The face button each Switch button is in a pad layout (docs/controls.md, runtime/input_map.c), the label an Xbox pad
# prints on it, and the layout each overlay is made for (runtime/input_map.c input_prompts_folder).
POSITION = {'nintendo': {'A': 'east', 'B': 'south', 'X': 'north', 'Y': 'west'},
'xbox': {'A': 'south', 'B': 'east', 'X': 'west', 'Y': 'north'}}
XBOX_LABEL = {'south': 'A', 'east': 'B', 'west': 'X', 'north': 'Y'}
STYLE_LAYOUT = {'xbox': 'nintendo', 'xbox-label': 'xbox'}
FACE = {'A': 0xE040, 'B': 0xE041, 'X': 0xE042, 'Y': 0xE043}
# The glyphs drawn as Xbox icons, with their advance: LB RB LT RT, Menu and View, the d-pad.
DRAWN = {0xE044: 820, 0xE045: 820, 0xE046: 760, 0xE047: 760, 0xE04B: 640, 0xE04C: 640, 0xE049: 640,
**{cp: 640 for cp in range(0xE060, 0xE066)}}
# restbl: the resource size table's name after its version, in both forms the tool takes.
VERSIONS = {'v121': dict(common='110', size=72, newest=121, restbl='Nin_NX_NVN.rsizetable.zs'),
'v143': dict(common='140', size=108, newest=143, restbl='rsizetable.zs')}
DEFAULT_FONT, REGIONAL_FONT = 'Font/Font.Nin_NX_NVN.bfarc.zs', 'Font/Font_KRko.Nin_NX_NVN.bfarc.zs'
PANES = {'T_A_00': 'A', 'T_B_00': 'B', 'T_X_00': 'X', 'T_Y_00': 'Y', 'T_Name_00': 'A'}
MESSAGES = {**PANES, 'Greeting': 'Hello'}
R8_UNORM, BC1 = 0x201, 0x1a01
IMAGE = 16384 # an R8 square up to 128 rows, two GOBs wide, block height 16: the size of the game's key images
# ---- the formats, written here from their layout so the tools' readers are checked against an independent writer
def sarc(members, align=0x80):
names, sfnt, name_at = list(members), b'', {}
for name in names:
name_at[name] = len(sfnt) // 4
sfnt += name.encode() + b'\0'
sfnt += bytes(-len(sfnt) % 4)
head = 0x14 + 12 + 16 * len(names) + 8 + len(sfnt)
start = head + -head % align
data, nodes = b'', b''
for name in names:
data += bytes(-len(data) % align)
nodes += struct.pack('<4I', zlib.crc32(name.encode()), 0x01000000 | name_at[name], len(data), len(data) + len(members[name]))
data += members[name]
out = (b'SARC' + struct.pack('<HHIIHH', 0x14, 0xFEFF, start + len(data), start, 0x100, 0)
+ b'SFAT' + struct.pack('<HHI', 12, len(names), 0x65) + nodes + b'SFNT' + struct.pack('<HH', 8, 0) + sfnt)
return out + bytes(start - len(out)) + data
def sarc_files(data):
header, start = struct.unpack_from('<H', data, 4)[0], struct.unpack_from('<I', data, 12)[0]
count, out = struct.unpack_from('<H', data, header + 6)[0], {}
names = header + 12 + count * 16 + 8
for i in range(count):
_, attribute, lo, hi = struct.unpack_from('<4I', data, header + 12 + i * 16)
at = names + (attribute & 0xffffff) * 4
out[data[at:data.index(0, at)].decode()] = data[start + lo:start + hi]
return out
def bflan(panes):
"""A FLAN whose pai1 section steps each pane's visibility from 0 at frame 0 to 1 at frame 1."""
entries = [name.encode().ljust(28, b'\0') + bytes([1, 0, 0, 0]) + struct.pack('<I', 36)
+ b'FLVI' + bytes([1, 0, 0, 0]) + struct.pack('<I', 12)
+ bytes([0, 0, 1, 0]) + struct.pack('<HHI', 2, 0, 12)
+ struct.pack('<fHxx', 0.0, 0) + struct.pack('<fHxx', 1.0, 1) for name in panes]
table, body = 20, b''
offsets = []
for e in entries:
offsets.append(table + 4 * len(entries) + len(body))
body += e
section = struct.pack('<HBxHHI', 0, 0, 0, len(entries), table) + struct.pack(f'<{len(entries)}I', *offsets) + body
section = b'pai1' + struct.pack('<I', 8 + len(section)) + section
return b'FLAN' + struct.pack('<HHIIHH', 0xFEFF, 0x14, 0x08000000, 0x14 + len(section), 1, 0) + section
def visibility(data):
"""pane -> [(frame, value)] of a FLAN made by bflan()."""
pos, out = 0x14, {}
count, table = struct.unpack_from('<H', data, pos + 14)[0], pos + struct.unpack_from('<I', data, pos + 16)[0]
for e in range(count):
eo = pos + struct.unpack_from('<I', data, table + 4 * e)[0]
keys = eo + 36 + 12 + 12
out[data[eo:eo + 28].split(b'\0')[0].decode()] = [struct.unpack_from('<fH', data, keys + 8 * k) for k in range(2)]
return out
def bntx(textures):
"""A BNTX of single-mip square textures: name -> (format, size, image)."""
out = bytearray(0x30)
out[0:4], out[0x20:0x24] = b'BNTX', b'NX '
table = len(out)
out += bytes(8 * len(textures))
struct.pack_into('<IQ', out, 0x24, len(textures), table)
for k, (name, (fmt, size, image)) in enumerate(textures.items()):
info = len(out)
out += bytes(0x80)
out[info:info + 4] = b'BRTI'
struct.pack_into('<Q', out, table + 8 * k, info)
struct.pack_into('<H', out, info + 0x16, 1)
struct.pack_into('<I', out, info + 0x1c, fmt)
struct.pack_into('<II', out, info + 0x24, size, size)
struct.pack_into('<I', out, info + 0x34, 4) # block height 1 << 4 GOBs
struct.pack_into('<I', out, info + 0x50, len(image))
struct.pack_into('<Q', out, info + 0x60, len(out))
out += struct.pack('<H', len(name)) + name.encode() + b'\0'
out += bytes(-len(out) % 8)
mips = len(out)
struct.pack_into('<Q', out, info + 0x70, mips)
out += bytes(8)
out += bytes(-len(out) % 0x200)
struct.pack_into('<Q', out, mips, len(out))
out += image
return bytes(out)
def bntx_images(data):
"""name without the ^ suffix -> (format, size, image) of a BNTX made by bntx()."""
count, table = struct.unpack_from('<IQ', data, 0x24)
out = {}
for k in range(count):
info = struct.unpack_from('<Q', data, table + 8 * k)[0]
at = struct.unpack_from('<Q', data, info + 0x60)[0]
name = data[at + 2:at + 2 + struct.unpack_from('<H', data, at)[0]].decode().split('^')[0]
start = struct.unpack_from('<Q', data, struct.unpack_from('<Q', data, info + 0x70)[0])[0]
out[name] = (struct.unpack_from('<I', data, info + 0x1c)[0], struct.unpack_from('<I', data, info + 0x24)[0],
data[start:start + struct.unpack_from('<I', data, info + 0x50)[0]])
return out
def bflyt(panes):
"""A FLYT with one txt1 section per pane (name -> text) after a pan1 section."""
sections = b'pan1' + struct.pack('<I', 0x4c) + bytes(0x44)
for name, text in panes.items():
s = bytearray(0x80)
s[0:4] = b'txt1'
struct.pack_into('<I', s, 4, len(s))
s[12:12 + len(name)] = name.encode()
struct.pack_into('<H', s, 0x56, 2 * len(text) + 2)
struct.pack_into('<I', s, 0x64, 0x70)
s[0x70:0x70 + 2 * len(text) + 2] = text.encode('utf-16-le') + b'\0\0'
sections += s
return b'FLYT' + struct.pack('<HHIIHH', 0xFEFF, 0x14, 0x08000000, 0x14 + len(sections), len(panes) + 1, 0) + sections
def pane_texts(data):
pos, out = struct.unpack_from('<H', data, 6)[0], {}
while pos + 8 <= len(data):
magic, size = data[pos:pos + 4], struct.unpack_from('<I', data, pos + 4)[0]
if magic == b'txt1':
length, at = struct.unpack_from('<H', data, pos + 0x56)[0], pos + struct.unpack_from('<I', data, pos + 0x64)[0]
out[data[pos + 12:pos + 36].split(b'\0')[0].decode()] = data[at:at + length - 2].decode('utf-16-le')
pos += size
return out
def msbt(messages):
"""A UTF-16 MSBT: one LBL1 group with every label, then TXT2 (label -> text)."""
labels = b''.join(bytes([len(name)]) + name.encode() + struct.pack('<I', i) for i, name in enumerate(messages))
lbl1 = struct.pack('<III', 1, len(messages), 12) + labels
texts = [text.encode('utf-16-le') + b'\0\0' for text in messages.values()]
offsets, body = [], b''
for text in texts:
offsets.append(4 + 4 * len(texts) + len(body))
body += text
txt2 = struct.pack(f'<I{len(texts)}I', len(texts), *offsets) + body
def section(magic, content):
s = magic + struct.pack('<I', len(content)) + bytes(8) + content
return s + bytes(-len(s) % 16)
sections = section(b'LBL1', lbl1) + section(b'TXT2', txt2)
return b'MsgStdBn' + struct.pack('<HHBBHHI', 0xFEFF, 0, 1, 3, 2, 0, 0x20 + len(sections)) + bytes(10) + sections
def message_texts(data):
sections, pos = {}, 0x20
for _ in range(struct.unpack_from('<H', data, 0x0e)[0]):
size = struct.unpack_from('<I', data, pos + 4)[0]
sections[data[pos:pos + 4]] = (pos + 16, size)
pos = (pos + 16 + size + 15) & ~15
(base, _), labels = sections[b'LBL1'], {}
for g in range(struct.unpack_from('<I', data, base)[0]):
count, at = struct.unpack_from('<II', data, base + 4 + 8 * g)
at += base
for _ in range(count):
n = data[at]
labels[struct.unpack_from('<I', data, at + 1 + n)[0]] = data[at + 1:at + 1 + n].decode()
at += 1 + n + 4
base, size = sections[b'TXT2']
count = struct.unpack_from('<I', data, base)[0]
offsets = list(struct.unpack_from(f'<{count}I', data, base + 4)) + [size]
return {labels[i]: data[base + offsets[i]:base + offsets[i + 1]].decode('utf-16-le').rstrip('\0') for i in range(count)}
def restbl(entries):
"""A RESTBL keyed by the CRC32 of each path without .zs (path -> size)."""
crcs = sorted((zlib.crc32(path.removesuffix('.zs').encode()), size) for path, size in entries.items())
return b'RESTBL' + struct.pack('<IIII', 1, 160, len(crcs), 0) + b''.join(struct.pack('<II', *c) for c in crcs)
def restbl_sizes(table):
count = struct.unpack_from('<I', table, 14)[0]
return dict(struct.unpack_from('<II', table, 22 + 8 * i) for i in range(count))
MAGIC = 0x18029a7f # MacNativeGames make_button_icons.py: a .bfotf's first word is its key XOR this
def bfotf(otf, key):
"""A .bfotf as MacNativeGames make_button_icons.py bfotf_encode writes it: 32-bit words XORed with the key."""
size = len(otf)
out = struct.pack('<II', MAGIC ^ key, struct.unpack('<I', size.to_bytes(4, 'big'))[0] ^ key)
out += b''.join(struct.pack('<I', struct.unpack_from('<I', otf, i)[0] ^ key) for i in range(0, size - size % 4, 4))
return out + otf[size - size % 4:]
def unbfotf(blob):
key = struct.unpack_from('<I', blob, 0)[0] ^ MAGIC
size = int.from_bytes(struct.pack('<I', struct.unpack_from('<I', blob, 4)[0] ^ key), 'big')
words = b''.join(struct.pack('<I', struct.unpack_from('<I', blob, 8 + i)[0] ^ key) for i in range(0, size - size % 4, 4))
return words + blob[8 + size - size % 4:8 + size]
def rect(pen, x0, y0, x1, y1, hole=False):
"""A rectangle, counter-clockwise (filled, the CFF way) or clockwise (a counter)."""
points = [(x0, y0), (x1, y0), (x1, y1), (x0, y1)]
points = points[::-1] if hole else points
pen.moveTo(points[0])
for p in points[1:]:
pen.lineTo(p)
pen.closePath()
def polygon(*points):
def draw(pen):
pen.moveTo(points[0])
for p in points[1:]:
pen.lineTo(p)
pen.closePath()
return draw
def cff_font(name, glyphs):
"""A CFF OpenType font with a 1000-unit em: code point -> (advance, draw(pen))."""
names = {cp: f'uni{cp:04X}' for cp in glyphs}
fb = FontBuilder(1000, isTTF=False)
fb.setupGlyphOrder(['.notdef'] + list(names.values()))
fb.setupCharacterMap(names)
charstrings, metrics = {}, {}
for gname, (advance, draw) in [('.notdef', (500, lambda pen: rect(pen, 50, 0, 450, 700)))] + \
[(names[cp], glyphs[cp]) for cp in glyphs]:
pen = T2CharStringPen(advance, None)
draw(pen)
charstrings[gname], metrics[gname] = pen.getCharString(), (advance, 0)
fb.setupCFF(name, {'FullName': name}, charstrings, {})
fb.setupHorizontalMetrics(metrics)
fb.setupHorizontalHeader(ascent=800, descent=-200)
fb.setupNameTable({'familyName': name, 'styleName': 'Regular'})
fb.setupOS2(sTypoAscender=800, usWinAscent=800, usWinDescent=200)
fb.setupPost()
out = io.BytesIO()
fb.save(out)
return out.getvalue()
# A distinct rectangle and advance for every button glyph, so a glyph that moved is found by its outline.
BUTTONS = {cp: (600 + 4 * (cp - 0xE040), lambda pen, i=cp - 0xE040: rect(pen, 20 + 5 * i, 20, 300 + 5 * i, 400 + 3 * i))
for cp in range(0xE040, 0xE069)}
LETTERS_1 = {ord('L'): (560, polygon((60, 0), (480, 0), (480, 100), (160, 100), (160, 700), (60, 700))),
ord('B'): (560, lambda pen: [rect(pen, 60, 0, 460, 700), rect(pen, 160, 100, 360, 300, True),
rect(pen, 160, 400, 360, 600, True)]),
ord('R'): (560, lambda pen: [rect(pen, 60, 0, 460, 700), rect(pen, 160, 400, 360, 600, True)])}
LETTERS_2 = {ord('T'): (560, polygon((230, 0), (330, 0), (330, 600), (520, 600), (520, 700), (40, 700), (40, 600), (230, 600)))}
# The other characters the PC prompts' keys show (button_icons.PC_LABELS of the defaults): a block with a counter, each
# its own size.
LETTERS_2.update({ord(ch): (560, lambda pen, i=i: [rect(pen, 60, 0, 300 + 10 * i, 700), rect(pen, 120, 100, 240, 600, True)])
for i, ch in enumerate('ACDEFMQSW1234⇧G')})
PC_PANES = {'A': 'E', 'B': '↑', 'X': '▬', 'Y': 'L'} # the diamond's letters with the default keys
class Romfs:
"""A RomFS (romfs.bin, index.json) holding the files make_romfs_patches.py reads, the originals kept to compare."""
def __init__(self, folder, version, text_fonts=True, restbl_skips=(), l_format=R8_UNORM):
v, rnd = VERSIONS[version], random.Random(version)
words = [b'Layout', b'Pane', b'Font', b'Msg', b'Texture', b'Anim', b'Group', b'Text', b'Base', b'Key']
samples = [b' '.join(rnd.choice(words) + str(rnd.randrange(999)).encode() for _ in range(rnd.randrange(20, 80)))
for _ in range(400)]
self.dictionary = zstd.train_dict(samples, 4096)
self.common = f'UI/LayoutArchive/Common.Product.{v["common"]}.Nin_NX_NVN.blarc.zs'
self.mals = [f'Mals/EUen.Product.{v["newest"]}.sarc.zs', f'Mals/USen.Product.{v["newest"]}.sarc.zs']
self.old_mals = 'Mals/USen.Product.110.sarc.zs'
self.restbl = f'System/Resource/ResourceSizeTable.Product.{v["newest"]}.{v["restbl"]}'
self.old_restbl = f'System/Resource/ResourceSizeTable.Product.100.{v["restbl"]}' # an older version's: left alone
size = v['size']
images = {f'Nt_KeyTex{b}_00': (R8_UNORM, size, rnd.randbytes(IMAGE)) for b in 'ABXY'}
images.update({'Nt_KeyTexL_00^z': (l_format, size, rnd.randbytes(IMAGE)),
'Nt_KeyTexZL_00': (R8_UNORM, size, rnd.randbytes(IMAGE)),
'Nt_KeyTexLSt_00': (R8_UNORM, size, rnd.randbytes(IMAGE)),
'Nt_KeyTexLSt_01': (R8_UNORM, size, rnd.randbytes(IMAGE))})
self.text_fonts = [bfotf(cff_font('Text1', LETTERS_1), 0x01020304), bfotf(cff_font('Text2', LETTERS_2), 0x0a0b0c0d)]
default = {'scft/External.bfotf': bfotf(cff_font('External', BUTTONS), 0x5a17c3e9), 'fnt/rest.bin': rnd.randbytes(700)}
if text_fonts:
default.update(zip(icons.TEXT_FONTS, self.text_fonts))
mals = sarc({'LayoutMsg/DoCommand_00.msbt': msbt(MESSAGES), 'LayoutMsg/Other_00.msbt': msbt({'T_A_00': 'A'})})
self.contents = { # RomFS path -> contents before compression
self.common: sarc({'anim/ShortCutBG_00_In.bflan': bflan(('C_CaptureBG_00', 'C_CaptureClearBG_00',
'P_CaptureClearBG_00', 'N_Other_00')),
'anim/AutoBuilder_00_In.bflan': bflan(('C_CaptureBG_00',)),
'blyt/DoCommand_00.bflyt': bflyt(PANES),
'blyt/Title_00.bflyt': bflyt({'N_Title_00': 'Z'}),
'timg/__Combined.bntx': bntx(images)}),
DEFAULT_FONT: sarc(default),
# The same font under another key: drawn once, stored under each archive's own key.
REGIONAL_FONT: sarc({'scft/External.bfotf': bfotf(cff_font('External', BUTTONS), 0x77777777),
'fnt/rest.bin': rnd.randbytes(500)}),
**{path: mals for path in self.mals + [self.old_mals]},
}
sizes = {path: len(data) + 256 for path, data in self.contents.items() if path not in restbl_skips}
self.contents[self.restbl] = restbl({**sizes, **{f'Other/{i}.bin': 1000 + i for i in range(20)}})
self.contents[self.old_restbl] = restbl({DEFAULT_FONT: 1})
plain = (REGIONAL_FONT, self.restbl, self.old_restbl) # stored without a dictionary
self.files = {'Pack/ZsDic.pack.zs': zstd.compress(sarc({'zs.zsdic': self.dictionary.dict_content}))}
self.files.update({path: zstd.compress(data, zstd_dict=None if path in plain else self.dictionary)
for path, data in self.contents.items()})
self.files['Other/file.bin'] = rnd.randbytes(100)
image, entries = bytearray(), []
for path, data in self.files.items():
image += bytes(-len(image) % 16)
entries.append(dict(path=path, offset=len(image), size=len(data)))
image += data
folder.mkdir(parents=True)
(folder / 'romfs.bin').write_bytes(image)
(folder / 'index.json').write_text(json.dumps(dict(status='pass', files=entries)))
self.folder, self.version = folder, version
def run(self, out, *flags):
return subprocess.run([sys.executable, '-I', str(TOOL), '--version', self.version, '--romfs', str(self.folder),
'--out', str(out), *flags], capture_output=True, text=True, encoding='utf-8')
def read(self, out, path):
"""The overlay's file, decompressed."""
raw = (out / path).read_bytes()
return zstd.decompress(raw, zstd_dict=self.dictionary if zstd.get_frame_info(raw).dictionary_id else None)
def font(blob):
return TTFont(io.BytesIO(unbfotf(blob)))
def glyph(f, cp):
"""(outline, advance) of the glyph of code point cp."""
name = f.getBestCmap()[cp]
pen = RecordingPen()
f.getGlyphSet()[name].draw(pen)
return pen.value, f['hmtx'][name][0]
def manifest(out):
return {line.split('\t')[0]: line.split('\t')[1:] for line in (out / 'manifest.tsv').read_text().splitlines()
if not line.startswith('#')}
class PromptsTest(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.tmp = tempfile.TemporaryDirectory(prefix='nx2navi-prompts-')
cls.base = Path(cls.tmp.name)
cls.romfs = {v: Romfs(cls.base / v / 'romfs', v) for v in VERSIONS}
cls.outs = {}
for v, romfs in cls.romfs.items():
for style in ('plain', 'xbox', 'xbox-label', 'pc'):
out = cls.base / v / style
proc = romfs.run(out, *(() if style == 'plain' else ('--prompts', style)))
assert proc.returncode == 0, proc.stderr
cls.outs[v, style] = (out, json.loads(proc.stdout.strip().splitlines()[-1]))
@classmethod
def tearDownClass(cls):
cls.tmp.cleanup()
def each(self, styles=('xbox', 'xbox-label')):
"""(RomFS, style, overlay folder, summary) of every run of these styles."""
return [(self.romfs[v], style, out, summary) for (v, style), (out, summary) in self.outs.items()
if style in styles]
def test_plain_overlay_has_only_the_frame_rate_patch(self):
for romfs, _, out, summary in self.each(('plain',)):
with self.subTest(version=romfs.version, style="plain"):
self.assertEqual(set(manifest(out)), {romfs.common})
self.assertIsNone(summary['prompts'])
keys = visibility(sarc_files(romfs.read(out, romfs.common))['anim/ShortCutBG_00_In.bflan'])
self.assertEqual(keys['C_CaptureBG_00'], [(0.0, 0), (0.0, 1)])
self.assertEqual(keys['N_Other_00'], [(0.0, 0), (1.0, 1)])
def test_face_buttons_name_the_xbox_label_of_the_button_pressed(self):
for romfs, style, out, _ in self.each():
with self.subTest(version=romfs.version, style=style):
layout = STYLE_LAYOUT[style]
label = {b: XBOX_LABEL[POSITION[layout][b]] for b in FACE} # Switch button -> the label of its button
before, after = sarc_files(romfs.contents[romfs.common]), sarc_files(romfs.read(out, romfs.common))
old, new = bntx_images(before['timg/__Combined.bntx']), bntx_images(after['timg/__Combined.bntx'])
panes, messages = pane_texts(after['blyt/DoCommand_00.bflyt']), {}
for path in romfs.mals if layout == 'nintendo' else []:
messages[path] = message_texts(sarc_files(romfs.read(out, path))['LayoutMsg/DoCommand_00.msbt'])
original = font(sarc_files(romfs.contents[DEFAULT_FONT])['scft/External.bfotf'])
for path in (DEFAULT_FONT, REGIONAL_FONT):
drawn = font(sarc_files(romfs.read(out, path))['scft/External.bfotf'])
for b, cp in FACE.items():
self.assertEqual(glyph(drawn, cp), glyph(original, FACE[label[b]]), f'{path}: glyph of {b}')
for b in FACE:
self.assertEqual(new[f'Nt_KeyTex{b}_00'], old[f'Nt_KeyTex{label[b]}_00'], f'action prompt image of {b}')
self.assertEqual(panes[f'T_{b}_00'], label[b], f'diamond letter of {b}')
for path, texts in messages.items():
self.assertEqual(texts[f'T_{b}_00'], label[b], f'{path}: diamond message of {b}')
# Text that only looks like a letter pane, and every other message, stays.
self.assertEqual((panes['T_Name_00'], pane_texts(after['blyt/Title_00.bflyt'])), ('A', {'N_Title_00': 'Z'}))
for path, texts in messages.items():
self.assertEqual((texts['T_Name_00'], texts['Greeting']), ('A', 'Hello'))
self.assertEqual(sarc_files(romfs.read(out, path))['LayoutMsg/Other_00.msbt'], msbt({'T_A_00': 'A'}))
def test_other_buttons_are_drawn_as_xbox_icons(self):
for romfs, style, out, _ in self.each():
with self.subTest(version=romfs.version, style=style):
original = font(sarc_files(romfs.contents[DEFAULT_FONT])['scft/External.bfotf'])
default = sarc_files(romfs.read(out, DEFAULT_FONT))
drawn = font(default['scft/External.bfotf'])
for cp in range(0xE040, 0xE069):
outline, advance = glyph(drawn, cp)
if cp in DRAWN:
self.assertNotEqual(outline, glyph(original, cp)[0], f'U+{cp:04X} drawn')
self.assertEqual(advance, DRAWN[cp], f'U+{cp:04X} advance')
elif cp not in FACE.values():
self.assertEqual((outline, advance), glyph(original, cp), f'U+{cp:04X} kept')
# The letters are holes in the key: LB is the key, L, and B with its two counters.
moves = sum(op == 'moveTo' for op, _ in glyph(drawn, 0xE044)[0])
self.assertEqual(moves, 5)
self.assertEqual(sum(op == 'moveTo' for op, _ in glyph(drawn, 0xE047)[0]), 1 + 1 + 1 + 1) # RT: key, R + counter, T
self.assertEqual(sum(op == 'moveTo' for op, _ in glyph(drawn, 0xE060)[0]), 2) # d-pad up: cross, arrow
# The text fonts and the archives' other members stay byte for byte; both archives hold the same font.
before = sarc_files(romfs.contents[DEFAULT_FONT])
self.assertEqual({k: v for k, v in default.items() if k != 'scft/External.bfotf'},
{k: v for k, v in before.items() if k != 'scft/External.bfotf'})
regional = sarc_files(romfs.read(out, REGIONAL_FONT))
self.assertEqual(regional['fnt/rest.bin'], sarc_files(romfs.contents[REGIONAL_FONT])['fnt/rest.bin'])
self.assertEqual(unbfotf(regional['scft/External.bfotf']), unbfotf(default['scft/External.bfotf']))
self.assertEqual(struct.unpack_from('<I', regional['scft/External.bfotf'])[0] ^ MAGIC, 0x77777777)
def test_button_guide_images_show_the_same_icons(self):
for romfs, style, out, _ in self.each():
with self.subTest(version=romfs.version, style=style):
size = VERSIONS[romfs.version]['size']
before = sarc_files(romfs.contents[romfs.common])
after = sarc_files(romfs.read(out, romfs.common))
self.assertEqual(len(after['timg/__Combined.bntx']), len(before['timg/__Combined.bntx']))
old, new = bntx_images(before['timg/__Combined.bntx']), bntx_images(after['timg/__Combined.bntx'])
glyphs = icons.xbox_icons(icons.text_glyphs(romfs.text_fonts))
for name, cp in icons.HUD_TEXTURES.items():
fmt, width, image = new[name]
pixels = bytes(image[icons.gob_offset(x, y, (size + 63) // 64, 16)] for y in range(size) for x in range(size))
self.assertEqual(pixels, icons.rasterize(glyphs[cp][1], size), name)
self.assertEqual({pixels[0], pixels[-1], max(pixels)}, {0, 255}, name) # a margin around a white key
self.assertEqual(new['Nt_KeyTexLSt_00'], old['Nt_KeyTexLSt_00'])
for member in before: # the 60 fps patch and the prompts touch only these
if member not in ('timg/__Combined.bntx', 'blyt/DoCommand_00.bflyt') and not member.startswith('anim/'):
self.assertEqual(after[member], before[member], member)
def test_resource_size_table_grows_with_the_fonts(self):
for romfs, style, out, summary in self.each(('xbox', 'xbox-label', 'pc')):
with self.subTest(version=romfs.version, style=style):
table = restbl_sizes(romfs.read(out, romfs.restbl))
original = restbl_sizes(romfs.contents[romfs.restbl])
grown = {}
for path in manifest(out):
if path != romfs.restbl:
grown[path] = len(romfs.read(out, path)) - len(romfs.contents[path])
self.assertTrue(grown[DEFAULT_FONT] > 0 and grown[REGIONAL_FONT] > 0, grown)
self.assertEqual(summary['prompts']['grown'], {p: g for p, g in grown.items() if g > 0})
for crc, entry in original.items():
paths = [p for p in grown if zlib.crc32(p.removesuffix('.zs').encode()) == crc and grown[p] > 0]
self.assertEqual(table[crc], entry + (grown[paths[0]] if paths else 0))
self.assertEqual(set(table), set(original))
def test_manifest_lists_what_changed_with_both_hashes(self):
for romfs, style, out, _ in self.each(('plain', 'xbox', 'xbox-label', 'pc')):
with self.subTest(version=romfs.version, style=style):
expect = {romfs.common}
if style != 'plain':
expect |= {DEFAULT_FONT, REGIONAL_FONT, romfs.restbl}
expect |= set(romfs.mals) if style in ('xbox', 'pc') else set()
lines = manifest(out)
self.assertEqual(set(lines), expect)
for path, (size, sha, original) in lines.items():
data = (out / path).read_bytes()
self.assertEqual((int(size), sha), (len(data), hashlib.sha256(data).hexdigest()), path)
self.assertEqual(original, hashlib.sha256(romfs.files[path]).hexdigest(), path)
def test_pc_prompts_draw_every_button_as_its_key(self):
for romfs, _, out, _ in self.each(('pc',)):
with self.subTest(version=romfs.version):
original = font(sarc_files(romfs.contents[DEFAULT_FONT])['scft/External.bfotf'])
drawn = font(sarc_files(romfs.read(out, DEFAULT_FONT))['scft/External.bfotf'])
advances = icons.font_advances(sarc_files(romfs.contents[DEFAULT_FONT])['scft/External.bfotf'])
pc = icons.pc_icons(icons.text_glyphs(romfs.text_fonts), advances)
self.assertIn(0xE05E, pc) # the left stick's W A S D
for cp in range(0xE040, 0xE069):
outline, advance = glyph(drawn, cp)
if cp in pc:
self.assertNotEqual(outline, glyph(original, cp)[0], f'U+{cp:04X} drawn')
else:
self.assertEqual(outline, glyph(original, cp)[0], f'U+{cp:04X} kept')
self.assertEqual(advance, glyph(original, cp)[1], f'U+{cp:04X} keeps its advance')
# The key's label is a hole: E is the key, the block and its counter.
self.assertEqual(sum(op == 'moveTo' for op, _ in glyph(drawn, 0xE040)[0]), 3)
after = sarc_files(romfs.read(out, romfs.common))
new = bntx_images(after['timg/__Combined.bntx'])
size = VERSIONS[romfs.version]['size']
for name, cp in icons.PC_HUD_TEXTURES.items():
image = new[name][2]
pixels = bytes(image[icons.gob_offset(x, y, (size + 63) // 64, 16)] for y in range(size) for x in range(size))
self.assertEqual(pixels, icons.rasterize(pc[cp][1], size), name)
panes = pane_texts(after['blyt/DoCommand_00.bflyt'])
for b, letter in PC_PANES.items():
self.assertEqual(panes[f'T_{b}_00'], letter, f'diamond letter of {b}')
for path in romfs.mals:
texts = message_texts(sarc_files(romfs.read(out, path))['LayoutMsg/DoCommand_00.msbt'])
self.assertEqual((texts[f'T_{b}_00'], texts['T_Name_00']), (letter, 'A'), path)
def test_pc_prompts_follow_the_bind_lines(self):
romfs = self.romfs['v143']
ini = self.base / 'binds' / 'nx2navi.ini'
ini.parent.mkdir(parents=True)
ini.write_text('[input]\n# bind_A = key:q\nbind_A = pad:east, key:F\nbind_B = pad:south\nbind_X = key:default\n',
encoding='utf-8')
out = self.base / 'binds' / 'out'
proc = romfs.run(out, '--prompts', 'pc', '--binds', str(ini))
self.assertEqual(proc.returncode, 0, proc.stderr)
panes = pane_texts(sarc_files(romfs.read(out, romfs.common))['blyt/DoCommand_00.bflyt'])
self.assertEqual({b: panes[f'T_{b}_00'] for b in PC_PANES}, {**PC_PANES, 'A': 'F'})
self.assertEqual(icons.pc_binds(ini.read_text(encoding='utf-8')), {**icons.PC_BINDS, 'A': 'f'})
def test_same_input_same_output_and_the_older_flag(self):
romfs = self.romfs['v121']
for flags in (('--prompts', 'xbox'), ('--xbox-prompts',)):
out = self.base / 'again' / '-'.join(flags)
proc = romfs.run(out, *flags)
self.assertEqual(proc.returncode, 0, proc.stderr)
self.assertEqual((out / 'manifest.tsv').read_bytes(), (self.outs['v121', 'xbox'][0] / 'manifest.tsv').read_bytes())
def test_unrecognised_romfs_fails_before_writing(self):
cases = {'no text fonts': (dict(text_fonts=False), icons.TEXT_FONTS[0]),
'no size entry': (dict(restbl_skips=(REGIONAL_FONT,)), 'has no entry in the resource size table'),
'compressed L image': (dict(l_format=BC1), 'Nt_KeyTexL_00: format')}
for i, (case, (options, message)) in enumerate(cases.items()):
with self.subTest(case):
romfs = Romfs(self.base / 'bad' / str(i) / 'romfs', 'v121', **options)
out = self.base / 'bad' / str(i) / 'out'
proc = romfs.run(out, '--prompts', 'xbox-label')
self.assertNotEqual(proc.returncode, 0)
self.assertIn(message, proc.stderr)
self.assertFalse(out.exists(), 'a failed run writes nothing')
if __name__ == '__main__':
unittest.main()

80
tests/spin_waits_test.py Normal file
View File

@@ -0,0 +1,80 @@
# Adapted from nxnative dm/tools/tests/test_spin_waits.py @ 6d34984 (MIT, Copyright (c) 2026 seedds).
"""tools/spin_waits.py on hand-assembled AArch64 loops, without game files.
Run: .venv/Scripts/python tests/spin_waits_test.py
"""
import struct
import sys
import unittest
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tools'))
from spin_waits import Search # noqa: E402
# A job queue poll (the shape of Breath of the Wild's main+0xef6354): take the queue's spin lock, look whether the wait
# is over, release the lock, again.
# 0x00 T: cbz x20, X 0x2c str w12, [x20, #0x100] 0x50 ldr w13, [x21, #0x90]
# 0x04 str w11, [x20, #0x100] 0x30 ldrb w14, [x21, #0x98] 0x54 sub w8, w13, w25
# 0x08 L0: ldr w14, [x9] 0x34 cmp w0, #0 0x58 add w15, w8, w25
# 0x0c cmp w14, #1 0x38 ccmp w14, #0, #4, ne 0x5c str w15, [x21, #0x94]
# 0x10 b.eq L0 0x3c ldr w25, [x21, #0x94] 0x60 cset w13, hs
# 0x14 L1: ldaxr w14, [x9] 0x40 ccmp w25, #0, #0, ne 0x64 b U
# 0x18 cbz w14, L2 0x44 cset w14, gt 0x68 C: clrex
# 0x1c clrex 0x48 cmp w14, #1 0x6c U: ldxr w15, [x9] / cmp w15, #1 / b.ne C
# 0x20 b L1 0x4c b.ne U 0x78 stlxr w15, wzr, [x9] / cbnz w15, U
# 0x24 L2: stxr w14, w10, [x9] / cbnz w14, L1 0x80 cbz w14, T 0x84 X: ret
# 0x50-0x64: the wait is over, take an item, store the new cursor, release and leave.
POLL = [0xb4000434, 0xb901028b, 0xb940012e, 0x710005df, 0x54ffffc0, 0x885ffd2e, 0x3400006e, 0xd5033f5f, 0x17fffffd,
0x880e7d2a, 0x35ffff6e, 0xb901028c, 0x394262ae, 0x7100001f, 0x7a4019c4, 0xb94096b9, 0x7a401b20, 0x1a9fd7ee,
0x710005df, 0x54000101, 0xb94092ad, 0x4b1901a8, 0x0b19010f, 0xb90096af, 0x1a9f37ed, 0x14000002, 0xd5033f5f,
0x885f7d2f, 0x710005ff, 0x54ffffa1, 0x880ffd3f, 0x35ffff8f, 0x34fffc0e, 0xd65f03c0]
def region(words, T, bes):
img = struct.pack('<' + 'I' * len(words), *words) + bytes(64)
return Search(img, 4 * len(words), set()).region(T, bes)
def patched(**at):
w = list(POLL)
for off, word in at.items():
w[int(off[1:], 16) // 4] = word
return w
class SpinWaits(unittest.TestCase):
def test_lock_poll_is_a_wait(self):
g = region(POLL, 0, [0x80])
self.assertIsNotNone(g)
self.assertEqual(g['valid'], {0x80})
self.assertNotIn(0x5c, g['R']) # the path that takes an item stores a new cursor: not part of the wait
self.assertTrue(set(g['kinds']) >= {'acq', 'rel', 'tst', 'stc'})
def test_found_by_the_scan(self):
self.assertIn(0, {g['T'] for g in Search(struct.pack('<' + 'I' * len(POLL), *POLL), 4 * len(POLL), set()).all()})
def test_patch_site_inside(self): # a site of the patch table in the region keeps it out
img = struct.pack('<' + 'I' * len(POLL), *POLL) + bytes(64)
self.assertIsNone(Search(img, 4 * len(POLL), {0x3c}).region(0, [0x80]))
def test_store_of_a_changing_value(self): # the wait path stores w14, which the pass writes
self.assertIsNone(region(patched(o2c=0xb901028e), 0, [0x80]))
def test_lock_not_released(self): # the wait path skips the release (b.ne U -> b.ne to the back-edge)
self.assertIsNone(region(patched(o4c=0x540001a1), 0, [0x80]))
def test_nothing_polled(self): # str xzr, [x8]; b back: stores of a constant and no load
self.assertIsNone(region([0xf900011f, 0x17ffffff], 0, [4]))
def test_plain_test_loop(self): # ldr w8, [x0]; cbz w8, back
g = region([0xb9400008, 0x34ffffe8], 0, [4])
self.assertIsNotNone(g)
self.assertEqual(g['valid'], {4})
def test_counted_loop(self): # ldr w8, [x0]; subs w1, w1, #1; b.ne back: a counter, it leaves by itself
self.assertIsNone(region([0xb9400008, 0x71000421, 0x54ffffc1], 0, [8]))
if __name__ == '__main__':
unittest.main()

130
tests/worktree_test.py Normal file
View File

@@ -0,0 +1,130 @@
"""tools/worktree.py against a scratch repo whose "main checkout" holds dummy ignored data. No game files needed.
Proves that marker files in the linked directories and in the linked file survive:
1. tools/worktree.py remove,
2. git worktree remove --force with the junctions still in place,
3. git worktree prune after the worktree directory is deleted with its junctions in place.
Also checks that git status in a worktree shows no untracked or typechanged entry for the links.
Run with the repo's Python: tests/worktree_test.py
"""
import os
import shutil
import subprocess
import sys
import tempfile
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT / 'tools'))
import worktree # noqa: E402
IGNORED = '\n'.join('/' + d + '/' for d in worktree.LINK_DIRS) + '\n/' + worktree.LINK_FILES[0] + '\n/.worktrees/\n'
def git(cwd, *a):
r = subprocess.run(['git', '-C', str(cwd), *a], capture_output=True, text=True)
assert r.returncode == 0, f'git {a}: {r.stderr}'
return r.stdout
def tool(scratch, *a):
r = subprocess.run([sys.executable, str(scratch / 'tools' / 'worktree.py'), *a], capture_output=True, text=True)
assert r.returncode == 0, f'worktree.py {a}: {r.stdout}{r.stderr}'
return r.stdout
def markers(main):
return [main / d / 'deep' / 'marker.txt' for d in worktree.LINK_DIRS if (main / d).is_dir()] + \
[main / f for f in worktree.LINK_FILES]
def check(main, what):
for m in markers(main):
assert m.is_file() and m.read_text() == 'keep', f'{what}: {m} lost'
print(f'ok: {what}: {len(markers(main))} markers intact')
def make_repo(base):
main = base / 'main'
(main / 'tools').mkdir(parents=True)
shutil.copy(ROOT / 'tools' / 'worktree.py', main / 'tools' / 'worktree.py')
(main / '.gitignore').write_text(IGNORED)
(main / 'README.md').write_text('x\n')
git(main, 'init', '-b', 'main')
git(main, 'config', 'user.email', 't@t')
git(main, 'config', 'user.name', 't')
git(main, 'add', '.')
git(main, 'commit', '-m', 'init')
for d in worktree.LINK_DIRS:
if d == 'game':
continue # absent in a real checkout too: link only what exists
(main / d / 'deep').mkdir(parents=True)
(main / d / 'deep' / 'marker.txt').write_text('keep')
for f in worktree.LINK_FILES:
(main / f).write_text('keep')
return main
def main():
base = Path(tempfile.mkdtemp(prefix='wt-test-')).resolve()
try:
main_dir = make_repo(base)
check(main_dir, 'setup')
# tools/worktree.py new / status / remove
wt = Path(tool(main_dir, 'new', 'a').strip())
assert wt.resolve() == (main_dir / '.worktrees' / 'a').resolve(), wt
for d in worktree.LINK_DIRS:
if (main_dir / d).is_dir():
assert os.path.isjunction(wt / d), d
assert (wt / 'build' / 'deep' / 'marker.txt').read_text() == 'keep'
assert os.path.samefile(wt / 'config.local.toml', main_dir / 'config.local.toml')
status = git(wt, 'status', '--porcelain', '--ignored=no')
assert status == '', f'links show in git status: {status!r}'
assert git(wt, 'ls-files', '--others', '--exclude-standard') == ''
print('ok: git status in the worktree is clean, links are ignored')
assert '.worktrees/a' in tool(main_dir, 'list').replace(os.sep, '/')
(wt / 'dirty.txt').write_text('x')
r = subprocess.run([sys.executable, str(main_dir / 'tools' / 'worktree.py'), 'remove', 'a'],
capture_output=True, text=True)
assert r.returncode != 0 and (wt / 'build').is_dir(), 'remove must refuse a dirty worktree'
check(main_dir, 'dirty remove refused')
(wt / 'dirty.txt').unlink()
tool(main_dir, 'remove', 'a')
assert not wt.exists()
check(main_dir, 'tools/worktree.py remove')
# unmerged branch: --delete-branch refuses, --force deletes
wt = Path(tool(main_dir, 'new', 'b').strip())
(wt / 'f.txt').write_text('x')
git(wt, 'add', 'f.txt')
git(wt, 'commit', '-m', 'work')
r = subprocess.run([sys.executable, str(main_dir / 'tools' / 'worktree.py'), 'remove', 'b', '--delete-branch'],
capture_output=True, text=True)
assert r.returncode != 0 and wt.is_dir(), 'unmerged branch must block --delete-branch'
git(main_dir, 'merge', '--ff-only', 'task/b')
tool(main_dir, 'remove', 'b', '--delete-branch')
assert 'task/b' not in git(main_dir, 'branch', '--list')
check(main_dir, 'remove --delete-branch after merge')
# raw git: worktree remove --force with the links in place
wt = Path(tool(main_dir, 'new', 'c').strip())
git(main_dir, 'worktree', 'remove', '--force', str(wt))
check(main_dir, 'raw git worktree remove --force')
# raw: delete the worktree directory with the junctions in place, then prune
wt = Path(tool(main_dir, 'new', 'e').strip())
shutil.rmtree(wt)
git(main_dir, 'worktree', 'prune')
check(main_dir, 'rmtree of a linked worktree, then git worktree prune')
print('PASS')
finally:
# never follow a junction while cleaning up
for w in (base / 'main' / '.worktrees').glob('*'):
for d in worktree.LINK_DIRS:
worktree.drop_link(w / d)
shutil.rmtree(base, ignore_errors=True)
if __name__ == '__main__':
main()