mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-20 10:19:57 -04:00
Import the base tree (MIT, see LICENSE)
This commit is contained in:
221
tests/diff/atomic.py
Normal file
221
tests/diff/atomic.py
Normal file
@@ -0,0 +1,221 @@
|
||||
"""Differential test of exclusives and atomics against Unicorn (max CPU model, which has the ARMv8.1 LSE atomics).
|
||||
|
||||
Single steps cannot cover these: a store exclusive only succeeds after a load exclusive, and the A57 model rejects
|
||||
LSE. So this test enumerates every variant and runs short sequences, with the monitor cleared before each case:
|
||||
|
||||
- LDXR/LDAXR then STXR/STLXR to the same address (succeeds), all sizes;
|
||||
- LDXP/LDAXP then STXP/STLXP to the same address (succeeds) or 16 bytes further (fails), W and X pairs;
|
||||
- STXR and STXP alone (fail without a monitor);
|
||||
- LDADD, LDCLR, LDEOR, LDSET, LDSMAX, LDSMIN, LDUMAX, LDUMIN and SWP, every size and ordering;
|
||||
- CAS and CASP, every size and ordering, with the compare value equal to memory in half of the cases.
|
||||
|
||||
Needs no game files. Usage: atomic.py [--cases K] [--seed S] [--jobs J]
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import platform
|
||||
import random
|
||||
import struct
|
||||
import sys
|
||||
import time
|
||||
import zlib
|
||||
|
||||
import unicorn as uc
|
||||
from unicorn import arm64_const as A
|
||||
|
||||
from common import BUF, BUF_SIZE, ROOT, Emu, build_harness, diff_state, dis, rand_fpcr, rand_v, rand_x, run_harness
|
||||
from lifter import Lifter
|
||||
|
||||
BUF_MID = BUF + 0x10000
|
||||
CODE = 0x7220000000
|
||||
SLOT = 16 # bytes of code per test
|
||||
CLREX = 0xD5033F5F
|
||||
|
||||
|
||||
def lse(size, a, r, rs, o3, opc, rn, rt):
|
||||
return 0x38200000 | size << 30 | a << 23 | r << 22 | rs << 16 | o3 << 15 | opc << 12 | rn << 5 | rt
|
||||
|
||||
|
||||
def cas(size, l, rs, o0, rn, rt):
|
||||
return 0x08A07C00 | size << 30 | l << 22 | rs << 16 | o0 << 15 | rn << 5 | rt
|
||||
|
||||
|
||||
def casp(sz, l, rs, o0, rn, rt):
|
||||
return 0x08207C00 | sz << 30 | l << 22 | rs << 16 | o0 << 15 | rn << 5 | rt
|
||||
|
||||
|
||||
def ldxp(sz, o0, rt2, rn, rt):
|
||||
return 0x887F0000 | sz << 30 | o0 << 15 | rt2 << 10 | rn << 5 | rt
|
||||
|
||||
|
||||
def stxp(sz, o0, rs, rt2, rn, rt):
|
||||
return 0x88200000 | sz << 30 | rs << 16 | o0 << 15 | rt2 << 10 | rn << 5 | rt
|
||||
|
||||
|
||||
def ldxr(size, o0, rn, rt):
|
||||
return 0x085F7C00 | size << 30 | o0 << 15 | rn << 5 | rt
|
||||
|
||||
|
||||
def stxr(size, o0, rs, rn, rt):
|
||||
return 0x08007C00 | size << 30 | rs << 16 | o0 << 15 | rn << 5 | rt
|
||||
|
||||
|
||||
def regs(r, n, exclude=()):
|
||||
"""n distinct general registers 0-30 outside exclude."""
|
||||
return r.sample([i for i in range(31) if i not in exclude], n)
|
||||
|
||||
|
||||
def tests(r):
|
||||
"""[(words, rn, kind)]: kind 'cas'/'casp' marks compare-and-swap tests (state gets a matching compare value)."""
|
||||
out = []
|
||||
for size in range(4):
|
||||
for a in (0, 1):
|
||||
for rel in (0, 1):
|
||||
for o3, opc in [(0, k) for k in range(8)] + [(1, 0)]:
|
||||
rn = r.choice(regs(r, 1) + [31])
|
||||
rs, rt = r.randrange(32), r.randrange(32)
|
||||
out.append(([lse(size, a, rel, rs, o3, opc, rn, rt)], rn, 'lse'))
|
||||
rn = r.choice(regs(r, 1) + [31])
|
||||
rs, rt = regs(r, 2, {rn})
|
||||
out.append(([cas(size, a, rs, rel, rn, rt)], rn, 'cas'))
|
||||
for sz in (0, 1):
|
||||
for l in (0, 1):
|
||||
for o0 in (0, 1):
|
||||
rn = r.choice(regs(r, 1) + [31])
|
||||
rs, rt = (2 * i for i in r.sample(range(15), 2))
|
||||
if rn in (rs, rs + 1):
|
||||
rs = 30 if rn < 30 else 0 # Rs+1 = XZR or away from Rn
|
||||
out.append(([casp(sz, l, rs, o0, rn, rt)], rn, 'casp'))
|
||||
for size in range(4):
|
||||
for o0 in (0, 1):
|
||||
rn = r.choice(regs(r, 1) + [31])
|
||||
rt, rs, rt_st = regs(r, 3, {rn})
|
||||
out.append(([ldxr(size, o0, rn, rt), stxr(size, o0, rs, rn, rt_st)], rn, 'excl'))
|
||||
out.append(([stxr(size, o0, rs, rn, rt_st)], rn, 'excl'))
|
||||
for sz in (0, 1):
|
||||
for o0 in (0, 1):
|
||||
rn = r.choice(regs(r, 1) + [31])
|
||||
rt, rt2, rs, d1, d2 = regs(r, 5, {rn})
|
||||
out.append(([ldxp(sz, o0, rt2, rn, rt)], rn, 'excl'))
|
||||
out.append(([ldxp(sz, o0, rt2, rn, rt), stxp(sz, o0, rs, d2, rn, d1)], rn, 'excl'))
|
||||
out.append(([stxp(sz, o0, rs, d2, rn, d1)], rn, 'excl'))
|
||||
# A store exclusive to another address fails; Rn is advanced between the two by an ADD.
|
||||
if rn != 31:
|
||||
add = 0x91004000 | rn << 5 | rn # add xn, xn, #16
|
||||
out.append(([ldxp(sz, o0, rt2, rn, rt), add, stxp(sz, o0, rs, d2, rn, d1)], rn, 'excl'))
|
||||
return out
|
||||
|
||||
|
||||
def unicorn_wrong(w):
|
||||
"""LDSMAX/LDSMIN on bytes, halfwords and words: Unicorn 2.1 (QEMU 5.0) compares them unsigned."""
|
||||
return (w & 0x3F200C00) == 0x38200000 and (w >> 30) < 3 and not (w >> 15) & 1 and (w >> 12) & 7 in (4, 5)
|
||||
|
||||
|
||||
def model_signed_minmax(w, x, sp, nzcv, v, buf0):
|
||||
"""Reference for unicorn_wrong(w) words, from the ARM pseudocode: returns the expected state and memory CRC."""
|
||||
nb = 1 << (w >> 30)
|
||||
rs, rn, rt = (w >> 16) & 31, (w >> 5) & 31, w & 31
|
||||
addr = sp if rn == 31 else x[rn]
|
||||
off = addr - BUF
|
||||
old = int.from_bytes(buf0[off:off + nb], 'little')
|
||||
operand = (x[rs] if rs != 31 else 0) & ((1 << 8 * nb) - 1)
|
||||
sign = lambda u: u - (1 << 8 * nb) if u >> (8 * nb - 1) else u
|
||||
pick = max if (w >> 12) & 7 == 4 else min
|
||||
new = old if pick(sign(old), sign(operand)) == sign(old) else operand
|
||||
mem = bytearray(buf0)
|
||||
mem[off:off + nb] = new.to_bytes(nb, 'little')
|
||||
ox = list(x[:31])
|
||||
if rt != 31:
|
||||
ox[rt] = old
|
||||
return (ox, sp, nzcv, 0, list(v), zlib.crc32(bytes(mem)))
|
||||
|
||||
|
||||
def make_case(r, words, rn, kind, buf0):
|
||||
addr = BUF_MID + r.randrange(0, 0x100, 16)
|
||||
x = [rand_x(r) for _ in range(31)]
|
||||
sp = addr if rn == 31 else BUF_MID + r.randrange(0, 0x100, 16)
|
||||
if rn != 31:
|
||||
x[rn] = addr
|
||||
w = words[0]
|
||||
if kind in ('cas', 'casp') and r.random() < 0.5:
|
||||
off = addr - BUF
|
||||
rs = (w >> 16) & 31
|
||||
if kind == 'cas':
|
||||
nb = 1 << (w >> 30)
|
||||
if rs != 31:
|
||||
x[rs] = int.from_bytes(buf0[off:off + nb], 'little')
|
||||
else:
|
||||
nb = 8 if (w >> 30) & 1 else 4
|
||||
for i in (0, 1):
|
||||
if rs + i != 31:
|
||||
x[rs + i] = int.from_bytes(buf0[off + i * nb:off + (i + 1) * nb], 'little')
|
||||
return rand_fpcr(r), x, sp, r.getrandbits(4) << 28, [rand_v(r) for _ in range(32)]
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument('--cases', type=int, default=64)
|
||||
ap.add_argument('--seed', type=int, default=1)
|
||||
ap.add_argument('--jobs', type=int, default=os.cpu_count() or 4)
|
||||
ap.add_argument('--cflags', default='-march=x86-64-v3' if platform.machine().lower() in ('amd64', 'x86_64') else '')
|
||||
args = ap.parse_args()
|
||||
r = random.Random(args.seed)
|
||||
t0 = time.time()
|
||||
|
||||
ts = tests(r)
|
||||
code = [CLREX, 0, 0, 0]
|
||||
bodies = []
|
||||
lifter = Lifter(0, 'c')
|
||||
for i, (words, _, _) in enumerate(ts):
|
||||
pc = CODE + SLOT * (i + 1)
|
||||
code += words + [0] * (SLOT // 4 - len(words))
|
||||
bodies.append(' '.join(lifter.lift(w, pc + 4 * j).code for j, w in enumerate(words)))
|
||||
code_bytes = struct.pack(f'<{len(code)}I', *code)
|
||||
work = ROOT / 'build' / 'test' / 'atomic'
|
||||
work.mkdir(parents=True, exist_ok=True)
|
||||
code_path = work / 'code.bin'
|
||||
code_path.write_bytes(code_bytes)
|
||||
exe = build_harness(work, bodies, args.jobs, args.cflags)
|
||||
|
||||
emu = Emu([(CODE, code_bytes)], A.UC_CPU_ARM64_MAX)
|
||||
mu = emu.mu
|
||||
buf0 = bytes(r.getrandbits(8) for _ in range(4096)) * (BUF_SIZE // 4096)
|
||||
mu.mem_write(BUF, buf0)
|
||||
cases, expect = [], []
|
||||
for k, (words, rn, kind) in enumerate(ts):
|
||||
pc = CODE + SLOT * (k + 1)
|
||||
for _ in range(args.cases):
|
||||
fpcr, x, sp, nzcv, v = make_case(r, words, rn, kind, buf0)
|
||||
if unicorn_wrong(words[0]):
|
||||
cases.append((k, fpcr, x, sp, nzcv, v, True))
|
||||
expect.append(model_signed_minmax(words[0], x, sp, nzcv, v, buf0))
|
||||
continue
|
||||
mu.emu_start(CODE, CODE + 4, count=1) # clear the exclusive monitor
|
||||
try:
|
||||
ox, osp, onz, opc, ov = emu.run(pc, fpcr, x, sp, nzcv, v, count=len(words))
|
||||
except uc.UcError as e:
|
||||
raise SystemExit(f'unicorn rejected {" ; ".join(dis(w) for w in words)}: {e}')
|
||||
mh = zlib.crc32(bytes(mu.mem_read(BUF, BUF_SIZE)))
|
||||
mu.mem_write(BUF, buf0)
|
||||
cases.append((k, fpcr, x, sp, nzcv, v, True))
|
||||
expect.append((ox, osp, onz, opc, ov, mh))
|
||||
|
||||
recs, err = run_harness(exe, work, buf0, cases, [(CODE, code_path)])
|
||||
if err:
|
||||
k = cases[len(recs)][0] if len(recs) < len(cases) else None
|
||||
print(f'harness failed: {err}' + (f', stopped at {" ; ".join(dis(w) for w in ts[k][0])}' if k is not None else ''))
|
||||
fails, nfail = {}, 0
|
||||
for case, want, got in zip(cases, expect, recs):
|
||||
d = diff_state(got, want, check_pc=False, check_mem=True)
|
||||
if d:
|
||||
nfail += 1
|
||||
fails.setdefault(case[0], d[:4])
|
||||
print(f'{len(ts)} tests, {len(cases)} cases: {nfail} failing cases, {len(fails)} failing tests '
|
||||
f'({time.time() - t0:.0f}s)')
|
||||
for k, d in fails.items():
|
||||
print(f' {" ; ".join(dis(w) for w in ts[k][0]):70} {"; ".join(d)}')
|
||||
return 1 if fails or err else 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
276
tests/diff/common.py
Normal file
276
tests/diff/common.py
Normal file
@@ -0,0 +1,276 @@
|
||||
"""Shared parts of the differential tests: Unicorn reference, operand generators, build and run of harness.c.
|
||||
|
||||
Adapted from LAS Recomp tools/test_lift.py and tools/test_simd.py @ da35b83 (MIT, Copyright (c) 2026 Peiikai).
|
||||
"""
|
||||
import os
|
||||
import shutil
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import capstone
|
||||
import unicorn as uc
|
||||
from unicorn import arm64_const as A
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / 'recomp'))
|
||||
|
||||
BUF, BUF_SIZE = 0x7200000000, 0x40000
|
||||
UC_INVALID = (uc.UC_ERR_INSN_INVALID, uc.UC_ERR_EXCEPTION)
|
||||
|
||||
# Record layouts of struct In / struct Out in harness.c.
|
||||
IN_FMT = '<QQ32QQQ'
|
||||
IN_TAIL = '<QQ'
|
||||
OUT_SIZE = 32 * 8 + 4 * 8 + 32 * 16
|
||||
|
||||
MSYS_BIN = Path(r'C:\msys64\ucrt64\bin')
|
||||
|
||||
F32 = [0, 0x80000000, 0x7f800000, 0xff800000, 0x7fc00000, 0xffc00000, 0x7fa00001, 0xff812345, 0x7fd00abc,
|
||||
0x00000001, 0x80000001, 0x007fffff, 0x807fffff, 0x00400000, 0x00800000, 0x80800000, 0x00800001, 0x7f7fffff,
|
||||
0xff7fffff, 0x3f800000, 0xbf800000, 0x3f000000, 0xbf000000, 0x40000000, 0x40400000, 0x3fc00000, 0xbfc00000,
|
||||
0x40200000, 0x40100000, 0x4f000000, 0xcf000000, 0x4f800000, 0x5f000000, 0xdf000000, 0x5f800000, 0x4effffff,
|
||||
0x4f7fffff, 0x5effffff, 0x5f7fffff, 0x3effffff, 0x3f7fffff, 0x7effffff, 0x01000000, 0x3e800000, 0x7e800000,
|
||||
0x7e7fffff, 0x00200000, 0x001fffff, 0x00200001, 0x47800000, 0x477fe000, 0x477ff000, 0x38800000, 0x33800000,
|
||||
0x33000000, 0x387fc000, 0x3f7ff000, 0x46fffe00, 0x80000000 | 0x00200000, 0x7f000000, 0x7f7ffffe]
|
||||
F64 = [0, 1 << 63, 0x7ff0000000000000, 0xfff0000000000000, 0x7ff8000000000000, 0xfff8000000000000,
|
||||
0x7ff4000000000001, 0xfff0000000012345, 0x7ffc00000abcdef0, 1, (1 << 63) | 1, 0x000fffffffffffff,
|
||||
0x0008000000000000, 0x0010000000000000, 0x8010000000000000, 0x7fefffffffffffff, 0xffefffffffffffff,
|
||||
0x3ff0000000000000, 0xbff0000000000000, 0x3fe0000000000000, 0x4000000000000000, 0x4008000000000000,
|
||||
0x3ff8000000000000, 0xbff8000000000000, 0x4004000000000000, 0x41e0000000000000, 0xc1e0000000000000,
|
||||
0x41f0000000000000, 0x43e0000000000000, 0xc3e0000000000000, 0x43f0000000000000, 0x41dfffffffc00000,
|
||||
0x41efffffffe00000, 0x43dfffffffffffff, 0x43efffffffffffff, 0x3fdfffffffffffff, 0x7fdfffffffffffff,
|
||||
0x7fd0000000000000, 0x0004000000000000, 0x0003ffffffffffff, 0x47efffffe0000000, 0x47efffff00000000,
|
||||
0x36a0000000000000, 0x3810000000000000, 0x380fffffffffffff, 0x47f0000000000000, 0x3e70000000000000,
|
||||
0x3f0ffc0000000000, 0x40effc0000000000, 0x40effe0000000000, 0x3e6ffffffff00000, 0x369fffffffffffff,
|
||||
0x3810000010000000, 0x381000000fffffff, 0x3e80000000000000]
|
||||
F16 = [0, 0x8000, 0x7c00, 0xfc00, 0x7e00, 0xfe00, 0x7d00, 0x7e55, 0x0001, 0x8001, 0x03ff, 0x0400, 0x7bff, 0xfbff,
|
||||
0x3c00, 0xbc00, 0x3800, 0x4000, 0x3e00, 0x7fff, 0xffff]
|
||||
|
||||
_md = capstone.Cs(capstone.CS_ARCH_ARM64, capstone.CS_MODE_ARM)
|
||||
|
||||
|
||||
def dis(w, pc=0):
|
||||
d = next(_md.disasm(struct.pack('<I', w), pc), None)
|
||||
return f'{d.mnemonic} {d.op_str}' if d else '?'
|
||||
|
||||
|
||||
def mnemonic(w):
|
||||
return dis(w).split(' ')[0]
|
||||
|
||||
|
||||
def rand_lane(r, sz):
|
||||
"""Random interesting integer lane of 8 << sz bits."""
|
||||
bits_ = 8 << sz
|
||||
k = r.random()
|
||||
if k < 0.35:
|
||||
return r.choice([0, 1, 2, 3, (1 << bits_) - 1, (1 << bits_) - 2, 1 << (bits_ - 1), (1 << (bits_ - 1)) - 1,
|
||||
(1 << (bits_ - 1)) + 1, 0x80 % (1 << bits_), 0x7f, bits_ - 1, bits_, bits_ + 1,
|
||||
(1 << bits_) - bits_, 1 << (bits_ // 2)])
|
||||
if k < 0.5:
|
||||
return r.getrandbits(r.randint(1, bits_))
|
||||
if k < 0.6:
|
||||
return ((1 << bits_) - r.getrandbits(r.randint(1, bits_ - 1))) & ((1 << bits_) - 1)
|
||||
return r.getrandbits(bits_)
|
||||
|
||||
|
||||
def rand_f32(r):
|
||||
k = r.random()
|
||||
if k < 0.4:
|
||||
return r.choice(F32)
|
||||
if k < 0.8:
|
||||
e = r.choice([r.randint(110, 145), r.randint(0, 255), r.randint(120, 135)])
|
||||
return (r.getrandbits(1) << 31) | (e << 23) | (r.getrandbits(23) if r.random() < 0.7 else
|
||||
r.getrandbits(4) << 19)
|
||||
return struct.unpack('<I', struct.pack('<f', r.choice([r.uniform(-4, 4), float(r.randint(-300, 300)) + 0.5,
|
||||
float(r.randint(-5, 5)), r.uniform(-1e6, 1e6)])))[0]
|
||||
|
||||
|
||||
def rand_f64(r):
|
||||
k = r.random()
|
||||
if k < 0.4:
|
||||
return r.choice(F64)
|
||||
if k < 0.8:
|
||||
e = r.choice([r.randint(1000, 1050), r.randint(0, 2047), r.randint(1020, 1030), r.randint(0, 40),
|
||||
r.randint(2000, 2047), r.randint(860, 920)])
|
||||
return (r.getrandbits(1) << 63) | (e << 52) | (r.getrandbits(52) if r.random() < 0.7 else
|
||||
r.getrandbits(6) << 46)
|
||||
return struct.unpack('<Q', struct.pack('<d', r.choice([r.uniform(-4, 4), float(r.randint(-300, 300)) + 0.5,
|
||||
r.uniform(-1e12, 1e12)])))[0]
|
||||
|
||||
|
||||
def rand_v(r):
|
||||
k = r.random()
|
||||
out = 0
|
||||
if k < 0.3:
|
||||
for i in range(4):
|
||||
out |= rand_f32(r) << (32 * i)
|
||||
elif k < 0.55:
|
||||
for i in range(2):
|
||||
out |= rand_f64(r) << (64 * i)
|
||||
elif k < 0.6:
|
||||
for i in range(8):
|
||||
out |= (r.choice(F16) if r.random() < 0.5 else r.getrandbits(16)) << (16 * i)
|
||||
elif k < 0.9:
|
||||
sz = r.randrange(4)
|
||||
for i in range(16 >> sz):
|
||||
out |= rand_lane(r, sz) << ((8 << sz) * i)
|
||||
else:
|
||||
out = r.getrandbits(128)
|
||||
if r.random() < 0.15: # upper half zero, like most 64-bit results
|
||||
out &= (1 << 64) - 1
|
||||
return out
|
||||
|
||||
|
||||
def rand_x(r):
|
||||
k = r.random()
|
||||
if k < 0.3:
|
||||
return r.choice([0, 1, 2, 0xFFFFFFFFFFFFFFFF, 0x7FFFFFFFFFFFFFFF, 0x8000000000000000, 0xFFFFFFFF,
|
||||
0x7FFFFFFF, 0x80000000, 0x100000000, 0xFFFFFFFF80000000, 0x8000000000000001, 0xFFFFFF,
|
||||
0x1000001, 0x20000000000001, 0xFFFFFFFFFF000001, 63, 64, 31, 32, 0xFF, 0xFFFF])
|
||||
if k < 0.45:
|
||||
return rand_f32(r) | (r.getrandbits(32) << 32 if r.random() < 0.3 else 0)
|
||||
if k < 0.6:
|
||||
return rand_f64(r)
|
||||
if k < 0.75:
|
||||
return r.getrandbits(r.randint(1, 64))
|
||||
if k < 0.8:
|
||||
return (-r.getrandbits(r.randint(1, 63))) & 0xFFFFFFFFFFFFFFFF
|
||||
return r.getrandbits(64)
|
||||
|
||||
|
||||
def rand_fpcr(r):
|
||||
k = r.random()
|
||||
if k < 0.55:
|
||||
return 0
|
||||
v = 0
|
||||
if r.random() < 0.4:
|
||||
v |= 1 << 25 # DN
|
||||
if r.random() < 0.4:
|
||||
v |= 1 << 24 # FZ
|
||||
if r.random() < 0.4:
|
||||
v |= r.randrange(4) << 22 # RMode
|
||||
if r.random() < 0.15:
|
||||
v |= 1 << 26 # AHP
|
||||
return v
|
||||
|
||||
|
||||
class Emu:
|
||||
"""Unicorn as a Cortex-A57, the Switch's CPU, unless another model is given (LSE atomics need
|
||||
UC_CPU_ARM64_MAX). Images are (base, bytes) pairs mapped at their guest address."""
|
||||
|
||||
def __init__(self, images, model=A.UC_CPU_ARM64_A57):
|
||||
mu = uc.Uc(uc.UC_ARCH_ARM64, uc.UC_MODE_ARM)
|
||||
mu.ctl_set_cpu_model(model)
|
||||
mu.mem_map(BUF, BUF_SIZE)
|
||||
for base, data in images:
|
||||
mu.mem_map(base, (len(data) + 0xFFFF) & ~0xFFFF)
|
||||
mu.mem_write(base, bytes(data))
|
||||
mu.reg_write(A.UC_ARM64_REG_CPACR_EL1, 0x300000) # enable FP/SIMD
|
||||
self.mu = mu
|
||||
self.xregs = [A.UC_ARM64_REG_X0 + i for i in range(29)] + [A.UC_ARM64_REG_X29, A.UC_ARM64_REG_X30]
|
||||
self.qregs = [A.UC_ARM64_REG_Q0 + i for i in range(32)]
|
||||
self.out_regs = self.xregs + [A.UC_ARM64_REG_SP, A.UC_ARM64_REG_NZCV, A.UC_ARM64_REG_PC] + self.qregs
|
||||
|
||||
def run(self, pc, fpcr, x, sp, nzcv, v, count=1):
|
||||
"""Run count instructions from pc. Returns (x[31], sp, nzcv, pc, v[32])."""
|
||||
mu = self.mu
|
||||
mu.reg_write_batch(list(zip(self.xregs, x[:31])) + list(zip(self.qregs, v)) +
|
||||
[(A.UC_ARM64_REG_SP, sp), (A.UC_ARM64_REG_NZCV, nzcv), (A.UC_ARM64_REG_FPCR, fpcr)])
|
||||
mu.emu_start(pc, pc + 4 * count + 4, count=count)
|
||||
o = mu.reg_read_batch(self.out_regs)
|
||||
return list(o[:31]), o[31], o[32] & 0xF0000000, o[33], list(o[34:])
|
||||
|
||||
|
||||
def compiler():
|
||||
cc = os.environ.get('CC')
|
||||
if cc:
|
||||
return cc
|
||||
if os.name == 'nt' and (MSYS_BIN / 'clang.exe').exists():
|
||||
return str(MSYS_BIN / 'clang.exe')
|
||||
return shutil.which('clang') or 'clang'
|
||||
|
||||
|
||||
def build_harness(work, bodies, jobs, cflags):
|
||||
"""Compile one C function per test body plus harness.c into work/harness[.exe]."""
|
||||
cc = compiler()
|
||||
env = dict(os.environ)
|
||||
if os.name == 'nt' and MSYS_BIN.exists():
|
||||
env['PATH'] = str(MSYS_BIN) + os.pathsep + env.get('PATH', '')
|
||||
flags = ['-O1', '-fno-strict-aliasing', '-Wno-unused-label', '-I', str(ROOT / 'runtime')] + cflags.split()
|
||||
n = max(1, min(jobs, len(bodies) // 200 + 1))
|
||||
per = (len(bodies) + n - 1) // n
|
||||
srcs = []
|
||||
for j in range(n):
|
||||
path = work / f'tests_{j}.c'
|
||||
with open(path, 'w') as f:
|
||||
f.write('#include "cpu.h"\n')
|
||||
for k in range(j * per, min(len(bodies), (j + 1) * per)):
|
||||
f.write(f'void t{k}(Cpu *c) {{ {bodies[k]} }}\n')
|
||||
srcs.append(path)
|
||||
with open(work / 'table.c', 'w') as f:
|
||||
f.write('#include "cpu.h"\n')
|
||||
f.write(''.join(f'void t{k}(Cpu *c);\n' for k in range(len(bodies))))
|
||||
f.write('GuestFn tests[] = {' + ','.join(f't{k}' for k in range(len(bodies))) + '};\n')
|
||||
f.write(f'int ntests = {len(bodies)};\n')
|
||||
# fma.c: a --cflags=-march=x86-64-v2 run must exercise the software fma the v2 build links, not the C runtime's.
|
||||
srcs += [work / 'table.c', ROOT / 'tests' / 'diff' / 'harness.c', ROOT / 'runtime' / 'fma.c']
|
||||
objs, procs = [], []
|
||||
for s in srcs:
|
||||
o = work / (s.stem + '.o')
|
||||
objs.append(o)
|
||||
procs.append(subprocess.Popen([cc] + flags + ['-c', str(s), '-o', str(o)], env=env))
|
||||
if any(p.wait() for p in procs):
|
||||
raise SystemExit('compile failed')
|
||||
exe = work / ('harness.exe' if os.name == 'nt' else 'harness')
|
||||
subprocess.check_call([cc, '-fuse-ld=lld', '-o', str(exe)] + [str(o) for o in objs], env=env)
|
||||
return exe
|
||||
|
||||
|
||||
def write_cases(path, cases, stubs=False):
|
||||
"""cases: (k, fpcr, x[31], sp, nzcv, v[32], memchk). stubs: calls go to harness.c's call stub (rt_stub)."""
|
||||
with open(path, 'wb') as f:
|
||||
for k, fpcr, x, sp, nzcv, v, memchk in cases:
|
||||
f.write(struct.pack(IN_FMT, k, fpcr, *(list(x[:31]) + [0]), sp, nzcv))
|
||||
for q in v:
|
||||
f.write(q.to_bytes(16, 'little'))
|
||||
f.write(struct.pack(IN_TAIL, int(memchk), int(stubs)))
|
||||
|
||||
|
||||
def run_harness(exe, work, buf0, cases, images, stubs=False):
|
||||
"""Run the lifted tests. Returns (records, stderr tail or None). images: (base, path) pairs."""
|
||||
with open(work / 'buf.bin', 'wb') as f:
|
||||
f.write(buf0)
|
||||
write_cases(work / 'cases.bin', cases, stubs)
|
||||
out = work / 'out.bin'
|
||||
args = [str(exe), str(work / 'buf.bin'), str(work / 'cases.bin'), str(out)]
|
||||
for base, path in images:
|
||||
args += [str(path), f'{base:x}']
|
||||
p = subprocess.run(args, capture_output=True)
|
||||
err = None if p.returncode == 0 else f'exit {p.returncode:#x}: {p.stderr.decode(errors="replace")[-2000:]}'
|
||||
data = out.read_bytes() if out.exists() else b''
|
||||
recs = []
|
||||
for i in range(len(data) // OUT_SIZE):
|
||||
rec = data[i * OUT_SIZE:(i + 1) * OUT_SIZE]
|
||||
x = struct.unpack_from('<31Q', rec, 0)
|
||||
sp, nzcv, pc, mem = struct.unpack_from('<4Q', rec, 256)
|
||||
v = [int.from_bytes(rec[288 + 16 * j:304 + 16 * j], 'little') for j in range(32)]
|
||||
recs.append((x, sp, nzcv, pc, mem, v))
|
||||
return recs, err
|
||||
|
||||
|
||||
def diff_state(got, want, check_pc, check_mem, check_nzcv=True):
|
||||
"""got: harness record; want: (x, sp, nzcv, pc, v, mem). Returns a list of mismatches."""
|
||||
hx, hsp, hnz, hpc, hmem, hv = got
|
||||
ex, esp, enz, epc, ev, emem = want
|
||||
d = [f'x{j}: got {hx[j]:x} want {ex[j]:x}' for j in range(31) if hx[j] != ex[j]]
|
||||
if hsp != esp:
|
||||
d.append(f'sp: got {hsp:x} want {esp:x}')
|
||||
if check_nzcv and hnz != enz:
|
||||
d.append(f'nzcv: got {hnz:x} want {enz:x}')
|
||||
if check_pc and hpc != epc:
|
||||
d.append(f'pc: got {hpc:x} want {epc:x}')
|
||||
if check_mem and hmem != emem:
|
||||
d.append('memory differs')
|
||||
d += [f'v{j}: got {hv[j]:032x} want {ev[j]:032x}' for j in range(32) if hv[j] != ev[j]]
|
||||
return d
|
||||
428
tests/diff/func.py
Normal file
428
tests/diff/func.py
Normal file
@@ -0,0 +1,428 @@
|
||||
"""Differential test of whole emitted functions: recomp/gen.py's C for a guest function against Unicorn running the
|
||||
same function from its entry until it returns.
|
||||
|
||||
Picks functions from the ExeFS modules: no branches out of the function, no SVC, BRK, system register, exclusive or
|
||||
atomic instructions; with conditional branches, and loops, jump tables, SIMD and calls where the module has them.
|
||||
Emits them with the generator's own Emitter, compiles them with tests/diff/harness.c, and runs each from random
|
||||
states: x0-x7 point into the scratch buffer (or hold small integers), the buffer holds pointers into itself and random
|
||||
data, SP sits near the buffer's top and LR is a sentinel address. Unicorn (Cortex-A57) stops when PC reaches the
|
||||
sentinel. A case is skipped when Unicorn faults, writes outside the scratch buffer (module images are mapped
|
||||
read-only), leaves the function, or runs longer than --steps instructions. Compares X0-X30, SP, V0-V31 and the
|
||||
scratch buffer; NZCV is local to a generated function (recomp/flags.py).
|
||||
|
||||
Every BL and BLR goes to one stub on both sides (rt_stub in harness.c, Runner.stub here) instead of its callee. The
|
||||
stub folds the guest state at the call into a digest and sets the caller-saved registers from it, so the test sees
|
||||
whether the emitted code had every register in the Cpu at the call and reads them back after it. Callers of another
|
||||
picked function are dropped, as the C side would run that function instead of the stub.
|
||||
|
||||
With --patches, the spin waits of the table (recomp/patches.py `spin`) are emitted with their gates, and functions
|
||||
that hold one come first (a quarter of the picks at most); a case whose loop does not leave runs into --steps.
|
||||
|
||||
Usage: func.py [--exefs DIR] [--modules main,subsdk0,sdk] [--functions N] [--cases K] [--seed S] [--jobs J]
|
||||
[--patches FILE]
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import platform
|
||||
import random
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import zlib
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import unicorn as uc
|
||||
from unicorn import arm64_const as A
|
||||
|
||||
from common import BUF, BUF_SIZE, MSYS_BIN, ROOT, Emu, compiler, diff_state, rand_fpcr, rand_v, rand_x, run_harness
|
||||
import gen
|
||||
import patches
|
||||
from layout import layout
|
||||
from module import Module
|
||||
|
||||
SENTINEL = 0x7210000000 # LR on entry; never executed
|
||||
PTR_LO, PTR_HI = BUF + 0x8000, BUF + 0x28000
|
||||
STACK_TOP = BUF + 0x3F000
|
||||
MIN_INSNS, MAX_INSNS = 6, 600
|
||||
|
||||
|
||||
FNV = 0x100000001B3
|
||||
M64 = (1 << 64) - 1
|
||||
|
||||
|
||||
def splitmix(z):
|
||||
z = (z + 0x9E3779B97F4A7C15) & M64
|
||||
z = ((z ^ (z >> 30)) * 0xBF58476D1CE4E5B9) & M64
|
||||
z = ((z ^ (z >> 27)) * 0x94D049BB133111EB) & M64
|
||||
return z ^ (z >> 31)
|
||||
|
||||
|
||||
def call_mask(w):
|
||||
return ((w & 0xFC000000) == 0x94000000) | ((w & 0xFFFFFC1F) == 0xD63F0000) # BL, BLR
|
||||
|
||||
|
||||
def forbidden_mask(w):
|
||||
"""Words a picked function must not contain."""
|
||||
m = (w & 0xFF000000) == 0xD4000000 # SVC, HVC, SMC, BRK, HLT, DCPS
|
||||
sysi = (w & 0xFFC00000) == 0xD5000000 # system: MRS, MSR, SYS (DC ZVA), hints, barriers
|
||||
hint_or_barrier = ((w & 0xFFFFF000) == 0xD5032000) | ((w & 0xFFFFF000) == 0xD5033000)
|
||||
m |= sysi & ~hint_or_barrier
|
||||
m |= (w & 0x3F000000) == 0x08000000 # exclusives, load-acquire / store-release, CAS
|
||||
m |= (w & 0x3F200C00) == 0x38200000 # LSE atomics
|
||||
return m
|
||||
|
||||
|
||||
def cond_mask(w):
|
||||
return (((w & 0xFE000010) == 0x54000000) | ((w & 0x7E000000) == 0x34000000) |
|
||||
((w & 0x7E000000) == 0x36000000))
|
||||
|
||||
|
||||
def simd_mask(w):
|
||||
return ((w & 0x0E000000) == 0x0E000000) | (((w & 0x0A000000) == 0x08000000) & (((w >> 26) & 1) == 1))
|
||||
|
||||
|
||||
def pick_functions(name, mod, count, r, spins=None):
|
||||
"""[(entry, end, tags)] of up to count functions of mod, plus the Emitter that will emit them. spins: {spin wait
|
||||
back-edge: head}, emitted with gates."""
|
||||
starts, _ = gen.discover(mod)
|
||||
w = mod.words
|
||||
ends = np.append(starts[1:], len(w))
|
||||
|
||||
def per_fn(mask):
|
||||
cs = np.concatenate([[0], np.cumsum(mask.astype(np.int64))])
|
||||
return cs[ends] - cs[starts]
|
||||
|
||||
size = ends - starts
|
||||
ok = (per_fn(forbidden_mask(w)) == 0) & (per_fn(cond_mask(w)) > 0) & (size >= MIN_INSNS) & (size <= MAX_INSNS)
|
||||
has_br = per_fn((w & 0xFFFFFC1F) == 0xD61F0000) > 0
|
||||
has_call = per_fn(call_mask(w)) > 0
|
||||
cand = np.flatnonzero(ok)
|
||||
with_br = [int(i) for i in cand if has_br[i]]
|
||||
with_call = [int(i) for i in cand if has_call[i] and not has_br[i]]
|
||||
rest = [int(i) for i in cand if not has_br[i] and not has_call[i]]
|
||||
spins = spins or {}
|
||||
gated = np.searchsorted(mod.lo + starts * 4, np.array(sorted(spins), dtype=np.int64), side='right') - 1
|
||||
with_spin = sorted({int(i) for i in gated} & set(cand.tolist()))
|
||||
for pool in (with_spin, with_br, with_call, rest):
|
||||
r.shuffle(pool)
|
||||
emitter = gen.Emitter(mod, set((mod.lo + starts * 4).tolist()), spins=spins)
|
||||
picks, tags_seen = [], Counter()
|
||||
# Functions with a spin wait first, then with a br (jump tables, fallbacks), at most a quarter of the picks each;
|
||||
# then functions with calls, at most half.
|
||||
for pool, quota in ((with_spin, count // 4), (with_br, count // 4), (with_call, count // 2), (rest, count)):
|
||||
taken = 0
|
||||
for i in pool:
|
||||
if len(picks) >= count or taken >= quota:
|
||||
break
|
||||
entry, end = mod.lo + int(starts[i]) * 4, mod.lo + int(ends[i]) * 4
|
||||
if any(e == entry for e, _, _ in picks):
|
||||
continue
|
||||
tags = check_function(emitter, entry, end)
|
||||
if tags is None:
|
||||
continue
|
||||
if any(spins.get(x) is not None for x in range(entry, end, 4)):
|
||||
tags.add('spin_gate')
|
||||
picks.append((entry, end, tags))
|
||||
taken += 1
|
||||
picked = {e for e, _, _ in picks}
|
||||
picks = [p for p in picks if not callees(emitter, p[0], p[1]) & picked]
|
||||
for _, _, tags in picks:
|
||||
tags_seen.update(tags)
|
||||
print(f'{name}: {len(starts)} functions, {len(cand)} candidates, picked {len(picks)}: '
|
||||
f'{dict(tags_seen)}', flush=True)
|
||||
return picks, emitter
|
||||
|
||||
|
||||
def call_sites(mod, entry, end):
|
||||
"""Addresses of the BL and BLR instructions of [entry, end)."""
|
||||
return [a for a in range(entry, end, 4) if mod.lift(a).kind in ('bl', 'blr')]
|
||||
|
||||
|
||||
def callees(emitter, entry, end):
|
||||
"""Function starts the emitted C of [entry, end) calls directly (`f_<addr>(c)`, not through rt_dispatch)."""
|
||||
mod = emitter.mod
|
||||
return {mod.lift(a).target for a in call_sites(mod, entry, end)
|
||||
if mod.lift(a).kind == 'bl' and mod.lift(a).target in emitter.starts}
|
||||
|
||||
|
||||
def check_function(emitter, entry, end):
|
||||
"""Tags of a function the test can run in isolation, or None."""
|
||||
mod = emitter.mod
|
||||
insns = [mod.lift(a) for a in range(entry, end, 4)]
|
||||
if insns[-1].kind not in gen.TERMINATORS or not any(i.kind == 'ret' for i in insns):
|
||||
return None
|
||||
tags = set()
|
||||
for k, ins in enumerate(insns):
|
||||
a = entry + 4 * k
|
||||
if ins.kind in ('bl', 'blr'):
|
||||
tags.add('calls')
|
||||
if ins.kind in ('b', 'bcond'):
|
||||
if not entry <= ins.target < end:
|
||||
return None
|
||||
if ins.target <= a:
|
||||
tags.add('loop')
|
||||
if ins.kind == 'br':
|
||||
jt = gen.jumptables.resolve(mod, a, entry, end)
|
||||
if jt:
|
||||
tags.add('jump_table')
|
||||
elif gen.jumptables.is_computed(mod, a, entry):
|
||||
tags.add('all_label_fallback')
|
||||
else:
|
||||
return None
|
||||
if simd_mask(mod.words[(entry - mod.lo) >> 2:(end - mod.lo) >> 2]).any():
|
||||
tags.add('simd')
|
||||
return tags
|
||||
|
||||
|
||||
def build(work, emitted, jobs, cflags):
|
||||
"""Compile the emitted functions with harness.c into work/func[.exe]; tests[k] is emitted[k]."""
|
||||
cc = compiler()
|
||||
env = dict(os.environ)
|
||||
if os.name == 'nt' and MSYS_BIN.exists():
|
||||
env['PATH'] = str(MSYS_BIN) + os.pathsep + env.get('PATH', '')
|
||||
flags = ['-O1', '-fno-strict-aliasing', '-Wno-return-type', '-Wno-unused-label', '-I', str(ROOT / 'runtime')]
|
||||
flags += cflags.split()
|
||||
groups = {}
|
||||
for name, entry, end, emitter in emitted:
|
||||
groups.setdefault(name, []).append((entry, end))
|
||||
srcs, n = [], max(1, jobs)
|
||||
for name, fns in groups.items():
|
||||
emitter = next(e for m, _, _, e in emitted if m == name)
|
||||
per = (len(fns) + n - 1) // n
|
||||
for j in range(0, len(fns), per):
|
||||
path = work / f'{name}_{j // per:03d}.c'
|
||||
path.write_text(gen.chunk_text(name, j // per, emitter, fns[j:j + per]))
|
||||
srcs.append(path)
|
||||
table = work / 'table.c'
|
||||
called = set()
|
||||
for _, entry, end, emitter in emitted:
|
||||
called |= callees(emitter, entry, end)
|
||||
table.write_text('#include "cpu.h"\n' + ''.join(f'void {gen.fname(e)}(Cpu *c);\n' for _, e, _, _ in emitted) +
|
||||
'GuestFn tests[] = {' + ','.join(gen.fname(e) for _, e, _, _ in emitted) + '};\n' +
|
||||
f'int ntests = {len(emitted)};\n' + 'void rt_stub_call(Cpu *c, uint64_t target);\n' +
|
||||
''.join(f'void {gen.fname(t)}(Cpu *c) {{ rt_stub_call(c, {hex(t)}ull); }}\n'
|
||||
for t in sorted(called)))
|
||||
# fma.c: a --cflags=-march=x86-64-v2 run must exercise the software fma the v2 build links, not the C runtime's.
|
||||
srcs += [table, ROOT / 'tests' / 'diff' / 'harness.c', ROOT / 'runtime' / 'fma.c']
|
||||
objs, procs = [], []
|
||||
for s in srcs:
|
||||
o = work / (s.stem + '.o')
|
||||
objs.append(o)
|
||||
procs.append(subprocess.Popen([cc] + flags + ['-c', str(s), '-o', str(o)], env=env))
|
||||
if any(p.wait() for p in procs):
|
||||
raise SystemExit('compile failed')
|
||||
exe = work / ('func.exe' if os.name == 'nt' else 'func')
|
||||
subprocess.check_call([cc, '-fuse-ld=lld', '-o', str(exe)] + [str(o) for o in objs], env=env)
|
||||
return exe
|
||||
|
||||
|
||||
def make_buffer(r):
|
||||
out = bytearray()
|
||||
for _ in range(BUF_SIZE // 8):
|
||||
k = r.random()
|
||||
if k < 0.4:
|
||||
v = r.randrange(PTR_LO, PTR_HI, 8)
|
||||
elif k < 0.7:
|
||||
v = r.randrange(0, 256)
|
||||
else:
|
||||
v = r.getrandbits(64)
|
||||
out += v.to_bytes(8, 'little')
|
||||
return bytes(out)
|
||||
|
||||
|
||||
def make_case(r):
|
||||
x = [rand_x(r) for _ in range(31)]
|
||||
for i in range(8):
|
||||
k = r.random()
|
||||
if k < 0.7:
|
||||
x[i] = r.randrange(PTR_LO, PTR_HI, 16)
|
||||
elif k < 0.9:
|
||||
x[i] = r.randrange(0, 64)
|
||||
x[30] = SENTINEL
|
||||
sp = STACK_TOP - r.randrange(0, 0x100, 16)
|
||||
return rand_fpcr(r), x, sp, r.getrandbits(4) << 28, [rand_v(r) for _ in range(32)]
|
||||
|
||||
|
||||
class Runner:
|
||||
"""Unicorn with the module images read-only, a block hook that stops when execution leaves the function, and a
|
||||
code hook on every call site of the picked functions that stops before the call, for the stub."""
|
||||
|
||||
MAX_CALLS = 1000
|
||||
|
||||
def __init__(self, images, buf0, sites):
|
||||
self.emu = Emu(images)
|
||||
mu = self.emu.mu
|
||||
for base, data in images:
|
||||
mu.mem_protect(base, (len(data) + 0xFFFF) & ~0xFFFF, uc.UC_PROT_READ | uc.UC_PROT_EXEC)
|
||||
mu.mem_map(SENTINEL, 0x1000)
|
||||
mu.mem_write(BUF, buf0)
|
||||
self.buf0 = buf0
|
||||
self.lo = self.hi = self.h = 0
|
||||
self.left = False
|
||||
self.at_call = None
|
||||
mu.hook_add(uc.UC_HOOK_BLOCK, self._block)
|
||||
# One hook per call site, all added before anything is translated, so no cached block lacks one.
|
||||
for a in sorted(sites):
|
||||
mu.hook_add(uc.UC_HOOK_CODE, self._call, begin=a, end=a)
|
||||
|
||||
def _block(self, mu, address, size, user):
|
||||
if not self.lo <= address < self.hi and address != SENTINEL:
|
||||
self.left = True
|
||||
mu.emu_stop()
|
||||
|
||||
def _call(self, mu, address, size, user):
|
||||
if self.lo <= address < self.hi:
|
||||
self.at_call = address
|
||||
mu.emu_stop()
|
||||
|
||||
def stub(self, a):
|
||||
"""harness.c rt_stub for the call at a, which Unicorn has not executed yet. Returns the address to resume at."""
|
||||
e, mu = self.emu, self.emu.mu
|
||||
w = int.from_bytes(mu.mem_read(a, 4), 'little')
|
||||
if (w & 0xFC000000) == 0x94000000: # BL
|
||||
imm = w & 0x03FFFFFF
|
||||
target = (a + ((imm - (1 << 26) if imm >> 25 else imm) << 2)) & M64
|
||||
else: # BLR
|
||||
target = mu.reg_read(e.xregs[(w >> 5) & 31])
|
||||
mu.reg_write(A.UC_ARM64_REG_X30, a + 4)
|
||||
h = ((self.h ^ target) * FNV) & M64
|
||||
for val in list(mu.reg_read_batch(e.xregs)) + [mu.reg_read(A.UC_ARM64_REG_SP)]:
|
||||
h = ((h ^ val) * FNV) & M64
|
||||
for val in mu.reg_read_batch(e.qregs[:8]):
|
||||
h = ((h ^ (val & M64)) * FNV) & M64
|
||||
h = ((h ^ (val >> 64)) * FNV) & M64
|
||||
self.h = h
|
||||
regs = []
|
||||
for i in range(19):
|
||||
s = splitmix((h + i) & M64)
|
||||
regs.append((e.xregs[i], PTR_LO + ((s >> 4) % ((PTR_HI - PTR_LO) // 16)) * 16 if i < 8 and s & 3 else s))
|
||||
regs.append((A.UC_ARM64_REG_NZCV, splitmix((h + 19) & M64) >> 60 << 28))
|
||||
for i in range(8):
|
||||
regs.append((e.qregs[i], splitmix((h + 33 + 2 * i) & M64) << 64 | splitmix((h + 32 + 2 * i) & M64)))
|
||||
mu.reg_write_batch(regs)
|
||||
top = BUF + BUF_SIZE - 16
|
||||
n = int.from_bytes(mu.mem_read(top, 8), 'little')
|
||||
mu.mem_write(top, ((n + 1) & M64).to_bytes(8, 'little') + h.to_bytes(8, 'little'))
|
||||
return a + 4
|
||||
|
||||
def run(self, entry, end, case, steps):
|
||||
"""(x, sp, nzcv, pc, v, crc of the buffer) or None if the case is skipped."""
|
||||
fpcr, x, sp, nzcv, v = case
|
||||
e, mu = self.emu, self.emu.mu
|
||||
self.lo, self.hi, self.left, self.h = entry, end, False, 0
|
||||
mu.reg_write_batch(list(zip(e.xregs, x[:31])) + list(zip(e.qregs, v)) +
|
||||
[(A.UC_ARM64_REG_SP, sp), (A.UC_ARM64_REG_NZCV, nzcv), (A.UC_ARM64_REG_FPCR, fpcr)])
|
||||
pc, ok = entry, False
|
||||
try:
|
||||
for _ in range(self.MAX_CALLS):
|
||||
self.at_call = None
|
||||
mu.emu_start(pc, SENTINEL, count=steps)
|
||||
if self.at_call is None:
|
||||
ok = not self.left and mu.reg_read(A.UC_ARM64_REG_PC) == SENTINEL
|
||||
break
|
||||
pc = self.stub(self.at_call)
|
||||
except uc.UcError:
|
||||
ok = False
|
||||
if not ok:
|
||||
mu.mem_write(BUF, self.buf0)
|
||||
return None
|
||||
o = mu.reg_read_batch(e.out_regs)
|
||||
crc = zlib.crc32(bytes(mu.mem_read(BUF, BUF_SIZE)))
|
||||
mu.mem_write(BUF, self.buf0)
|
||||
return list(o[:31]), o[31], o[32] & 0xF0000000, o[33], list(o[34:]), crc
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument('--exefs', type=Path, default=ROOT / 'build' / 'extracted' / 'v121' / 'exefs')
|
||||
ap.add_argument('--modules', default='main,subsdk0,sdk')
|
||||
ap.add_argument('--functions', type=int, default=400, help='per module')
|
||||
ap.add_argument('--cases', type=int, default=12, help='random states tried per function')
|
||||
ap.add_argument('--steps', type=int, default=200000, help='Unicorn instruction cap per case')
|
||||
ap.add_argument('--seed', type=int, default=1)
|
||||
ap.add_argument('--jobs', type=int, default=os.cpu_count() or 4)
|
||||
ap.add_argument('--show', type=int, default=20)
|
||||
ap.add_argument('--patches', type=Path, help='patch table whose spin waits are emitted with their gates')
|
||||
ap.add_argument('--cflags', default='-march=x86-64-v3' if platform.machine().lower() in ('amd64', 'x86_64') else '')
|
||||
args = ap.parse_args()
|
||||
r = random.Random(args.seed)
|
||||
t0 = time.time()
|
||||
want = args.modules.split(',')
|
||||
|
||||
work = ROOT / 'build' / 'test' / 'func'
|
||||
work.mkdir(parents=True, exist_ok=True)
|
||||
images, emitted, tags = [], [], []
|
||||
mods = layout(args.exefs)
|
||||
table = patches.load(args.patches, mods) if args.patches else patches.Table()
|
||||
for name, base, _ in mods:
|
||||
if name not in want:
|
||||
continue
|
||||
mod = Module(name, args.exefs / name, base, 'c')
|
||||
img = work / f'{name}.img'
|
||||
img.write_bytes(mod.image)
|
||||
images.append((base, img, mod.image))
|
||||
picks, emitter = pick_functions(name, mod, args.functions, r, table.spins_for(name, base))
|
||||
for entry, end, t in picks:
|
||||
emitted.append((name, entry, end, emitter))
|
||||
tags.append(t)
|
||||
print(f'picked {len(emitted)} functions ({time.time() - t0:.0f}s)', flush=True)
|
||||
|
||||
t1 = time.time()
|
||||
exe = build(work, emitted, args.jobs, args.cflags)
|
||||
print(f'compiled ({time.time() - t1:.0f}s)', flush=True)
|
||||
|
||||
buf0 = make_buffer(r)
|
||||
sites = set()
|
||||
for _, entry, end, emitter in emitted:
|
||||
sites.update(call_sites(emitter.mod, entry, end))
|
||||
runner = Runner([(base, data) for base, _, data in images], buf0, sites)
|
||||
cases, expect, tested, skipped = [], [], set(), 0
|
||||
t2 = time.time()
|
||||
for k, (name, entry, end, _) in enumerate(emitted):
|
||||
for _ in range(args.cases):
|
||||
case = make_case(r)
|
||||
want_state = runner.run(entry, end, case, args.steps)
|
||||
if want_state is None:
|
||||
skipped += 1
|
||||
continue
|
||||
fpcr, x, sp, nzcv, v = case
|
||||
cases.append((k, fpcr, x, sp, nzcv, v, True))
|
||||
expect.append(want_state)
|
||||
tested.add(k)
|
||||
print(f'unicorn: {len(cases)} cases on {len(tested)} functions, {skipped} skipped ({time.time() - t2:.0f}s)',
|
||||
flush=True)
|
||||
|
||||
t3 = time.time()
|
||||
recs, err = run_harness(exe, work, buf0, cases, [(base, img) for base, img, _ in images], stubs=True)
|
||||
print(f'harness: {len(recs)} of {len(cases)} cases ({time.time() - t3:.0f}s)')
|
||||
if err:
|
||||
k = cases[len(recs)][0] if len(recs) < len(cases) else None
|
||||
where = f', stopped in {emitted[k][0]} {emitted[k][1]:#x}' if k is not None else ''
|
||||
print(f'harness failed: {err}{where}')
|
||||
|
||||
fails, nfail = {}, 0
|
||||
for case, want_state, got in zip(cases, expect, recs):
|
||||
# Generated functions keep NZCV in a local (recomp/flags.py) and leave c->nzcv as it was: flags only count
|
||||
# where the function reads them, which the registers and memory show.
|
||||
d = diff_state(got, want_state, check_pc=False, check_mem=True, check_nzcv=False)
|
||||
if d:
|
||||
nfail += 1
|
||||
fails.setdefault(case[0], d[:4])
|
||||
by_tag, fail_tag = Counter(), Counter()
|
||||
for k in tested:
|
||||
by_tag.update(tags[k] or {'plain'})
|
||||
if k in fails:
|
||||
fail_tag.update(tags[k] or {'plain'})
|
||||
print(f'{len(tested)} functions, {len(cases)} cases: {nfail} failing cases, {len(fails)} failing functions '
|
||||
f'({time.time() - t0:.0f}s total)')
|
||||
print(f' tested by tag: {dict(by_tag)}; failing by tag: {dict(fail_tag)}')
|
||||
for k, d in list(fails.items())[:args.show]:
|
||||
name, entry, end, _ = emitted[k]
|
||||
print(f' {name:8} {gen.fname(entry)} ({(end - entry) // 4} insns, {sorted(tags[k])}): {"; ".join(d)}')
|
||||
return 1 if fails or err else 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
162
tests/diff/harness.c
Normal file
162
tests/diff/harness.c
Normal file
@@ -0,0 +1,162 @@
|
||||
// Runs lifted tests and writes the resulting CPU state per case.
|
||||
// Shared by tests/diff/lift.py, simd.py, atomic.py and func.py; the expected state comes from Unicorn on the Python
|
||||
// side.
|
||||
//
|
||||
// argv: buf.bin cases.bin out.bin image base [image base ...]
|
||||
// Each image is mapped at its guest address (guest == host), the scratch buffer at BUF.
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#ifdef _WIN32
|
||||
#include <windows.h>
|
||||
#else
|
||||
#include <sys/mman.h>
|
||||
#endif
|
||||
|
||||
#include "cpu.h"
|
||||
|
||||
extern GuestFn tests[];
|
||||
extern int ntests;
|
||||
|
||||
#define BUF 0x7200000000ull
|
||||
#define BUF_SIZE 0x40000
|
||||
|
||||
#define PTR_LO (BUF + 0x8000)
|
||||
#define PTR_SPAN 0x20000
|
||||
|
||||
static int g_stubs; // In.stubs of the running case
|
||||
static uint64_t g_stub_h; // digest of every call's state so far
|
||||
|
||||
static uint64_t splitmix(uint64_t z) {
|
||||
z += 0x9E3779B97F4A7C15ull;
|
||||
z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ull;
|
||||
z = (z ^ (z >> 27)) * 0x94D049BB133111EBull;
|
||||
return z ^ (z >> 31);
|
||||
}
|
||||
|
||||
static uint64_t mix(uint64_t h, uint64_t w) { return (h ^ w) * 0x100000001B3ull; }
|
||||
|
||||
// The callee of every call in a function under tests/diff/func.py; Runner.stub there is the same function on the
|
||||
// Unicorn side. Folds the target, X0-X30, SP and V0-V7 into a digest, so a register the caller left unwritten at the
|
||||
// call changes it, then sets X0-X18, NZCV and V0-V7 from the digest, so a caller reading a register it held across
|
||||
// the call sees a value the callee could have left. X0-X7 mostly get pointers into the scratch buffer. The digest
|
||||
// and the call count go to the top of the scratch buffer, which the memory check covers.
|
||||
static void rt_stub(Cpu *c, uint64_t target) {
|
||||
uint64_t h = mix(g_stub_h, target);
|
||||
for (int i = 0; i < 31; i++) h = mix(h, c->x[i]);
|
||||
h = mix(h, c->sp);
|
||||
for (int i = 0; i < 8; i++) h = mix(mix(h, (uint64_t)c->v[i]), (uint64_t)(c->v[i] >> 64));
|
||||
g_stub_h = h;
|
||||
for (int i = 0; i < 19; i++) {
|
||||
uint64_t s = splitmix(h + i);
|
||||
c->x[i] = i < 8 && (s & 3) ? PTR_LO + ((s >> 4) % (PTR_SPAN / 16)) * 16 : s;
|
||||
}
|
||||
c->nzcv = splitmix(h + 19) >> 60 << 28;
|
||||
for (int i = 0; i < 8; i++)
|
||||
c->v[i] = (__uint128_t)splitmix(h + 33 + 2 * i) << 64 | splitmix(h + 32 + 2 * i);
|
||||
uint64_t *slot = (uint64_t *)(uintptr_t)(BUF + BUF_SIZE - 16);
|
||||
slot[0]++;
|
||||
slot[1] = h;
|
||||
}
|
||||
|
||||
void rt_dispatch(Cpu *c) {
|
||||
if (g_stubs) rt_stub(c, c->pc);
|
||||
}
|
||||
void rt_stub_call(Cpu *c, uint64_t target) { rt_stub(c, target); }
|
||||
void rt_spin_gate(Cpu *c, uint64_t head) { (void)c; (void)head; }
|
||||
void rt_svc(Cpu *c, uint32_t imm, uint64_t pc) { (void)c; (void)imm; (void)pc; }
|
||||
void rt_brk(Cpu *c, uint32_t imm, uint64_t pc) { (void)c; (void)imm; (void)pc; abort(); }
|
||||
void rt_udf(Cpu *c, uint64_t pc) { (void)c; (void)pc; abort(); }
|
||||
uint64_t rt_ticks(void) { return 0; }
|
||||
volatile int g_fnprof;
|
||||
void fnprof_hit(uint64_t addr, uint64_t lr, uint64_t x0) { (void)addr; (void)lr; (void)x0; }
|
||||
|
||||
static void *map_at(uint64_t addr, size_t len) {
|
||||
len = (len + 0xffff) & ~(size_t)0xffff;
|
||||
#ifdef _WIN32
|
||||
void *p = VirtualAlloc((void *)addr, len, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
|
||||
#else
|
||||
void *p = mmap((void *)addr, len, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON | MAP_FIXED, -1, 0);
|
||||
#endif
|
||||
if (p != (void *)addr) {
|
||||
fprintf(stderr, "map %llx (+%zx) failed\n", (unsigned long long)addr, len);
|
||||
exit(1);
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
static void *load_at(const char *path, uint64_t addr) {
|
||||
FILE *f = fopen(path, "rb");
|
||||
if (!f) { perror(path); exit(1); }
|
||||
fseek(f, 0, SEEK_END);
|
||||
size_t size = (size_t)ftell(f);
|
||||
fseek(f, 0, SEEK_SET);
|
||||
void *p = map_at(addr, size);
|
||||
if (fread(p, 1, size, f) != size) { perror(path); exit(1); }
|
||||
fclose(f);
|
||||
return p;
|
||||
}
|
||||
|
||||
static uint32_t crc_tab[256];
|
||||
static uint32_t crc32_buf(const uint8_t *p, size_t n) {
|
||||
uint32_t c = 0xffffffffu;
|
||||
for (size_t i = 0; i < n; i++) c = crc_tab[(c ^ p[i]) & 0xff] ^ (c >> 8);
|
||||
return c ^ 0xffffffffu;
|
||||
}
|
||||
|
||||
// Layouts match IN_FMT / OUT_FMT in tests/diff/common.py.
|
||||
struct In { uint64_t k, fpcr, x[32], sp, nzcv; __uint128_t v[32]; uint64_t memchk, stubs; };
|
||||
struct Out { uint64_t x[32], sp, nzcv, pc, mem; __uint128_t v[32]; };
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
if (argc < 6 || (argc - 4) % 2) {
|
||||
fprintf(stderr, "usage: %s buf.bin cases.bin out.bin image base [image base ...]\n", argv[0]);
|
||||
return 2;
|
||||
}
|
||||
for (uint32_t i = 0; i < 256; i++) {
|
||||
uint32_t c = i;
|
||||
for (int k = 0; k < 8; k++) c = c & 1 ? (c >> 1) ^ 0xEDB88320u : c >> 1;
|
||||
crc_tab[i] = c;
|
||||
}
|
||||
for (int i = 4; i < argc; i += 2) load_at(argv[i], strtoull(argv[i + 1], 0, 16));
|
||||
uint8_t *buf = map_at(BUF, BUF_SIZE);
|
||||
FILE *bf = fopen(argv[1], "rb");
|
||||
if (!bf || fread(buf, 1, BUF_SIZE, bf) != BUF_SIZE) { perror(argv[1]); return 1; }
|
||||
fclose(bf);
|
||||
uint8_t *pristine = malloc(BUF_SIZE);
|
||||
memcpy(pristine, buf, BUF_SIZE);
|
||||
|
||||
FILE *f = fopen(argv[2], "rb"), *o = fopen(argv[3], "wb");
|
||||
if (!f || !o) return 1;
|
||||
static struct In in;
|
||||
static struct Out out;
|
||||
static Cpu c;
|
||||
while (fread(&in, sizeof in, 1, f) == 1) {
|
||||
if (in.k >= (uint64_t)ntests) { fprintf(stderr, "bad test index %llu\n", (unsigned long long)in.k); return 1; }
|
||||
memset(&c, 0, sizeof c);
|
||||
memcpy(c.x, in.x, sizeof c.x);
|
||||
c.sp = in.sp;
|
||||
c.nzcv = in.nzcv;
|
||||
c.fpcr = in.fpcr;
|
||||
rt_fpcr_sync(&c);
|
||||
memcpy(c.v, in.v, sizeof c.v);
|
||||
c.excl_addr = ~0ull;
|
||||
g_stubs = (int)in.stubs;
|
||||
g_stub_h = 0;
|
||||
tests[in.k](&c);
|
||||
memcpy(out.x, c.x, sizeof out.x);
|
||||
out.sp = c.sp;
|
||||
out.nzcv = c.nzcv & 0xf0000000ull;
|
||||
out.pc = c.pc;
|
||||
out.mem = 0;
|
||||
if (in.memchk) {
|
||||
out.mem = crc32_buf(buf, BUF_SIZE);
|
||||
memcpy(buf, pristine, BUF_SIZE);
|
||||
}
|
||||
memcpy(out.v, c.v, sizeof out.v);
|
||||
fwrite(&out, sizeof out, 1, o);
|
||||
// Flushed per case so a crash leaves the completed cases readable.
|
||||
fflush(o);
|
||||
}
|
||||
fclose(o);
|
||||
return 0;
|
||||
}
|
||||
249
tests/diff/lift.py
Normal file
249
tests/diff/lift.py
Normal file
@@ -0,0 +1,249 @@
|
||||
"""Differential test of the lifter on the game's own instructions: lifted C against Unicorn, one instruction at a time.
|
||||
|
||||
Samples distinct instruction words from the .text of each ExeFS module, one per opcode signature first, lifts each
|
||||
with the portable FP/SIMD lowering, runs it from random register state, and compares X0-X30, SP, NZCV, PC (for
|
||||
branches), V0-V31 and the scratch buffer (for loads and stores) with Unicorn as a Cortex-A57.
|
||||
|
||||
Skipped: system instructions and exceptions, whose single step cannot be compared, and exclusives and ARMv8.1
|
||||
atomics, which atomic.py covers. Encodings the lifter rejects are counted per module and mnemonic; --coverage lifts
|
||||
every distinct word instead of testing.
|
||||
|
||||
Adapted from LAS Recomp tools/test_lift.py @ da35b83 (MIT, Copyright (c) 2026 Peiikai).
|
||||
|
||||
Usage: lift.py [--exefs DIR] [--modules main,sdk,subsdk0] [--encodings N] [--cases K] [--seed S] [--jobs J]
|
||||
lift.py --coverage
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import platform
|
||||
import random
|
||||
import sys
|
||||
import time
|
||||
import zlib
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import unicorn as uc
|
||||
|
||||
from common import BUF, BUF_SIZE, ROOT, Emu, build_harness, diff_state, dis, mnemonic, rand_fpcr, rand_v, rand_x, \
|
||||
run_harness
|
||||
from lifter import Lifter, Unsupported, bits
|
||||
from nso import Nso
|
||||
|
||||
BUF_MID = BUF + 0x10000
|
||||
# Test load addresses, below 2^39 and clear of BUF.
|
||||
BASES = {'main': 0x7000000000, 'subsdk0': 0x7008000000, 'sdk': 0x7010000000}
|
||||
|
||||
|
||||
def is_mem(w):
|
||||
return bits(w, 25, 1) == 0 and bits(w, 27, 1) == 1
|
||||
|
||||
|
||||
def classify(w):
|
||||
"""'skip' for instructions whose single-step behaviour cannot be compared."""
|
||||
if (w & 0xFFC00000) == 0xD5000000: # system: hints, barriers, MRS/MSR, cache maintenance
|
||||
return 'skip'
|
||||
if (w & 0x3F000000) == 0x08000000 and (bits(w, 23, 1) == 0 or bits(w, 21, 1)):
|
||||
return 'skip' # exclusives need a paired load; CAS is ARMv8.1; both are covered by atomic.py
|
||||
if (w & 0x3F200C00) == 0x38200000: # LSE atomics, ARMv8.1: atomic.py
|
||||
return 'skip'
|
||||
if (w & 0xFF000000) == 0xD4000000: # SVC, BRK, HLT
|
||||
return 'skip'
|
||||
if (w & 0xFFFF0000) == 0: # UDF
|
||||
return 'skip'
|
||||
return 'ok'
|
||||
|
||||
|
||||
def class_keys(words):
|
||||
"""Opcode signature per word: register fields (Rd, Rn, Rm) cleared."""
|
||||
return words & np.uint32(0xFFE0FC00)
|
||||
|
||||
|
||||
def to_test_body(ins, pc):
|
||||
if ins.kind == 'op':
|
||||
return ins.code
|
||||
nxt = f'0x{pc + 4:x}ull'
|
||||
if ins.kind == 'b':
|
||||
return f'c->pc = 0x{ins.target:x}ull;'
|
||||
if ins.kind == 'bl':
|
||||
return f'c->x[30] = {nxt}; c->pc = 0x{ins.target:x}ull;'
|
||||
if ins.kind == 'bcond':
|
||||
return f'c->pc = ({ins.cond}) ? 0x{ins.target:x}ull : {nxt};'
|
||||
if ins.kind == 'br':
|
||||
return f'c->pc = {ins.target};'
|
||||
if ins.kind == 'blr':
|
||||
return f'{{ uint64_t t = {ins.target}; c->x[30] = {nxt}; c->pc = t; }}'
|
||||
if ins.kind == 'ret':
|
||||
return 'c->pc = c->x[30];'
|
||||
raise Unsupported(ins.kind)
|
||||
|
||||
|
||||
def pick_module(name, nso, base, count, r, unsupported):
|
||||
"""Distinct words of .text, one per opcode signature first, lifted. Returns [(word, pc, body)]."""
|
||||
lo, size = nso.text
|
||||
words = np.frombuffer(bytes(nso.image[lo:lo + size]), dtype='<u4')
|
||||
uniq, first = np.unique(words, return_index=True)
|
||||
keys = class_keys(uniq)
|
||||
order = list(range(len(uniq)))
|
||||
r.shuffle(order)
|
||||
seen, primary, rest = set(), [], []
|
||||
for i in order:
|
||||
k = int(keys[i])
|
||||
(rest if k in seen else primary).append(i)
|
||||
seen.add(k)
|
||||
lifter = Lifter(base, 'c')
|
||||
picks = []
|
||||
for i in primary + rest:
|
||||
if len(picks) >= count:
|
||||
break
|
||||
w = int(uniq[i])
|
||||
if classify(w) == 'skip':
|
||||
continue
|
||||
pc = base + lo + int(first[i]) * 4
|
||||
try:
|
||||
body = to_test_body(lifter.lift(w, pc), pc)
|
||||
except Unsupported:
|
||||
unsupported[(name, mnemonic(w))] += 1
|
||||
continue
|
||||
picks.append((w, pc, body))
|
||||
print(f'{name}: {len(words)} words, {len(uniq)} distinct, {len(seen)} signatures, {len(picks)} picked')
|
||||
return picks
|
||||
|
||||
|
||||
def coverage(name, nso, base):
|
||||
"""Lift every distinct word of .text. Returns {kind mnemonic: [distinct words, occurrences]} of words the lifter
|
||||
rejects (Unsupported) or turns into an undefined-instruction trap (rt_udf)."""
|
||||
lo, size = nso.text
|
||||
words = np.frombuffer(bytes(nso.image[lo:lo + size]), dtype='<u4')
|
||||
uniq, first, counts = np.unique(words, return_index=True, return_counts=True)
|
||||
lifter = Lifter(base, 'c')
|
||||
rejected = {}
|
||||
for w, i, n in zip(uniq.tolist(), first.tolist(), counts.tolist()):
|
||||
try:
|
||||
ins = lifter.lift(w, base + lo + i * 4)
|
||||
if ins.kind != 'trap' or not ins.code.startswith('rt_udf'):
|
||||
continue
|
||||
key = f'udf {mnemonic(w)}'
|
||||
except Unsupported:
|
||||
key = f'unsupported {mnemonic(w)}'
|
||||
e = rejected.setdefault(key, [0, 0])
|
||||
e[0] += 1
|
||||
e[1] += n
|
||||
total = sum(e[1] for e in rejected.values())
|
||||
print(f'{name}: {len(words)} words, {len(uniq)} distinct, {total} rejected '
|
||||
f'({sum(e[0] for e in rejected.values())} distinct)')
|
||||
for mn, (d, n) in sorted(rejected.items(), key=lambda kv: -kv[1][1]):
|
||||
print(f' {mn:24} {n:8} occurrences {d:6} distinct')
|
||||
return rejected
|
||||
|
||||
|
||||
def make_case(r, w):
|
||||
if is_mem(w):
|
||||
x = [BUF_MID + r.randrange(0, 0x100, 8) for _ in range(31)]
|
||||
if (bits(w, 21, 1) and bits(w, 10, 2) == 2) or (w & 0xBFA00000) == 0x0C800000: # register offset
|
||||
rm = bits(w, 16, 5)
|
||||
if rm != bits(w, 5, 5) and rm < 31:
|
||||
x[rm] = r.randrange(0, 0x100)
|
||||
else:
|
||||
x = [rand_x(r) for _ in range(31)]
|
||||
sp = BUF_MID + r.randrange(0, 0x100, 16)
|
||||
nzcv = r.getrandbits(4) << 28
|
||||
v = [rand_v(r) for _ in range(32)]
|
||||
fpcr = rand_fpcr(r)
|
||||
return fpcr, x, sp, nzcv, v
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument('--exefs', type=Path, default=ROOT / 'build' / 'extracted' / 'v121' / 'exefs')
|
||||
ap.add_argument('--modules', default='main,sdk,subsdk0')
|
||||
ap.add_argument('--encodings', type=int, default=20000, help='per module')
|
||||
ap.add_argument('--cases', type=int, default=3, help='random states per encoding')
|
||||
ap.add_argument('--seed', type=int, default=1)
|
||||
ap.add_argument('--jobs', type=int, default=os.cpu_count() or 4)
|
||||
ap.add_argument('--show', type=int, default=40)
|
||||
ap.add_argument('--cflags', default='-march=x86-64-v3' if platform.machine().lower() in ('amd64', 'x86_64') else '')
|
||||
ap.add_argument('--coverage', action='store_true',
|
||||
help='only lift every distinct word and list the rejected ones (capstone "?" = undecodable data)')
|
||||
args = ap.parse_args()
|
||||
r = random.Random(args.seed)
|
||||
t0 = time.time()
|
||||
names = args.modules.split(',')
|
||||
if args.coverage:
|
||||
for name in names:
|
||||
coverage(name, Nso(str(args.exefs / name)), BASES[name])
|
||||
print(f'({time.time() - t0:.0f}s)')
|
||||
return 0
|
||||
|
||||
work = ROOT / 'build' / 'test' / 'lift'
|
||||
work.mkdir(parents=True, exist_ok=True)
|
||||
images, picks, unsupported = [], [], Counter()
|
||||
for name in names:
|
||||
nso = Nso(str(args.exefs / name))
|
||||
base = BASES[name]
|
||||
img = work / f'{name}.img'
|
||||
img.write_bytes(bytes(nso.image))
|
||||
images.append((base, img, nso.image))
|
||||
picks += [(name,) + p for p in pick_module(name, nso, base, args.encodings, r, unsupported)]
|
||||
print(f'lifted {len(picks)} encodings ({time.time() - t0:.0f}s)')
|
||||
|
||||
t1 = time.time()
|
||||
exe = build_harness(work, [p[3] for p in picks], args.jobs, args.cflags)
|
||||
print(f'compiled ({time.time() - t1:.0f}s)')
|
||||
|
||||
emu = Emu([(base, data) for base, _, data in images])
|
||||
mu = emu.mu
|
||||
buf0 = bytes(r.getrandbits(8) for _ in range(4096)) * (BUF_SIZE // 4096)
|
||||
mu.mem_write(BUF, buf0)
|
||||
cases, expect = [], []
|
||||
t2 = time.time()
|
||||
for k, (name, w, pc, _) in enumerate(picks):
|
||||
mem = is_mem(w)
|
||||
for _ in range(args.cases):
|
||||
fpcr, x, sp, nzcv, v = make_case(r, w)
|
||||
try:
|
||||
ox, osp, onz, opc, ov = emu.run(pc, fpcr, x, sp, nzcv, v)
|
||||
except uc.UcError:
|
||||
if mem:
|
||||
mu.mem_write(BUF, buf0)
|
||||
continue
|
||||
mh = 0
|
||||
if mem:
|
||||
mh = zlib.crc32(bytes(mu.mem_read(BUF, BUF_SIZE)))
|
||||
mu.mem_write(BUF, buf0)
|
||||
cases.append((k, fpcr, x, sp, nzcv, v, mem))
|
||||
expect.append((ox, osp, onz, opc, ov, mh))
|
||||
print(f'unicorn: {len(cases)} cases ({time.time() - t2:.0f}s)')
|
||||
|
||||
t3 = time.time()
|
||||
recs, err = run_harness(exe, work, buf0, cases, [(base, img) for base, img, _ in images])
|
||||
print(f'harness: {len(recs)} of {len(cases)} cases ({time.time() - t3:.0f}s)')
|
||||
if err:
|
||||
k = cases[len(recs)][0] if len(recs) < len(cases) else None
|
||||
where = f', stopped at {picks[k][1]:08x} {dis(picks[k][1])}' if k is not None else ''
|
||||
print(f'harness failed: {err}{where}')
|
||||
|
||||
fails, nfail = {}, 0
|
||||
for case, want, got in zip(cases, expect, recs):
|
||||
k = case[0]
|
||||
name, w, pc, _ = picks[k]
|
||||
d = diff_state(got, want, check_pc=got[3] != 0, check_mem=case[6])
|
||||
if d:
|
||||
nfail += 1
|
||||
fails.setdefault(w, (name, pc, d[:4]))
|
||||
|
||||
print('unsupported encodings (distinct words) by module and mnemonic:')
|
||||
for (name, mn), n in sorted(unsupported.items(), key=lambda kv: (kv[0][0], -kv[1])):
|
||||
print(f' {name:8} {mn:10} {n}')
|
||||
print(f'{len(picks)} encodings, {len(cases)} cases: {nfail} failing cases, {len(fails)} failing encodings '
|
||||
f'({time.time() - t0:.0f}s total)')
|
||||
if fails:
|
||||
print(' failing by mnemonic:', dict(Counter(mnemonic(w) for w in fails).most_common(40)))
|
||||
for w, (name, pc, d) in list(fails.items())[:args.show]:
|
||||
print(f' {name:8} {w:08x} {dis(w, pc):40} {"; ".join(d)}')
|
||||
return 1 if fails or err else 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
242
tests/diff/simd.py
Normal file
242
tests/diff/simd.py
Normal file
@@ -0,0 +1,242 @@
|
||||
"""Differential test of the portable FP/SIMD lowering (lifter simd='c') against Unicorn as a Cortex-A57.
|
||||
|
||||
Needs no game files. Enumerates every opcode bit pattern of the FP/SIMD data-processing space (bits 27..25 = 111),
|
||||
the FP/SIMD load/store space and CRC32 with random register fields, and compares which encodings the lifter and
|
||||
Unicorn accept. Then a stratified sample of encodings both accept is compiled and run with random interesting
|
||||
operands (NaNs, infinities, denormals, +-0, saturation edges, random FPCR.DN/FZ/RMode/AHP): X0-X30, SP, NZCV, V0-V31
|
||||
and, for stores, memory must match. FPSR is not compared; the lowering does not model the cumulative exception bits
|
||||
or QC.
|
||||
|
||||
Adapted from LAS Recomp tools/test_simd.py @ da35b83 (MIT, Copyright (c) 2026 Peiikai).
|
||||
|
||||
Usage: simd.py [--encodings N] [--cases K] [--seed S] [--jobs J] [--no-validity] [--filter HEX/MASK]
|
||||
"""
|
||||
import argparse
|
||||
import functools
|
||||
import os
|
||||
import platform
|
||||
import random
|
||||
import struct
|
||||
import sys
|
||||
import time
|
||||
import zlib
|
||||
from collections import Counter, defaultdict
|
||||
|
||||
import unicorn as uc
|
||||
from unicorn import arm64_const as A
|
||||
|
||||
from common import BUF, BUF_SIZE, ROOT, UC_INVALID, Emu, build_harness, diff_state, dis, mnemonic, rand_fpcr, \
|
||||
rand_v, rand_x, run_harness
|
||||
from lifter import Lifter, Unsupported
|
||||
|
||||
print = functools.partial(print, flush=True)
|
||||
BASE_AREA = BUF + 0x8000
|
||||
CODE, CODE_SIZE = 0x7210000000, 0x400000
|
||||
|
||||
|
||||
def is_ldst(w):
|
||||
return (w >> 25) & 1 == 0 and (w >> 27) & 1 == 1
|
||||
|
||||
|
||||
def is_store(w):
|
||||
if not is_ldst(w):
|
||||
return False
|
||||
if (w & 0x3B000000) == 0x18000000: # literal
|
||||
return False
|
||||
return not (w >> 22) & 1 # L bit, same position for structures, pairs and single registers
|
||||
|
||||
|
||||
def is_literal(w):
|
||||
return (w & 0x3F000000) == 0x1C000000
|
||||
|
||||
|
||||
def class_key(w):
|
||||
"""Opcode signature: register fields (Rd, Rn, Rm) cleared."""
|
||||
if is_ldst(w):
|
||||
return w & 0xFFE0FC00 if (w & 0xBE000000) == 0x0C000000 else w & 0xFFFFFC00
|
||||
return w & 0xFFE0FC00
|
||||
|
||||
|
||||
def candidates(r):
|
||||
"""Every opcode bit pattern of the FP/SIMD data-processing and load/store spaces, random registers."""
|
||||
out = []
|
||||
for p in range(1 << 19):
|
||||
hi, mid = p >> 15, p & 0x7FFF
|
||||
rr = r.getrandbits(10)
|
||||
out.append((hi << 28) | (0b111 << 25) | (mid << 10) | rr) # data processing
|
||||
out.append((hi << 28) | (1 << 27) | (1 << 26) | (mid << 10) | rr) # loads/stores
|
||||
for sf in (0, 1):
|
||||
for C in (0, 1):
|
||||
for sz in range(4):
|
||||
for _ in range(4):
|
||||
out.append((sf << 31) | (0b0011010110 << 21) | (r.getrandbits(5) << 16) | (0b010 << 13) |
|
||||
(C << 12) | (sz << 10) | r.getrandbits(10))
|
||||
return out
|
||||
|
||||
|
||||
def validity(lifter, ws, emu):
|
||||
"""Map word -> (lifter accepts, Unicorn accepts); words are placed at CODE + 4i."""
|
||||
mu = emu.mu
|
||||
res = {}
|
||||
step = CODE_SIZE // 4
|
||||
for base in range(0, len(ws), step):
|
||||
chunk = ws[base:base + step]
|
||||
mu.mem_write(CODE, struct.pack(f'<{len(chunk)}I', *chunk))
|
||||
mu.ctl_remove_cache(CODE, CODE + CODE_SIZE)
|
||||
state = list(zip(emu.xregs, [BASE_AREA] * 31)) + [(A.UC_ARM64_REG_SP, BASE_AREA)]
|
||||
for i, w in enumerate(chunk):
|
||||
pc = CODE + 4 * i
|
||||
try:
|
||||
lifter.lift(w, pc)
|
||||
lok = True
|
||||
except Unsupported:
|
||||
lok = False
|
||||
if i % 32 == 0:
|
||||
mu.reg_write_batch(state)
|
||||
try:
|
||||
mu.emu_start(pc, pc + 4, count=1)
|
||||
uok = True
|
||||
except uc.UcError as e:
|
||||
uok = e.errno not in UC_INVALID
|
||||
res[w] = (lok, uok)
|
||||
return res
|
||||
|
||||
|
||||
def make_case(r, w):
|
||||
fpcr = rand_fpcr(r)
|
||||
v = [rand_v(r) for _ in range(32)]
|
||||
nzcv = r.getrandbits(4) << 28
|
||||
sp = BASE_AREA + r.randrange(0, 0x100, 16)
|
||||
if is_ldst(w):
|
||||
x = [BASE_AREA + r.randrange(0, 0x100) for _ in range(31)]
|
||||
rm = (w >> 16) & 31
|
||||
if rm < 31 and rm != (w >> 5) & 31:
|
||||
reg_off = (w & 0x3B200C00) == 0x38200800 # register offset
|
||||
post_reg = (w & 0xBF800000) in (0x0C800000, 0x0D800000)
|
||||
if reg_off or post_reg:
|
||||
x[rm] = r.randrange(0, 0x100)
|
||||
else:
|
||||
x = [rand_x(r) for _ in range(31)]
|
||||
return fpcr, x, sp, nzcv, v
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument('--encodings', type=int, default=20000)
|
||||
ap.add_argument('--cases', type=int, default=8)
|
||||
ap.add_argument('--seed', type=int, default=1)
|
||||
ap.add_argument('--jobs', type=int, default=os.cpu_count() or 4)
|
||||
ap.add_argument('--no-validity', action='store_true', help='skip the full validity enumeration')
|
||||
ap.add_argument('--filter', help='only encodings with (w & MASK) == VAL, as VAL/MASK in hex')
|
||||
ap.add_argument('--show', type=int, default=40)
|
||||
ap.add_argument('--cflags', default='-march=x86-64-v3' if platform.machine().lower() in ('amd64', 'x86_64') else '')
|
||||
args = ap.parse_args()
|
||||
r = random.Random(args.seed)
|
||||
t0 = time.time()
|
||||
lifter = Lifter(0, 'c')
|
||||
emu = Emu([(CODE, bytes(CODE_SIZE))])
|
||||
cands = candidates(r)
|
||||
if args.filter:
|
||||
val, mask = (int(x, 16) for x in args.filter.split('/'))
|
||||
cands = [w for w in cands if (w & mask) == val]
|
||||
if args.no_validity:
|
||||
val = validity(lifter, r.sample(cands, min(len(cands), args.encodings * 4)), emu)
|
||||
else:
|
||||
val = validity(lifter, cands, emu)
|
||||
agree = Counter()
|
||||
only = defaultdict(list)
|
||||
for w, (lok, uok) in val.items():
|
||||
agree[(lok, uok)] += 1
|
||||
if lok != uok:
|
||||
only['lifter' if lok else 'unicorn'].append(w)
|
||||
print(f'validity: {len(val)} encodings, both valid {agree[(True, True)]}, both invalid {agree[(False, False)]}, '
|
||||
f'lifter-only {agree[(True, False)]}, unicorn-only {agree[(False, True)]} ({time.time() - t0:.0f}s)')
|
||||
for kind in ('lifter', 'unicorn'):
|
||||
ws = only[kind]
|
||||
if ws:
|
||||
print(f' {kind}-only by mnemonic: {dict(Counter(mnemonic(w) for w in ws).most_common(30))}')
|
||||
for w in ws[:args.show // 2]:
|
||||
print(f' {w:08x} {dis(w)}')
|
||||
|
||||
# Stratified sample of valid encodings: one per opcode signature first.
|
||||
valid = [w for w, (lok, uok) in val.items() if lok and uok]
|
||||
r.shuffle(valid)
|
||||
seen, pick = set(), []
|
||||
for w in valid:
|
||||
k = class_key(w)
|
||||
if k not in seen:
|
||||
seen.add(k)
|
||||
pick.append(w)
|
||||
picked = set(pick)
|
||||
pick = (pick + [w for w in valid if w not in picked])[:args.encodings]
|
||||
# Placed at CODE + 4i; literal loads get an offset into the code region.
|
||||
for i, w in enumerate(pick):
|
||||
pc = CODE + 4 * i
|
||||
if is_literal(w):
|
||||
tgt = CODE + r.randrange(0, (len(pick) * 4 + 0x1000) & ~3, 4)
|
||||
pick[i] = (w & ~(0x7FFFF << 5)) | ((((tgt - pc) >> 2) & 0x7FFFF) << 5)
|
||||
code_bytes = struct.pack(f'<{len(pick)}I', *pick) + bytes(0x1000 + 64)
|
||||
bodies = [lifter.lift(w, CODE + 4 * i).code for i, w in enumerate(pick)]
|
||||
work = ROOT / 'build' / 'test' / 'simd'
|
||||
work.mkdir(parents=True, exist_ok=True)
|
||||
t1 = time.time()
|
||||
exe = build_harness(work, bodies, args.jobs, args.cflags)
|
||||
print(f'compiled {len(bodies)} tests ({time.time() - t1:.0f}s)')
|
||||
|
||||
mu = emu.mu
|
||||
mu.mem_write(CODE, code_bytes)
|
||||
mu.ctl_remove_cache(CODE, CODE + CODE_SIZE)
|
||||
buf0 = bytes(r.getrandbits(8) for _ in range(4096)) * (BUF_SIZE // 4096)
|
||||
mu.mem_write(BUF, buf0)
|
||||
code_path = work / 'code.bin'
|
||||
code_path.write_bytes(code_bytes)
|
||||
cases, expect = [], []
|
||||
t2 = time.time()
|
||||
for k, w in enumerate(pick):
|
||||
pc = CODE + 4 * k
|
||||
st = is_store(w)
|
||||
for _ in range(args.cases):
|
||||
fpcr, x, sp, nzcv, v = make_case(r, w)
|
||||
try:
|
||||
ox, osp, onz, opc, ov = emu.run(pc, fpcr, x, sp, nzcv, v)
|
||||
except uc.UcError:
|
||||
if st:
|
||||
mu.mem_write(BUF, buf0)
|
||||
continue
|
||||
mh = 0
|
||||
if st:
|
||||
mh = zlib.crc32(bytes(mu.mem_read(BUF, BUF_SIZE)))
|
||||
mu.mem_write(BUF, buf0)
|
||||
cases.append((k, fpcr, x, sp, nzcv, v, st))
|
||||
expect.append((ox, osp, onz, opc, ov, mh))
|
||||
print(f'unicorn: {len(cases)} cases ({time.time() - t2:.0f}s)')
|
||||
|
||||
t3 = time.time()
|
||||
recs, err = run_harness(exe, work, buf0, cases, [(CODE, code_path)])
|
||||
print(f'harness: {len(recs)} of {len(cases)} cases ({time.time() - t3:.0f}s)')
|
||||
if err:
|
||||
k = cases[len(recs)][0] if len(recs) < len(cases) else None
|
||||
print(f'harness failed: {err}' + (f', stopped at {pick[k]:08x} {dis(pick[k])}' if k is not None else ''))
|
||||
|
||||
fails, nfail = {}, 0
|
||||
for case, want, got in zip(cases, expect, recs):
|
||||
k, v, st = case[0], case[5], case[6]
|
||||
d = diff_state(got, want, check_pc=False, check_mem=st)
|
||||
if d:
|
||||
nfail += 1
|
||||
w = pick[k]
|
||||
if w not in fails:
|
||||
ins = [f'v{j}={v[j]:032x}' for j in sorted({(w >> 5) & 31, (w >> 16) & 31, w & 31})]
|
||||
fails[w] = (case[1], d[:3], ins)
|
||||
print(f'{len(pick)} encodings, {len(cases)} cases: {nfail} failing cases, {len(fails)} failing encodings '
|
||||
f'({time.time() - t0:.0f}s total)')
|
||||
if fails:
|
||||
print(' failing by mnemonic:', dict(Counter(mnemonic(w) for w in fails).most_common(40)))
|
||||
for w, (fpcr, d, ins) in list(fails.items())[:args.show]:
|
||||
print(f' {w:08x} {dis(w):40} fpcr={fpcr:x} {"; ".join(d)}')
|
||||
print(f' {" ".join(ins)}')
|
||||
return 1 if fails or err else 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
147
tests/excl_mt.py
Normal file
147
tests/excl_mt.py
Normal file
@@ -0,0 +1,147 @@
|
||||
"""Multithreaded test of the exclusive monitor helpers in cpu.h (ldxN/stxN, ldxp128/stxp128).
|
||||
|
||||
- ABA: thread A loads X exclusively, thread B changes X->Y->X with its own exclusive pairs, then A's store exclusive
|
||||
must fail. Checked for the 32-bit, 64-bit and 128-bit pair forms, plus a control where B touches nothing.
|
||||
- Contention: N threads increment one counter with load/store-exclusive retry loops; the total must be exact.
|
||||
|
||||
Needs no game files. Usage: excl_mt.py [--runtime DIR] [--threads N] [--iters K]
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent / 'diff'))
|
||||
from common import MSYS_BIN, ROOT, compiler # noqa: E402
|
||||
|
||||
SRC = r'''
|
||||
#include <pthread.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include "cpu.h"
|
||||
|
||||
static volatile int step;
|
||||
static uint64_t cell[4] __attribute__((aligned(64)));
|
||||
static int nthreads, iters;
|
||||
static int fails;
|
||||
|
||||
static void wait_step(int s) { while (__atomic_load_n(&step, __ATOMIC_ACQUIRE) != s) {} }
|
||||
static void set_step(int s) { __atomic_store_n(&step, s, __ATOMIC_RELEASE); }
|
||||
|
||||
// Thread B: X -> Y -> X with exclusive pairs of the given width (0 = do nothing, the control).
|
||||
static int aba_width;
|
||||
static void *aba_b(void *arg) {
|
||||
(void)arg;
|
||||
Cpu c = {0};
|
||||
uint64_t a = (uint64_t)(uintptr_t)cell;
|
||||
wait_step(1);
|
||||
for (int k = 0; aba_width && k < 2; k++) {
|
||||
for (;;) {
|
||||
if (aba_width == 32) { uint32_t v = (uint32_t)ldx32(&c, a); if (!stx32(&c, a, v ^ 0x5a5a5a5au)) break; }
|
||||
else if (aba_width == 64) { uint64_t v = ldx64(&c, a); if (!stx64(&c, a, v ^ 0x5a5a5a5a5a5a5a5aull)) break; }
|
||||
else { __uint128_t v = ldxp128(&c, a); if (!stxp128(&c, a, (uint64_t)v ^ 1, (uint64_t)(v >> 64) ^ 2)) break; }
|
||||
}
|
||||
}
|
||||
set_step(2);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void aba_case(int width, int expect_fail) {
|
||||
pthread_t t;
|
||||
Cpu c = {0};
|
||||
uint64_t a = (uint64_t)(uintptr_t)cell;
|
||||
cell[0] = 0x1111222233334444ull;
|
||||
cell[1] = 0x5555666677778888ull;
|
||||
aba_width = width;
|
||||
set_step(0);
|
||||
pthread_create(&t, 0, aba_b, 0);
|
||||
uint32_t st;
|
||||
if (width == 128) {
|
||||
__uint128_t v = ldxp128(&c, a);
|
||||
set_step(1);
|
||||
wait_step(2);
|
||||
st = stxp128(&c, a, (uint64_t)v + 7, (uint64_t)(v >> 64));
|
||||
} else if (width == 32) {
|
||||
uint32_t v = (uint32_t)ldx32(&c, a);
|
||||
set_step(1);
|
||||
wait_step(2);
|
||||
st = stx32(&c, a, v + 7);
|
||||
} else {
|
||||
uint64_t v = ldx64(&c, a);
|
||||
set_step(1);
|
||||
wait_step(2);
|
||||
st = stx64(&c, a, v + 7);
|
||||
}
|
||||
pthread_join(t, 0);
|
||||
int want = expect_fail != 0;
|
||||
printf("aba width %3d: stx status %u, want %d %s\n", width, st, want, st == (uint32_t)want ? "ok" : "FAIL");
|
||||
if (st != (uint32_t)want) fails++;
|
||||
}
|
||||
|
||||
static int inc_width;
|
||||
static void *inc(void *arg) {
|
||||
(void)arg;
|
||||
Cpu c = {0};
|
||||
uint64_t a = (uint64_t)(uintptr_t)cell;
|
||||
for (int i = 0; i < iters; i++) {
|
||||
for (;;) {
|
||||
if (inc_width == 32) { uint32_t v = (uint32_t)ldx32(&c, a); if (!stx32(&c, a, v + 1)) break; }
|
||||
else if (inc_width == 64) { uint64_t v = ldx64(&c, a); if (!stx64(&c, a, v + 1)) break; }
|
||||
else { __uint128_t v = ldxp128(&c, a); if (!stxp128(&c, a, (uint64_t)v + 1, (uint64_t)(v >> 64) + 3)) break; }
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void inc_case(int width) {
|
||||
pthread_t t[64];
|
||||
cell[0] = cell[1] = 0;
|
||||
inc_width = width;
|
||||
for (int i = 0; i < nthreads; i++) pthread_create(&t[i], 0, inc, 0);
|
||||
for (int i = 0; i < nthreads; i++) pthread_join(t[i], 0);
|
||||
uint64_t want = (uint64_t)nthreads * iters;
|
||||
uint64_t got = width == 32 ? (uint32_t)cell[0] : cell[0];
|
||||
int ok = got == (width == 32 ? (uint32_t)want : want) && (width != 128 || cell[1] == 3 * want);
|
||||
printf("contention width %3d: %d threads x %d, counter %llu (hi %llu), want %llu %s\n", width, nthreads, iters,
|
||||
(unsigned long long)got, (unsigned long long)cell[1], (unsigned long long)want, ok ? "ok" : "FAIL");
|
||||
if (!ok) fails++;
|
||||
}
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
nthreads = atoi(argv[1]);
|
||||
iters = atoi(argv[2]);
|
||||
aba_case(0, 0);
|
||||
aba_case(32, 1);
|
||||
aba_case(64, 1);
|
||||
aba_case(128, 1);
|
||||
inc_case(32);
|
||||
inc_case(64);
|
||||
inc_case(128);
|
||||
printf("%s\n", fails ? "FAILED" : "PASSED");
|
||||
return fails != 0;
|
||||
}
|
||||
'''
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument('--runtime', type=Path, default=ROOT / 'runtime', help='directory holding cpu.h')
|
||||
ap.add_argument('--threads', type=int, default=8)
|
||||
ap.add_argument('--iters', type=int, default=200000)
|
||||
args = ap.parse_args()
|
||||
env = dict(os.environ)
|
||||
if os.name == 'nt' and MSYS_BIN.exists():
|
||||
env['PATH'] = str(MSYS_BIN) + os.pathsep + env.get('PATH', '')
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
src = Path(tmp) / 'excl_mt.c'
|
||||
exe = Path(tmp) / ('excl_mt.exe' if os.name == 'nt' else 'excl_mt')
|
||||
src.write_text(SRC)
|
||||
subprocess.check_call([compiler(), '-O2', '-march=x86-64-v3', '-fno-strict-aliasing', '-I', str(args.runtime),
|
||||
str(src), '-o', str(exe), '-lpthread'], env=env)
|
||||
return subprocess.run([str(exe), str(args.threads), str(args.iters)], env=env).returncode
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
111
tests/flags_check.c
Normal file
111
tests/flags_check.c
Normal file
@@ -0,0 +1,111 @@
|
||||
// The function-local flag helpers of runtime/cpu.h (Flags, fc_*, fnz_*, nz_*, adc*f) against the packed helpers
|
||||
// (subs64() and friends, COND_xx), which tests/diff/lift.py checks against Unicorn. Built and run by
|
||||
// tests/flags_check.py; prints the first mismatches and exits 1 on any.
|
||||
#include <stdio.h>
|
||||
|
||||
#include "cpu.h"
|
||||
|
||||
static const uint64_t EDGE[] = {0, 1, 2, 0x7f, 0x80, 0xff, 0x7fff, 0x8000, 0xffff, 0x7fffffff, 0x80000000,
|
||||
0x80000001, 0xfffffffe, 0xffffffff, 0x100000000ull, 0x7fffffffffffffffull,
|
||||
0x8000000000000000ull, 0x8000000000000001ull, 0xfffffffffffffffeull,
|
||||
0xffffffffffffffffull, 0x123456789abcdef0ull, 0xfedcba9876543210ull};
|
||||
#define NEDGE (sizeof EDGE / sizeof EDGE[0])
|
||||
|
||||
static uint64_t rng = 0x9E3779B97F4A7C15ull;
|
||||
static uint64_t next(void) {
|
||||
rng ^= rng << 13, rng ^= rng >> 7, rng ^= rng << 17;
|
||||
return rng;
|
||||
}
|
||||
|
||||
static int fails;
|
||||
static void bad(const char *what, int cc, uint64_t a, uint64_t b, uint64_t got, uint64_t want) {
|
||||
if (fails++ < 20)
|
||||
printf("%s cc %d a %llx b %llx: got %llx want %llx\n", what, cc, (unsigned long long)a, (unsigned long long)b,
|
||||
(unsigned long long)got, (unsigned long long)want);
|
||||
}
|
||||
|
||||
static int cond(const Cpu *c, int cc) {
|
||||
switch (cc) {
|
||||
case CC_EQ: return COND_EQ(c);
|
||||
case CC_NE: return COND_NE(c);
|
||||
case CC_CS: return COND_CS(c);
|
||||
case CC_CC: return COND_CC(c);
|
||||
case CC_MI: return COND_MI(c);
|
||||
case CC_PL: return COND_PL(c);
|
||||
case CC_VS: return COND_VS(c);
|
||||
case CC_VC: return COND_VC(c);
|
||||
case CC_HI: return COND_HI(c);
|
||||
case CC_LS: return COND_LS(c);
|
||||
case CC_GE: return COND_GE(c);
|
||||
case CC_LT: return COND_LT(c);
|
||||
case CC_GT: return COND_GT(c);
|
||||
default: return COND_LE(c);
|
||||
}
|
||||
}
|
||||
|
||||
// The condition argument is not constant here, so this also covers the switch default paths.
|
||||
#define CHECK(NAME, OLD, NEW, FC, FNZ) \
|
||||
do { \
|
||||
Cpu c = {0}; \
|
||||
Flags f = {0}; \
|
||||
uint64_t ro = OLD, rn = NEW; \
|
||||
if (ro != rn) bad(NAME " result", -1, a, b, rn, ro); \
|
||||
for (int cc = 0; cc < 14; cc++) \
|
||||
if (!FC(&f, cc) != !cond(&c, cc)) bad(NAME, cc, a, b, FC(&f, cc), cond(&c, cc)); \
|
||||
f.nz = 0; \
|
||||
if (FNZ(&f) != c.nzcv) bad(NAME " packed", -1, a, b, FNZ(&f), c.nzcv); \
|
||||
Flags g = {.nz = c.nzcv}; \
|
||||
for (int cc = 0; cc < 14; cc++) \
|
||||
if (!fc_nz(&g, cc) != !cond(&c, cc)) bad(NAME " nz", cc, a, b, fc_nz(&g, cc), cond(&c, cc)); \
|
||||
} while (0)
|
||||
|
||||
static void one(uint64_t a, uint64_t b) {
|
||||
uint32_t a32 = (uint32_t)a, b32 = (uint32_t)b;
|
||||
CHECK("sub64", subs64(&c, a, b), fsub64(&f, a, b), fc_sub64, fnz_sub64);
|
||||
CHECK("sub32", subs32(&c, a32, b32), fsub32(&f, a32, b32), fc_sub32, fnz_sub32);
|
||||
CHECK("add64", adds64(&c, a, b), fadd64(&f, a, b), fc_add64, fnz_add64);
|
||||
CHECK("add32", adds32(&c, a32, b32), fadd32(&f, a32, b32), fc_add32, fnz_add32);
|
||||
CHECK("logic64", logic64(&c, a & b), flogic64(&f, a & b), fc_logic64, fnz_logic64);
|
||||
CHECK("logic32", logic32(&c, a32 & b32), flogic32(&f, a32 & b32), fc_logic32, fnz_logic32);
|
||||
{
|
||||
Cpu c = {0};
|
||||
if (nz_sub64(a, b) != (subs64(&c, a, b), c.nzcv)) bad("nz_sub64", -1, a, b, nz_sub64(a, b), c.nzcv);
|
||||
if (nz_sub32(a32, b32) != (subs32(&c, a32, b32), c.nzcv)) bad("nz_sub32", -1, a, b, nz_sub32(a32, b32), c.nzcv);
|
||||
if (nz_add64(a, b) != (adds64(&c, a, b), c.nzcv)) bad("nz_add64", -1, a, b, nz_add64(a, b), c.nzcv);
|
||||
if (nz_add32(a32, b32) != (adds32(&c, a32, b32), c.nzcv)) bad("nz_add32", -1, a, b, nz_add32(a32, b32), c.nzcv);
|
||||
}
|
||||
for (int cin = 0; cin < 2; cin++) {
|
||||
Cpu c = {.nzcv = (uint64_t)cin << 29};
|
||||
Flags f = {0};
|
||||
if (adc64f(cin, a, b) != adc64(&c, a, b)) bad("adc64", cin, a, b, adc64f(cin, a, b), adc64(&c, a, b));
|
||||
if (sbc64f(cin, a, b) != sbc64(&c, a, b)) bad("sbc64", cin, a, b, sbc64f(cin, a, b), sbc64(&c, a, b));
|
||||
if (adc32f(cin, a32, b32) != adc32(&c, a32, b32)) bad("adc32", cin, a, b, adc32f(cin, a32, b32), adc32(&c, a32, b32));
|
||||
if (sbc32f(cin, a32, b32) != sbc32(&c, a32, b32)) bad("sbc32", cin, a, b, sbc32f(cin, a32, b32), sbc32(&c, a32, b32));
|
||||
uint64_t r;
|
||||
#define CARRY(NAME, NEW, OLD) \
|
||||
c.nzcv = (uint64_t)cin << 29; \
|
||||
r = NEW; \
|
||||
if (r != OLD || f.nz != c.nzcv) bad(NAME, cin, a, b, f.nz, c.nzcv);
|
||||
CARRY("adcs64", adcs64f(&f, cin, a, b), adcs64(&c, a, b))
|
||||
CARRY("sbcs64", sbcs64f(&f, cin, a, b), sbcs64(&c, a, b))
|
||||
CARRY("adcs32", adcs32f(&f, cin, a32, b32), adcs32(&c, a32, b32))
|
||||
CARRY("sbcs32", sbcs32f(&f, cin, a32, b32), sbcs32(&c, a32, b32))
|
||||
}
|
||||
}
|
||||
|
||||
int main(void) {
|
||||
long n = 0;
|
||||
for (size_t i = 0; i < NEDGE; i++)
|
||||
for (size_t j = 0; j < NEDGE; j++, n++) one(EDGE[i], EDGE[j]);
|
||||
for (int k = 0; k < 2000000; k++, n++) {
|
||||
uint64_t a = next(), b = next();
|
||||
switch (k & 3) {
|
||||
case 1: b = a; break; // equal operands
|
||||
case 2: b = a + (next() & 3) - 1; break; // neighbours
|
||||
case 3: a &= 0xff, b &= 0xff; break; // small values
|
||||
}
|
||||
one(a, b);
|
||||
}
|
||||
printf("%ld operand pairs, %d mismatches\n", n, fails);
|
||||
return fails != 0;
|
||||
}
|
||||
32
tests/flags_check.py
Normal file
32
tests/flags_check.py
Normal file
@@ -0,0 +1,32 @@
|
||||
"""Build and run tests/flags_check.c: runtime/cpu.h's function-local flag helpers against its packed flag helpers.
|
||||
|
||||
Usage: flags_check.py
|
||||
"""
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests' / 'diff'))
|
||||
from common import MSYS_BIN, compiler # noqa: E402
|
||||
|
||||
|
||||
def main():
|
||||
work = ROOT / 'build' / 'test' / 'flags'
|
||||
work.mkdir(parents=True, exist_ok=True)
|
||||
exe = work / ('flags_check.exe' if os.name == 'nt' else 'flags_check')
|
||||
env = dict(os.environ)
|
||||
if os.name == 'nt' and MSYS_BIN.exists():
|
||||
env['PATH'] = str(MSYS_BIN) + os.pathsep + env.get('PATH', '')
|
||||
for opt in ('-O0', '-O1'):
|
||||
subprocess.check_call([compiler(), opt, '-fno-strict-aliasing', '-I', str(ROOT / 'runtime'),
|
||||
str(ROOT / 'tests' / 'flags_check.c'), '-o', str(exe)], env=env)
|
||||
print(opt, end=': ', flush=True)
|
||||
if subprocess.run([str(exe)], env=env).returncode:
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
79
tests/installer_e2e.py
Normal file
79
tests/installer_e2e.py
Normal file
@@ -0,0 +1,79 @@
|
||||
"""End-to-end run of the installer in dev mode with the game files named in config.local.toml.
|
||||
|
||||
Reads input.base, the 1.2.1 entry of input.updates (the one build/extracted/v121 was extracted from, else the only
|
||||
one) and input.keys, and passes them to `python -m installer` started the way the release launcher starts it: the
|
||||
embeddable Python (build/py314-embed) with this checkout on sys.path. The paths are never printed. Events go to
|
||||
<events> (JSON lines); stdout gets each step's state and time, warnings and errors.
|
||||
|
||||
python tests/installer_e2e.py --dest build/install-test --lang en-US [--events FILE] [--no-inputs] [-- extra args]
|
||||
|
||||
--no-inputs leaves the game files out (update mode on an existing folder).
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
EMBED = ROOT / 'build' / 'py314-embed' / 'py' / 'python.exe'
|
||||
BOOT = ('import runpy, sys; sys.path.insert(0, sys.argv.pop(1)); '
|
||||
"runpy.run_module('installer', run_name='__main__', alter_sys=True)")
|
||||
|
||||
|
||||
def game_inputs():
|
||||
config = tomllib.loads((ROOT / 'config.local.toml').read_text(encoding='utf-8'))['input']
|
||||
updates = [Path(u) for u in config.get('updates', [])]
|
||||
manifest = ROOT / 'build' / 'extracted' / 'v121' / 'extraction-manifest.json'
|
||||
update = None
|
||||
if manifest.is_file():
|
||||
source = json.loads(manifest.read_text())['source']
|
||||
update = next((u for u in updates if str(u.resolve()) == source), None)
|
||||
if update is None and len(updates) == 1:
|
||||
update = updates[0]
|
||||
if update is None:
|
||||
sys.exit('installer_e2e: cannot tell which input.updates entry is 1.2.1')
|
||||
return ['--base', config['base'], '--update', str(update), '--keys', config['keys']]
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument('--dest', type=Path, required=True)
|
||||
ap.add_argument('--lang')
|
||||
ap.add_argument('--events', type=Path)
|
||||
ap.add_argument('--no-inputs', action='store_true')
|
||||
ap.add_argument('extra', nargs='*')
|
||||
a = ap.parse_args()
|
||||
python = EMBED if EMBED.is_file() else Path(sys.executable)
|
||||
command = [str(python), '-c', BOOT, str(ROOT), '--dest', str(a.dest.resolve())]
|
||||
if a.lang:
|
||||
command += ['--lang', a.lang]
|
||||
if not a.no_inputs:
|
||||
command += game_inputs()
|
||||
command += a.extra
|
||||
events = a.events or a.dest.resolve().parent / f'{a.dest.name}-events.jsonl'
|
||||
with open(events, 'w', encoding='utf-8') as log:
|
||||
proc = subprocess.Popen(command, stdout=subprocess.PIPE, text=True, encoding='utf-8', cwd=ROOT)
|
||||
for line in proc.stdout:
|
||||
log.write(line)
|
||||
log.flush()
|
||||
try:
|
||||
e = json.loads(line)
|
||||
except ValueError:
|
||||
print('non-JSON line on stdout:', line.rstrip())
|
||||
continue
|
||||
if e['type'] == 'step' and e['state'] != 'started':
|
||||
extra = f'{e["seconds"]:.1f} s' if 'seconds' in e else e.get('reason', '')
|
||||
print(f'[{e["t"]:8.1f}] {e["step"]:8} {e["state"]} {extra}', flush=True)
|
||||
elif e['type'] == 'step':
|
||||
print(f'[{e["t"]:8.1f}] {e["step"]:8} started', flush=True)
|
||||
elif e['type'] in ('warning', 'error', 'plan', 'end'):
|
||||
print(f'[{e["t"]:8.1f}] {json.dumps({k: v for k, v in e.items() if k != "t"})}', flush=True)
|
||||
elif e['type'] == 'start':
|
||||
print(f'[{e["t"]:8.1f}] start mode={e["mode"]} release={e["release"]}', flush=True)
|
||||
return proc.wait()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
451
tests/installer_gui_test.py
Normal file
451
tests/installer_gui_test.py
Normal file
@@ -0,0 +1,451 @@
|
||||
"""The installer window: event model, form, catalogs, cancel, a self-closing Tk run and one end-to-end run.
|
||||
|
||||
.venv-py314\\Scripts\\python tests\\installer_gui_test.py
|
||||
|
||||
The end-to-end test runs the real CLI in --verify mode on a finished install (every step skips) and folds its events
|
||||
through the same code the window uses (the fixture's step keys are recomputed for the checkout under test, because the keys fingerprint recomp/, tools/ and runtime/). The install is build/inst-gui/e2e-install: hard links to build/install-test, so
|
||||
nothing there is written, except the state file and nx2navi.ini, which a run rewrites and which are copied. The run hashes
|
||||
the 17 GB RomFS; it is skipped when the hash differs from the expected one in all three tries. On the developer machine
|
||||
(RAM with a bad bit) a buffered read of a file sometimes returns flipped bits from the page cache while the file on
|
||||
disk is intact (an unbuffered read of install-test's romfs.bin gives the expected hash), so such a skip is the machine,
|
||||
not the installer. It is also skipped when build/install-test is absent.
|
||||
"""
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import queue
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(ROOT))
|
||||
from installer import layout as layouts # noqa: E402
|
||||
from installer import state as statefile # noqa: E402
|
||||
from installer.events import Reporter # noqa: E402
|
||||
from installer.runner import Context # noqa: E402
|
||||
from installer.steps import Fonts, all_steps # noqa: E402
|
||||
from installer.guilogic import (DONE, FAILED, PENDING, RUNNING, SKIPPED, UI_LANGUAGE_LABELS, # noqa: E402
|
||||
UI_LANGUAGES, Catalog, Form, Model, Process, amount, default_game_language,
|
||||
describe_event, parse_line, ui_language_for_langid, windows_ui_language)
|
||||
|
||||
INSTALLED = ROOT / 'build' / 'install-test'
|
||||
FIXTURE = ROOT / 'build' / 'inst-gui' / 'e2e-install'
|
||||
COPIED = ('nx2navi.ini', statefile.NAME) # written by a run, so never linked
|
||||
START = {'type': 'start', 'release': 'r1', 'mode': 'install', 'dest': 'D:\\T', 'log': 'D:\\T.log',
|
||||
'steps': ['check', 'fonts', 'extract', 'lift', 'compile', 'prepare', 'cleanup']}
|
||||
|
||||
|
||||
def fold(*events):
|
||||
m = Model()
|
||||
for e in events:
|
||||
m.apply(e)
|
||||
return m
|
||||
|
||||
|
||||
class ParseTest(unittest.TestCase):
|
||||
def test_event_line(self):
|
||||
self.assertEqual(parse_line('{"type": "step", "step": "check", "state": "started"}\n')['step'], 'check')
|
||||
|
||||
def test_noise_is_not_an_event(self):
|
||||
for line in ('', 'Traceback (most recent call last):', '{broken', '[1, 2]', '{"no_type": 1}', '{"type": 3}'):
|
||||
self.assertIsNone(parse_line(line), line)
|
||||
|
||||
def test_unicode_survives(self):
|
||||
e = parse_line(json.dumps({'type': 'warning', 'id': 'x', 'params': {'p': 'Привет'}}, ensure_ascii=False))
|
||||
self.assertEqual(e['params']['p'], 'Привет')
|
||||
|
||||
|
||||
class ModelTest(unittest.TestCase):
|
||||
def test_steps_follow_the_events(self):
|
||||
m = fold(START, {'type': 'plan', 'run': ['check', 'extract'], 'skip': {'lift': 'current', 'compile': 'current'}},
|
||||
{'type': 'step', 'step': 'check', 'state': 'started'})
|
||||
self.assertEqual([m.states[s] for s in m.steps],
|
||||
[RUNNING, PENDING, PENDING, SKIPPED, SKIPPED, PENDING, PENDING])
|
||||
self.assertEqual(m.current, 'check')
|
||||
m.apply({'type': 'step', 'step': 'check', 'state': 'done', 'seconds': 1.0})
|
||||
self.assertEqual(m.states['check'], DONE)
|
||||
self.assertIsNone(m.current)
|
||||
m.apply({'type': 'step', 'step': 'extract', 'state': 'skipped', 'reason': 'current'})
|
||||
self.assertEqual(m.states['extract'], SKIPPED)
|
||||
|
||||
def test_progress_fraction(self):
|
||||
m = fold(START, {'type': 'step', 'step': 'extract', 'state': 'started'},
|
||||
{'type': 'progress', 'step': 'extract', 'phase': 'copy', 'done': 5, 'total': 20, 'unit': 'bytes'})
|
||||
self.assertEqual((m.current, m.phase, m.fraction()), ('extract', 'copy', 0.25))
|
||||
m.apply({'type': 'progress', 'step': 'extract', 'phase': 'merge'})
|
||||
self.assertIsNone(m.fraction()) # a phase without size is indeterminate
|
||||
m.apply({'type': 'progress', 'step': 'extract', 'phase': 'copy', 'done': 30, 'total': 20})
|
||||
self.assertEqual(m.fraction(), 1.0)
|
||||
|
||||
def test_a_new_step_resets_progress(self):
|
||||
m = fold(START, {'type': 'progress', 'step': 'check', 'phase': 'inputs', 'done': 1, 'total': 2},
|
||||
{'type': 'step', 'step': 'extract', 'state': 'started'})
|
||||
self.assertIsNone(m.phase)
|
||||
self.assertIsNone(m.fraction())
|
||||
|
||||
def test_error_marks_the_step(self):
|
||||
m = fold(START, {'type': 'step', 'step': 'check', 'state': 'started'},
|
||||
{'type': 'error', 'id': 'check.disk_space', 'step': 'check', 'params': {'path': 'D:\\T'}},
|
||||
{'type': 'end', 'result': 'failed', 'seconds': 2})
|
||||
self.assertEqual(m.states['check'], FAILED)
|
||||
self.assertEqual(m.error[0], 'check.disk_space')
|
||||
self.assertEqual(m.result, 'failed')
|
||||
|
||||
def test_space_text(self):
|
||||
m = fold({'type': 'space', 'dest': 'D:\\T', 'need': 36_400_000_000, 'free': 100_000_000_000, 'temp': 'C:\\t',
|
||||
'temp_need': 0, 'temp_free': 5, 'same_disk': True})
|
||||
text = m.need_text(Catalog('en'))
|
||||
self.assertIn('36.4', text)
|
||||
self.assertIn('100.0', text)
|
||||
self.assertEqual(Model().need_text(Catalog('en')), '')
|
||||
m = fold({'type': 'space', 'dest': 'D:\\T', 'need': 20_000_000_000, 'free': 1, 'temp': 'C:\\t',
|
||||
'temp_need': 2_000_000_000, 'temp_free': 3, 'same_disk': False})
|
||||
self.assertIn('C:\\t', m.need_text(Catalog('en')))
|
||||
|
||||
def test_warnings_collect(self):
|
||||
m = fold(START, {'type': 'warning', 'id': 'warn.lang_needs_fonts', 'params': {'lang': 'ko'}})
|
||||
self.assertEqual(m.warnings, [('warn.lang_needs_fonts', {'lang': 'ko'})])
|
||||
|
||||
def test_finish_without_end_event(self):
|
||||
m = fold(START, {'type': 'step', 'step': 'lift', 'state': 'started'})
|
||||
m.finish(-1073741510)
|
||||
self.assertEqual(m.result, 'failed')
|
||||
self.assertEqual(m.states['lift'], FAILED)
|
||||
self.assertEqual(m.error[0], 'ui.process_died')
|
||||
|
||||
def test_finish_after_cancel(self):
|
||||
m = fold(START, {'type': 'step', 'step': 'lift', 'state': 'started'}, {'type': 'end', 'result': 'cancelled'})
|
||||
m.finish(3)
|
||||
self.assertEqual(m.result, 'cancelled')
|
||||
self.assertIsNone(m.error)
|
||||
self.assertEqual(m.states['lift'], PENDING) # not finished; the next run resumes here
|
||||
|
||||
def test_exit_code_3_alone_is_cancelled(self):
|
||||
m = Model()
|
||||
m.finish(3)
|
||||
self.assertEqual(m.result, 'cancelled')
|
||||
|
||||
def test_unknown_event_is_ignored(self):
|
||||
m = fold(START, {'type': 'future_thing', 'x': 1})
|
||||
self.assertEqual(m.states['check'], PENDING)
|
||||
|
||||
|
||||
class RunStatesTest(unittest.TestCase):
|
||||
OK = [START, {'type': 'plan', 'run': ['check', 'extract'], 'skip': {'fonts': 'current', 'lift': 'current',
|
||||
'compile': 'current', 'prepare': 'current',
|
||||
'cleanup': 'not_needed'}},
|
||||
{'type': 'step', 'step': 'check', 'state': 'started'}, {'type': 'step', 'step': 'check', 'state': 'done'},
|
||||
{'type': 'step', 'step': 'extract', 'state': 'started'}, {'type': 'step', 'step': 'extract', 'state': 'done'},
|
||||
{'type': 'step', 'step': 'fonts', 'state': 'skipped'},
|
||||
{'type': 'step', 'step': 'lift', 'state': 'skipped'}, {'type': 'step', 'step': 'compile', 'state': 'skipped'},
|
||||
{'type': 'step', 'step': 'prepare', 'state': 'skipped'}, {'type': 'step', 'step': 'cleanup', 'state': 'done'},
|
||||
{'type': 'end', 'result': 'ok', 'seconds': 1}]
|
||||
FAILED = [START, {'type': 'step', 'step': 'check', 'state': 'started'}, {'type': 'step', 'step': 'check', 'state': 'done'},
|
||||
{'type': 'step', 'step': 'extract', 'state': 'started'},
|
||||
{'type': 'error', 'id': 'extract.romfs_mismatch', 'step': 'extract', 'params': {}},
|
||||
{'type': 'warning', 'id': 'warn.lang_needs_fonts', 'params': {'lang': 'ko'}},
|
||||
{'type': 'end', 'result': 'failed', 'seconds': 1}]
|
||||
|
||||
def test_ok_run_has_only_done_and_skipped_steps(self):
|
||||
m = fold(*self.OK)
|
||||
m.finish(0)
|
||||
self.assertEqual(m.result, 'ok')
|
||||
self.assertIsNone(m.error)
|
||||
self.assertEqual({m.states[s] for s in m.steps}, {DONE, SKIPPED})
|
||||
|
||||
def test_a_new_run_keeps_nothing_of_the_failed_one(self):
|
||||
m = fold(*self.FAILED)
|
||||
m.finish(1)
|
||||
self.assertEqual(m.states['extract'], FAILED)
|
||||
for e in self.OK: # the same model object fed a second run (the window makes a new one, but even then)
|
||||
m.apply(e)
|
||||
m.finish(0)
|
||||
self.assertEqual({m.states[s] for s in m.steps}, {DONE, SKIPPED})
|
||||
self.assertIsNone(m.error)
|
||||
self.assertEqual(m.warnings, [])
|
||||
self.assertEqual(m.result, 'ok')
|
||||
|
||||
|
||||
class FormTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.dir = Path(self.tmp.name)
|
||||
for name in ('base.nsp', 'upd.nsp', 'prod.keys'):
|
||||
(self.dir / name).write_bytes(b'x')
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def form(self, **kw):
|
||||
values = dict(base=str(self.dir / 'base.nsp'), update=str(self.dir / 'upd.nsp'), keys=str(self.dir / 'prod.keys'),
|
||||
lang='en-US', dest=str(self.dir / 'out'))
|
||||
values.update(kw)
|
||||
return Form(**values)
|
||||
|
||||
def test_complete_form(self):
|
||||
self.assertEqual(self.form().problems(), [])
|
||||
|
||||
def test_empty_form(self):
|
||||
self.assertEqual(Form().problems(), ['ui.need_dest', 'ui.need_base', 'ui.need_update', 'ui.need_keys',
|
||||
'ui.need_lang'])
|
||||
|
||||
def test_missing_files(self):
|
||||
self.assertEqual(self.form(base=str(self.dir / 'no.nsp'), keys=str(self.dir / 'no.keys')).problems(),
|
||||
['ui.missing_base', 'ui.missing_keys'])
|
||||
|
||||
def test_unknown_language(self):
|
||||
self.assertEqual(self.form(lang='xx').problems(), ['ui.need_lang'])
|
||||
|
||||
def test_firmware_folder(self):
|
||||
firmware = self.dir / 'firmware'
|
||||
firmware.mkdir()
|
||||
self.assertEqual(self.form(firmware=str(firmware)).problems(), [])
|
||||
self.assertEqual(self.form(firmware=str(self.dir / 'no-firmware')).problems(), ['ui.missing_firmware'])
|
||||
|
||||
def test_command(self):
|
||||
firmware = self.dir / 'firmware'
|
||||
firmware.mkdir()
|
||||
cmd = self.form(firmware=str(firmware)).command('py.exe', root=ROOT, temp=self.dir / 't', verify_hashes=True)
|
||||
self.assertEqual(cmd[:3], ['py.exe', '-m', 'installer'])
|
||||
for flag in ('--base', '--update', '--keys', '--firmware', '--lang', '--dest', '--root', '--temp', '--verify'):
|
||||
self.assertIn(flag, cmd)
|
||||
self.assertEqual(cmd[cmd.index('--firmware') + 1], str(firmware))
|
||||
self.assertEqual(cmd[cmd.index('--lang') + 1], 'en-US')
|
||||
|
||||
def test_installed_folder_needs_only_itself(self):
|
||||
out = self.dir / 'out'
|
||||
statefile.save(self._mkdir(out), statefile.empty('r0') | {'lang': 'ru'})
|
||||
form = Form(dest=str(out))
|
||||
self.assertTrue(form.update_mode())
|
||||
self.assertEqual(form.problems(), [])
|
||||
cmd = form.command('py.exe')
|
||||
self.assertEqual(cmd, ['py.exe', '-m', 'installer', '--dest', str(out)])
|
||||
|
||||
def test_installed_folder_still_checks_given_files(self):
|
||||
out = self.dir / 'out'
|
||||
statefile.save(self._mkdir(out), statefile.empty('r0'))
|
||||
self.assertEqual(Form(dest=str(out), base=str(self.dir / 'no.nsp')).problems(), ['ui.missing_base'])
|
||||
|
||||
def test_broken_state_is_a_new_install(self):
|
||||
out = self._mkdir(self.dir / 'out')
|
||||
(out / statefile.NAME).write_text('{nope', encoding='utf-8')
|
||||
self.assertFalse(Form(dest=str(out)).update_mode())
|
||||
|
||||
@staticmethod
|
||||
def _mkdir(path):
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
return path
|
||||
|
||||
|
||||
class CatalogTest(unittest.TestCase):
|
||||
def test_every_language_covers_english_with_the_same_placeholders(self):
|
||||
en = Catalog('en').english
|
||||
for lang in UI_LANGUAGES[1:]:
|
||||
with self.subTest(lang=lang):
|
||||
other = Catalog(lang).texts
|
||||
self.assertEqual(sorted(set(en) - set(other)), [])
|
||||
self.assertEqual(sorted(set(other) - set(en)), [])
|
||||
holes = [k for k in en
|
||||
if sorted(re.findall(r'\{\w+\}', en[k])) != sorted(re.findall(r'\{\w+\}', other[k]))]
|
||||
self.assertEqual(holes, [])
|
||||
|
||||
def test_language_names_tell_the_codes_apart(self):
|
||||
# gui.App.selected_lang maps the shown name back to its code across every window language
|
||||
names = {}
|
||||
for lang in UI_LANGUAGES:
|
||||
cat = Catalog(lang)
|
||||
for id in cat.english:
|
||||
if id.startswith('lang.'):
|
||||
names.setdefault(cat(id), set()).add(id)
|
||||
self.assertEqual({n: ids for n, ids in names.items() if len(ids) > 1}, {})
|
||||
self.assertEqual(sorted(UI_LANGUAGE_LABELS), sorted(UI_LANGUAGES))
|
||||
|
||||
def test_every_id_the_window_uses_exists(self):
|
||||
cat = Catalog('en')
|
||||
ids = ['ui.' + n for n in ('title', 'start', 'cancel', 'open_folder', 'notice', 'space_one', 'space_two',
|
||||
'process_died', 'result_ok', 'result_failed', 'result_cancelled')]
|
||||
ids += [f'step.{s}' for s in ('check', 'fonts', 'extract', 'lift', 'compile', 'prepare', 'cleanup')]
|
||||
ids += ['ui.state_' + n for n in ('started', 'done', 'skipped', 'pending', 'running', 'failed')]
|
||||
for lang in UI_LANGUAGES:
|
||||
cat.set_language(lang)
|
||||
self.assertEqual(cat.lang, lang)
|
||||
self.assertEqual([i for i in ids if not cat.has(i)], [])
|
||||
|
||||
def test_codes_become_names(self):
|
||||
for lang, korean in (('en', 'Korean'), ('ru', 'Корейский'), ('pt-BR', 'Coreano'), ('zh-Hans', '韩语')):
|
||||
cat = Catalog(lang)
|
||||
self.assertIn(korean, cat('warn.lang_needs_fonts', lang='ko'))
|
||||
self.assertNotIn(' ko ', cat('warn.lang_needs_fonts', lang='ko'))
|
||||
self.assertNotIn('extract', cat('state.output_changed', step='extract'))
|
||||
for word in ('base', 'update', 'keys'):
|
||||
self.assertIn(cat('param.input.' + word), cat('check.keys_wrong', input=word))
|
||||
self.assertNotIn('en-US', cat('check.lang_unknown', lang='xx', known='ja, en-US'))
|
||||
self.assertEqual(Catalog('en')('check.lang_unknown', lang='xx', known='ja, en-US'),
|
||||
"Unknown language 'xx'. Known: Japanese, English (US).")
|
||||
|
||||
def test_fallback_and_params(self):
|
||||
cat = Catalog('ru')
|
||||
self.assertIn('36.4', cat('ui.log_space', need='36.4', free='1.0'))
|
||||
self.assertEqual(cat('no.such.id'), 'no.such.id')
|
||||
self.assertIn('a, b', cat('check.release_incomplete', paths=['a', 'b']))
|
||||
cat.set_language('de') # unknown language falls back to English
|
||||
self.assertEqual(cat.lang, 'en')
|
||||
|
||||
def test_describe_and_amount(self):
|
||||
cat = Catalog('en')
|
||||
self.assertEqual(amount(cat, 6e9, 16e9, 'bytes'), '6.0 / 16.0 GB')
|
||||
self.assertEqual(amount(cat, 1, 9, 'shaders'), '1 / 9 shaders')
|
||||
self.assertEqual(amount(cat, 1, 9, 'widgets'), '1 / 9 widgets')
|
||||
line = describe_event(cat, {'type': 'step', 'step': 'check', 'state': 'done', 'seconds': 1.5})
|
||||
self.assertIn('1.5 s', line)
|
||||
self.assertEqual(describe_event(cat, {'type': 'start'}), '')
|
||||
|
||||
def test_windows_language_is_a_known_value(self):
|
||||
self.assertIn(windows_ui_language(), UI_LANGUAGES)
|
||||
|
||||
def test_windows_langid_picks_the_window_language(self):
|
||||
for langid, lang in ((0x0419, 'ru'), (0x0819, 'ru'), (0x0416, 'pt-BR'), (0x0816, 'en'),
|
||||
(0x0804, 'zh-Hans'), (0x1004, 'zh-Hans'),
|
||||
(0x0404, 'en'), (0x0c04, 'en'), (0x1404, 'en'), (0x0409, 'en'), (0x0411, 'en')):
|
||||
self.assertEqual(ui_language_for_langid(langid), lang, hex(langid))
|
||||
|
||||
def test_chinese_window_starts_with_a_game_language_that_needs_no_shared_fonts(self):
|
||||
self.assertEqual(default_game_language('zh-Hans'), 'en-US')
|
||||
self.assertEqual(default_game_language('ru'), 'ru')
|
||||
|
||||
|
||||
BREAK_CHILD = 'import time\nprint("ready", flush=True)\ntime.sleep(60)\n'
|
||||
|
||||
|
||||
class CancelTest(unittest.TestCase):
|
||||
def test_ctrl_break_ends_the_process_group(self):
|
||||
p = Process([sys.executable, '-c', BREAK_CHILD])
|
||||
try:
|
||||
self.assertEqual(p.lines.get(timeout=20).strip(), 'ready')
|
||||
t0 = time.monotonic()
|
||||
p.cancel()
|
||||
code = None
|
||||
while code is None and time.monotonic() - t0 < 20:
|
||||
code = p.poll()
|
||||
time.sleep(0.05)
|
||||
self.assertIsNotNone(code, 'the child ignored Ctrl+Break')
|
||||
self.assertNotEqual(code, 0)
|
||||
finally:
|
||||
p.kill()
|
||||
|
||||
|
||||
class SmokeTest(unittest.TestCase):
|
||||
def test_every_screen_in_both_languages(self):
|
||||
try:
|
||||
import tkinter # noqa: F401
|
||||
except ImportError:
|
||||
self.skipTest('no tkinter')
|
||||
r = subprocess.run([sys.executable, '-m', 'installer.gui', '--selftest'], cwd=ROOT, timeout=120,
|
||||
capture_output=True, text=True)
|
||||
self.assertEqual(r.returncode, 0, r.stderr[-2000:])
|
||||
|
||||
|
||||
def make_fixture():
|
||||
"""build/inst-gui/e2e-install: links to install-test (see the module docstring), with the step keys of the state
|
||||
set to the ones this checkout computes, so the install counts as current for it."""
|
||||
if not (FIXTURE / statefile.NAME).is_file():
|
||||
shutil.rmtree(FIXTURE, ignore_errors=True)
|
||||
for folder, _dirs, files in os.walk(INSTALLED):
|
||||
target = FIXTURE / Path(folder).relative_to(INSTALLED)
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
for name in files:
|
||||
src, dst = Path(folder) / name, target / name
|
||||
if name in COPIED and target == FIXTURE:
|
||||
shutil.copy2(src, dst)
|
||||
else:
|
||||
os.link(src, dst)
|
||||
layout = layouts.find(ROOT)
|
||||
state = statefile.load(FIXTURE)
|
||||
options = SimpleNamespace(base=None, update=None, keys=None, lang=None)
|
||||
ctx = Context(layout, FIXTURE, ROOT / 'build' / 'inst-gui' / 'e2e-temp', Reporter(out=io.StringIO()), options, state=state)
|
||||
ctx.steps = all_steps()
|
||||
ctx.plan()
|
||||
# An install made before the fonts step has no record for it: record it as current when its output is (the
|
||||
# languages that need the fonts have them in shared-fonts/).
|
||||
if Fonts().current(ctx, {}):
|
||||
state['steps'].setdefault('fonts', {'release': state.get('release', ''), 'key': ctx.keys['fonts']})
|
||||
for step_id, record in state['steps'].items():
|
||||
record['key'] = ctx.keys[step_id]
|
||||
statefile.save(FIXTURE, state)
|
||||
return FIXTURE
|
||||
|
||||
|
||||
@unittest.skipUnless((INSTALLED / statefile.NAME).is_file(), 'build/install-test is absent')
|
||||
class EndToEndTest(unittest.TestCase):
|
||||
@staticmethod
|
||||
def run_cli(form, temp, tries=1):
|
||||
"""Runs the CLI in --verify mode through Process, up to `tries` times while the RomFS hash reads wrong."""
|
||||
for attempt in range(tries):
|
||||
p = Process(form.command(sys.executable, root=ROOT, temp=temp, verify_hashes=True), cwd=ROOT)
|
||||
model, stray = Model(), []
|
||||
t0, code = time.monotonic(), None
|
||||
while code is None and time.monotonic() - t0 < 600:
|
||||
try:
|
||||
line = p.lines.get(timeout=0.2)
|
||||
except queue.Empty:
|
||||
line = ''
|
||||
if line:
|
||||
event = parse_line(line)
|
||||
if event:
|
||||
model.apply(event)
|
||||
elif line.strip():
|
||||
stray.append(line)
|
||||
code = p.poll()
|
||||
p.kill()
|
||||
model.finish(code)
|
||||
if not any(w[0] == 'state.output_changed' for w in model.warnings):
|
||||
break
|
||||
return code, model, stray, time.monotonic() - t0
|
||||
|
||||
def test_verify_run_skips_every_step(self):
|
||||
form = Form(dest=str(make_fixture()))
|
||||
self.assertTrue(form.update_mode())
|
||||
code, model, stray, elapsed = self.run_cli(form, ROOT / 'build' / 'inst-gui' / 'e2e-temp', tries=3)
|
||||
print(f'\nend-to-end: exit {code}, result {model.result}, {elapsed:.1f} s, states {model.states}')
|
||||
if any(w[0] == 'state.output_changed' for w in model.warnings):
|
||||
self.skipTest('the RomFS hash read wrong three times (page cache of the faulty RAM, see the module docstring)')
|
||||
self.assertEqual(code, 0, stray[-5:] or model.error)
|
||||
self.assertEqual(model.result, 'ok')
|
||||
self.assertIn(model.mode, ('resume', 'update'))
|
||||
self.assertEqual({s: model.states[s] for s in model.steps if s not in ('check', 'cleanup')},
|
||||
{s: SKIPPED for s in model.steps if s not in ('check', 'cleanup')})
|
||||
self.assertIsNotNone(model.space)
|
||||
self.assertEqual(stray, [])
|
||||
|
||||
def test_cancel_during_the_check(self):
|
||||
cmd = Form(dest=str(make_fixture())).command(sys.executable, root=ROOT, temp=ROOT / 'build' / 'inst-gui' / 'e2e-temp',
|
||||
verify_hashes=True)
|
||||
p = Process(cmd, cwd=ROOT)
|
||||
model, t0, code, cancelled = Model(), time.monotonic(), None, False
|
||||
while code is None and time.monotonic() - t0 < 120:
|
||||
try:
|
||||
event = parse_line(p.lines.get(timeout=0.2))
|
||||
except queue.Empty:
|
||||
event = None
|
||||
if event:
|
||||
model.apply(event)
|
||||
if event.get('phase') == 'verify_romfs' and not cancelled:
|
||||
cancelled = True
|
||||
p.cancel()
|
||||
code = p.poll()
|
||||
p.kill()
|
||||
model.finish(code)
|
||||
self.assertTrue(cancelled)
|
||||
self.assertEqual((code, model.result), (3, 'cancelled'))
|
||||
self.assertIsNone(model.error)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main(verbosity=2)
|
||||
655
tests/installer_test.py
Normal file
655
tests/installer_test.py
Normal file
@@ -0,0 +1,655 @@
|
||||
"""installer/ bookkeeping with fake steps: fresh install, resume after a failure or cancel, transient steps, update
|
||||
with a newer release, nx2navi.ini and the event stream. No game files, no tools; stdlib only.
|
||||
|
||||
Run: python tests/installer_test.py
|
||||
"""
|
||||
import http.client
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest import mock
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(ROOT))
|
||||
from installer import ini, runner, state, steps # noqa: E402
|
||||
from installer.events import Reporter # noqa: E402
|
||||
from installer.layout import Layout # noqa: E402
|
||||
|
||||
|
||||
class Fake(runner.Step):
|
||||
"""A step whose output is <dir>/<id>.out holding its key; dir is the temp folder for a transient step."""
|
||||
|
||||
def __init__(self, id, transient=False, always=False, previous=None):
|
||||
self.id, self.transient, self.always, self.previous = id, transient, always, previous
|
||||
|
||||
def out(self, ctx):
|
||||
return (ctx.temp if self.transient else ctx.dest) / f'{self.id}.out'
|
||||
|
||||
def key(self, ctx):
|
||||
upstream = ctx.keys[self.previous] if self.previous else ''
|
||||
return f'{self.id}:{ctx.options.inputs.get(self.id, "v1")}:{upstream}'
|
||||
|
||||
def current(self, ctx, record):
|
||||
p = self.out(ctx)
|
||||
return p.is_file() and p.read_text() == ctx.keys[self.id]
|
||||
|
||||
def verify(self, ctx, record):
|
||||
ctx.options.verified.append(self.id)
|
||||
return ctx.options.verify_ok.get(self.id, True)
|
||||
|
||||
def run(self, ctx):
|
||||
o = ctx.options
|
||||
o.ran.append(self.id)
|
||||
if self.id in ctx.forced:
|
||||
o.forced.append(self.id)
|
||||
if self.id == 'check':
|
||||
ctx.verify_kept(self.id)
|
||||
failure = o.fail.pop(self.id, None)
|
||||
if failure == 'error':
|
||||
raise runner.InstallError('fake.failed', step=self.id)
|
||||
if failure == 'cancel':
|
||||
raise KeyboardInterrupt
|
||||
if self.id == 'cleanup':
|
||||
for p in ctx.temp.glob('*.out'):
|
||||
p.unlink()
|
||||
return {}
|
||||
p = self.out(ctx)
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
p.write_text(ctx.keys[self.id])
|
||||
return {'note': self.id}
|
||||
|
||||
|
||||
def fake_steps():
|
||||
return [Fake('check', always=True), Fake('extract'), Fake('lift', transient=True, previous='extract'),
|
||||
Fake('compile', previous='lift'), Fake('prepare', previous='compile'), Fake('cleanup', always=True)]
|
||||
|
||||
|
||||
class Bed:
|
||||
"""A dest and temp folder plus one run of the fake steps per call."""
|
||||
|
||||
def __init__(self, base):
|
||||
self.dest, self.temp = base / 'dest', base / 'temp'
|
||||
self.temp.mkdir()
|
||||
|
||||
def run(self, release='1.0', inputs=None, fail=None, verify_ok=None, verify=False):
|
||||
layout = Layout(root=Path('.'), dev=True, release=release, python=Path('python'), tools=Path('tools'),
|
||||
recomp=Path('recomp'), llvm=Path('llvm'),
|
||||
deps=Path('deps'), runtime=Path('rt'), runtime_lib=Path('lib'), ninja=Path('ninja'),
|
||||
hactool=Path('hactool'), xlate=Path('xlate'))
|
||||
options = SimpleNamespace(inputs=inputs or {}, fail=dict(fail or {}), verify_ok=verify_ok or {}, ran=[],
|
||||
verified=[], forced=[])
|
||||
out = io.StringIO()
|
||||
ctx = runner.Context(layout, self.dest, self.temp, Reporter(out=out), options, state=state.load(self.dest),
|
||||
verify=verify)
|
||||
mode = ctx.mode
|
||||
result = runner.run_steps(fake_steps(), ctx)
|
||||
events = [json.loads(line) for line in out.getvalue().splitlines()]
|
||||
return SimpleNamespace(result=result, ran=options.ran, verified=options.verified, forced=options.forced,
|
||||
events=events, mode=mode,
|
||||
state=state.load(self.dest))
|
||||
|
||||
@staticmethod
|
||||
def skipped(r):
|
||||
return {e['step']: e['reason'] for e in r.events if e['type'] == 'step' and e['state'] == 'skipped'}
|
||||
|
||||
|
||||
ALL = ['check', 'extract', 'lift', 'compile', 'prepare', 'cleanup']
|
||||
|
||||
|
||||
class RunnerTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.bed = Bed(Path(self.tmp.name))
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def test_fresh_install_runs_everything_and_records_it(self):
|
||||
r = self.bed.run()
|
||||
self.assertEqual((r.result, r.mode, r.ran), ('ok', 'install', ALL))
|
||||
self.assertEqual(set(r.state['steps']), {'extract', 'lift', 'compile', 'prepare'})
|
||||
self.assertEqual(r.state['steps']['compile']['note'], 'compile')
|
||||
self.assertEqual(r.state['release'], '1.0')
|
||||
started = [e['step'] for e in r.events if e['type'] == 'step' and e['state'] == 'started']
|
||||
done = [e['step'] for e in r.events if e['type'] == 'step' and e['state'] == 'done']
|
||||
self.assertEqual((started, done), (ALL, ALL))
|
||||
|
||||
def test_rerun_skips_finished_steps_and_unneeded_transient(self):
|
||||
self.bed.run()
|
||||
r = self.bed.run()
|
||||
self.assertEqual((r.result, r.mode, r.ran), ('ok', 'resume', ['check', 'cleanup']))
|
||||
self.assertEqual(self.bed.skipped(r), {'extract': 'current', 'lift': 'not_needed', 'compile': 'current',
|
||||
'prepare': 'current'})
|
||||
self.assertEqual(r.verified, []) # same release: no full checks
|
||||
|
||||
def test_resume_after_failure(self):
|
||||
r = self.bed.run(fail={'compile': 'error'})
|
||||
self.assertEqual((r.result, r.ran), ('failed', ['check', 'extract', 'lift', 'compile']))
|
||||
error = [e for e in r.events if e['type'] == 'error'][0]
|
||||
self.assertEqual((error['id'], error['step'], error['params']), ('fake.failed', 'compile', {'step': 'compile'}))
|
||||
self.assertEqual(set(r.state['steps']), {'extract', 'lift'})
|
||||
r = self.bed.run()
|
||||
# lift's output is still in the temp folder (no cleanup ran), so only compile and later run.
|
||||
self.assertEqual((r.result, r.mode, r.ran), ('ok', 'resume', ['check', 'compile', 'prepare', 'cleanup']))
|
||||
|
||||
def test_resume_after_cancel_with_transient_output_gone(self):
|
||||
r = self.bed.run(fail={'prepare': 'cancel'})
|
||||
self.assertEqual(r.result, 'cancelled')
|
||||
self.assertNotIn('prepare', r.state['steps'])
|
||||
for p in self.bed.temp.glob('*.out'):
|
||||
p.unlink()
|
||||
r = self.bed.run()
|
||||
self.assertEqual(r.ran, ['check', 'prepare', 'cleanup'])
|
||||
self.assertEqual(self.bed.skipped(r)['lift'], 'not_needed')
|
||||
|
||||
def test_failed_check_writes_nothing(self):
|
||||
r = self.bed.run(fail={'check': 'error'})
|
||||
self.assertEqual(r.result, 'failed')
|
||||
self.assertFalse(self.bed.dest.exists())
|
||||
|
||||
def test_removed_output_runs_again(self):
|
||||
self.bed.run()
|
||||
(self.bed.dest / 'compile.out').unlink()
|
||||
r = self.bed.run()
|
||||
self.assertEqual(r.ran, ['check', 'lift', 'compile', 'cleanup'])
|
||||
|
||||
def test_update_with_new_lifter_keeps_extraction(self):
|
||||
self.bed.run(release='1.0')
|
||||
r = self.bed.run(release='1.1', inputs={'lift': 'v2'})
|
||||
self.assertEqual((r.result, r.mode), ('ok', 'update'))
|
||||
self.assertEqual(r.ran, ['check', 'lift', 'compile', 'prepare', 'cleanup'])
|
||||
self.assertEqual(r.verified, ['extract']) # kept from another release: full check
|
||||
self.assertEqual(r.state['steps']['extract']['release'], '1.0')
|
||||
self.assertEqual(r.state['steps']['compile']['release'], '1.1')
|
||||
self.assertEqual(r.state['release'], '1.1')
|
||||
|
||||
def test_update_that_changes_nothing_skips_the_rebuild(self):
|
||||
self.bed.run(release='1.0')
|
||||
r = self.bed.run(release='1.1')
|
||||
self.assertEqual(r.ran, ['check', 'cleanup'])
|
||||
self.assertEqual(r.verified, ['extract', 'compile', 'prepare'])
|
||||
|
||||
def test_update_with_changed_kept_files_extracts_again(self):
|
||||
self.bed.run(release='1.0')
|
||||
r = self.bed.run(release='1.1', verify_ok={'extract': False})
|
||||
self.assertEqual(r.ran, ['check', 'extract', 'cleanup'])
|
||||
self.assertEqual(r.forced, ['extract']) # the step knows not to trust what it left behind
|
||||
warnings = [e for e in r.events if e['type'] == 'warning']
|
||||
self.assertEqual([(w['id'], w['params']) for w in warnings], [('state.output_changed', {'step': 'extract'})])
|
||||
|
||||
def test_verify_option_checks_same_release(self):
|
||||
self.bed.run()
|
||||
r = self.bed.run(verify=True)
|
||||
self.assertEqual(r.verified, ['extract', 'compile', 'prepare'])
|
||||
|
||||
def test_state_errors(self):
|
||||
self.bed.dest.mkdir()
|
||||
(self.bed.dest / state.NAME).write_text('{not json')
|
||||
with self.assertRaises(state.StateError) as e:
|
||||
state.load(self.bed.dest)
|
||||
self.assertEqual(e.exception.id, 'state.unreadable')
|
||||
(self.bed.dest / state.NAME).write_text(json.dumps({'format': 99, 'steps': {}}))
|
||||
with self.assertRaises(state.StateError) as e:
|
||||
state.load(self.bed.dest)
|
||||
self.assertEqual(e.exception.id, 'state.newer_format')
|
||||
|
||||
|
||||
class ReleaseFilesTest(unittest.TestCase):
|
||||
def test_release_ships_every_tool_the_steps_run(self):
|
||||
sys.path.insert(0, str(ROOT / 'tools'))
|
||||
import make_release
|
||||
text = (ROOT / 'installer' / 'steps.py').read_text(encoding='utf-8')
|
||||
used = set(re.findall(r"layout\.tools / '(\w+\.py)'", text))
|
||||
for names in re.findall(r"layout\.tools / n for n in \((.*?)\)", text, re.S):
|
||||
used |= set(re.findall(r"'(\w+\.py)'", names))
|
||||
self.assertIn('cheats.py', used)
|
||||
self.assertEqual(used - set(make_release.TOOLS_FILES), set())
|
||||
|
||||
|
||||
class RenamedVersionTest(unittest.TestCase):
|
||||
"""An install the setup before 0.3.1 recorded as 1.1.1 is the 1.1.0 update (#332)."""
|
||||
|
||||
def setUp(self):
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(tmp.cleanup)
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
layout = Layout(root=root, dev=True, release='9.9', python=Path('python'), tools=root / 'tools',
|
||||
recomp=root / 'recomp', llvm=Path('llvm'), deps=Path('deps'), runtime=Path('rt'),
|
||||
runtime_lib=Path('lib'), ninja=Path('ninja'), hactool=Path('hactool'), xlate=Path('xlate'))
|
||||
state = {'format': 1, 'release': '0.2.8', 'version': '1.1.1', 'inputs': {},
|
||||
'steps': {'extract': {'key': 'recorded by 0.2.8', 'release': '0.2.8'}}}
|
||||
out = io.StringIO()
|
||||
self.ctx = runner.Context(layout, Path(tmp.name), Path(tmp.name), Reporter(out=out), SimpleNamespace(),
|
||||
state=state)
|
||||
|
||||
def test_old_name_is_read_as_1_1_0(self):
|
||||
steps.migrate_state(self.ctx)
|
||||
self.assertEqual(steps.game_version(self.ctx), '1.1.0')
|
||||
self.assertEqual(steps.version_tag(steps.game_version(self.ctx)), 'v110')
|
||||
self.assertEqual(steps.expected_update(self.ctx)['display_version'], '1.1.0')
|
||||
self.assertTrue(self.ctx.layout.patches(steps.game_version(self.ctx)).is_file())
|
||||
|
||||
def test_extraction_is_kept_under_the_new_name(self):
|
||||
steps.migrate_state(self.ctx)
|
||||
self.assertEqual(self.ctx.state['steps']['extract']['key'], steps.Extract().key(self.ctx))
|
||||
self.assertEqual(self.ctx.state['steps']['extract']['release'], '0.2.8') # full check of kept output runs
|
||||
|
||||
def test_other_versions_are_untouched(self):
|
||||
self.ctx.state['version'] = '1.4.3'
|
||||
steps.migrate_state(self.ctx)
|
||||
self.assertEqual(self.ctx.state['version'], '1.4.3')
|
||||
self.assertEqual(self.ctx.state['steps']['extract']['key'], 'recorded by 0.2.8')
|
||||
|
||||
|
||||
class LiftKeyTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(tmp.cleanup)
|
||||
self.dest = Path(tmp.name)
|
||||
layout = SimpleNamespace(recomp=self.dest / 'no-recomp', tools=self.dest / 'no-tools',
|
||||
patches=lambda version: self.dest / 'no-patches')
|
||||
self.ctx = SimpleNamespace(dest=self.dest, layout=layout, keys={'extract': 'x'}, state={'version': '1.4.3'})
|
||||
|
||||
def key(self):
|
||||
return steps.Lift().key(self.ctx)
|
||||
|
||||
def add_cheats(self, name, text, mod='Cheats', where='cheats'):
|
||||
folder = self.dest / 'mods' / mod / where
|
||||
folder.mkdir(parents=True, exist_ok=True)
|
||||
(folder / name).write_text(text)
|
||||
|
||||
def test_extracting_main_keeps_the_key(self):
|
||||
before = self.key()
|
||||
(self.dest / 'exefs').mkdir()
|
||||
(self.dest / 'exefs' / 'main').write_bytes(bytes(0x48))
|
||||
self.assertEqual(self.key(), before)
|
||||
|
||||
def test_cheat_files_change_the_key(self):
|
||||
before = self.key()
|
||||
self.add_cheats('277178B7DBA1B6D4.txt', '[a]')
|
||||
added = self.key()
|
||||
self.assertNotEqual(added, before)
|
||||
self.add_cheats('277178B7DBA1B6D4.txt', '[b]')
|
||||
edited = self.key()
|
||||
self.assertNotEqual(edited, added)
|
||||
self.add_cheats('277178B7DBA1B6D4.txt', '[c]', mod='Other', where='atmosphere/contents/0100F2C0115B6000/cheats')
|
||||
self.assertNotEqual(self.key(), edited)
|
||||
|
||||
def test_exefs_patch_files_change_the_key(self):
|
||||
before = self.key()
|
||||
self.add_cheats('dynamic-fps.pchtxt', '@nsobid-277178B7DBA1B6D4', mod='Fps', where='exefs')
|
||||
added = self.key()
|
||||
self.assertNotEqual(added, before)
|
||||
self.add_cheats('277178B7DBA1B6D4.ips', 'PATCHEOF', mod='Ips', where='atmosphere/exefs_patches/x')
|
||||
both = self.key()
|
||||
self.assertNotEqual(both, added)
|
||||
self.add_cheats('main.npdm', 'not a patch', mod='Fps', where='exefs')
|
||||
self.assertEqual(self.key(), both)
|
||||
|
||||
def test_renaming_a_mod_folder_changes_the_key(self):
|
||||
# tools/cheats.py and tools/exefs_patches.py name a patch set after "<mod folder>/<cheat or file>": a renamed
|
||||
# folder needs a lift even though its files are the same
|
||||
self.add_cheats('277178B7DBA1B6D4.txt', '[a]', mod='MyNXCheats')
|
||||
self.add_cheats('dynamic-fps.pchtxt', '@nsobid-277178B7DBA1B6D4', mod='Fps', where='exefs')
|
||||
before = self.key()
|
||||
mods = self.dest / 'mods'
|
||||
(mods / 'MyNXCheats').rename(mods / 'MyNXCheats 1.4.3')
|
||||
cheats_renamed = self.key()
|
||||
self.assertNotEqual(cheats_renamed, before)
|
||||
(mods / 'Fps').rename(mods / 'Fps 2')
|
||||
self.assertNotEqual(self.key(), cheats_renamed)
|
||||
|
||||
def test_status_and_other_files_keep_the_key(self):
|
||||
self.add_cheats('277178B7DBA1B6D4.txt', '[a]')
|
||||
before = self.key()
|
||||
(self.dest / 'build-cache').mkdir()
|
||||
(self.dest / 'build-cache' / 'cheats.status').write_text('written by lift')
|
||||
self.add_cheats('readme.md', 'not a cheat file')
|
||||
(self.dest / 'mods' / 'Cheats' / 'romfs').mkdir()
|
||||
(self.dest / 'mods' / 'Cheats' / 'romfs' / 'a.txt').write_text('a RomFS file')
|
||||
self.assertEqual(self.key(), before)
|
||||
|
||||
|
||||
class IniTest(unittest.TestCase):
|
||||
EXAMPLE = '[display]\n# vk_present = mailbox\n\n[language]\n# locale = en-US\n# lang = en-US\n# x = 1\n'
|
||||
|
||||
def test_example_line_is_activated(self):
|
||||
out = ini.set_language(self.EXAMPLE, 'ru')
|
||||
self.assertIn('\nlang = ru\n', out)
|
||||
self.assertNotIn('# lang', out)
|
||||
self.assertIn('# locale = en-US', out)
|
||||
|
||||
def test_active_line_is_rewritten_and_the_rest_kept(self):
|
||||
text = '[display]\nvk_present = fifo\n[language]\n LANG = ja \nfps60 = 0\n'
|
||||
out = ini.set_language(text, 'de')
|
||||
self.assertEqual(out, '[display]\nvk_present = fifo\n[language]\n lang = de\nfps60 = 0\n')
|
||||
self.assertEqual(ini.get_language(out), 'de')
|
||||
|
||||
def test_missing_section_is_added(self):
|
||||
self.assertEqual(ini.set_language('fps60 = 0\n', 'fr'), 'fps60 = 0\n\n[language]\nlang = fr\n')
|
||||
self.assertEqual(ini.set_language('[language]\nlocale = de-DE\n', 'fr'),
|
||||
'[language]\nlang = fr\nlocale = de-DE\n')
|
||||
|
||||
def test_old_chinese_codes_are_written_as_the_canonical_ones(self):
|
||||
self.assertEqual(ini.canonical_language('zh-CN'), 'zh-Hans')
|
||||
self.assertEqual(ini.canonical_language('zh-TW'), 'zh-Hant')
|
||||
self.assertEqual(ini.canonical_language('ja'), 'ja')
|
||||
self.assertNotIn('zh-CN', ini.LANGUAGES)
|
||||
self.assertNotIn('zh-TW', ini.LANGUAGES)
|
||||
out = ini.set_language(self.EXAMPLE, 'zh-CN')
|
||||
self.assertEqual(ini.get_language(out), 'zh-Hans')
|
||||
self.assertEqual(ini.get_language(ini.set_language(out, 'zh-TW')), 'zh-Hant')
|
||||
|
||||
def test_write_keeps_user_lines(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
d = Path(d)
|
||||
example = d / 'nx2navi.ini.example'
|
||||
example.write_text(self.EXAMPLE)
|
||||
path = ini.write(d / '', 'en-GB', example)
|
||||
self.assertEqual(ini.get_language(path.read_text()), 'en-GB')
|
||||
path.write_text(path.read_text().replace('# vk_present = mailbox', 'vk_present = fifo'))
|
||||
ini.write(d, 'ja', example)
|
||||
text = path.read_text()
|
||||
self.assertIn('vk_present = fifo', text)
|
||||
self.assertEqual(ini.get_language(text), 'ja')
|
||||
|
||||
|
||||
class CleanupTest(unittest.TestCase):
|
||||
"""The real Cleanup step on a finished install whose romfs/ is intact, emptied or gone (#107)."""
|
||||
|
||||
def setUp(self):
|
||||
self.steps = steps
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
base = Path(self.tmp.name)
|
||||
self.dest, temp = base / 'dest', base / 'temp'
|
||||
(self.dest / 'romfs').mkdir(parents=True)
|
||||
temp.mkdir()
|
||||
image = b'romfs image'
|
||||
(self.dest / 'romfs' / 'romfs.bin').write_bytes(image)
|
||||
digest = steps.extract.sha256(self.dest / 'romfs' / 'romfs.bin')
|
||||
(self.dest / 'romfs' / 'index.json').write_text(json.dumps({'romfs_size': len(image), 'romfs_sha256': digest}))
|
||||
self.expected = {'romfs': {'size': len(image), 'sha256': digest}}
|
||||
self.ctx = SimpleNamespace(reporter=Reporter(out=io.StringIO(), log_path=base / 'x.log'), dest=self.dest,
|
||||
temp=temp, lang='en-US', state={},
|
||||
layout=SimpleNamespace(ini_example=ROOT / 'nx2navi.ini.example', dev=False,
|
||||
python=base / 'python' / 'python.exe', root=base / 'release'))
|
||||
self.saved = steps.expected_update
|
||||
steps.expected_update = lambda ctx: self.expected
|
||||
|
||||
def tearDown(self):
|
||||
self.steps.expected_update = self.saved
|
||||
self.ctx.reporter.close()
|
||||
self.tmp.cleanup()
|
||||
|
||||
def run_cleanup(self):
|
||||
try:
|
||||
self.steps.Cleanup().run(self.ctx)
|
||||
return 'ok'
|
||||
except self.steps.InstallError as e:
|
||||
return e.id
|
||||
|
||||
def test_intact_romfs_passes(self):
|
||||
self.assertEqual(self.run_cleanup(), 'ok')
|
||||
|
||||
def test_keeps_build_and_writes_rebuild_command(self):
|
||||
for name in ('gen', 'build', 'check'):
|
||||
(self.ctx.temp / name).mkdir()
|
||||
self.assertEqual(self.run_cleanup(), 'ok')
|
||||
self.assertTrue((self.ctx.temp / 'gen').is_dir() and (self.ctx.temp / 'build').is_dir())
|
||||
self.assertFalse((self.ctx.temp / 'check').exists())
|
||||
lines = (self.dest / 'build-cache' / 'rebuild.txt').read_text(encoding='utf-8').splitlines()
|
||||
self.assertEqual(lines[1:], ['-c', steps.REBUILD_BOOT, str(self.ctx.layout.root), '--root',
|
||||
str(self.ctx.layout.root), '--rebuild', str(self.dest)])
|
||||
|
||||
def test_emptied_romfs_fails(self):
|
||||
for p in (self.dest / 'romfs').iterdir():
|
||||
p.unlink()
|
||||
self.assertEqual(self.run_cleanup(), 'cleanup.romfs_unreadable')
|
||||
|
||||
def test_missing_romfs_fails(self):
|
||||
self.steps.remove(self.dest / 'romfs')
|
||||
self.assertEqual(self.run_cleanup(), 'cleanup.romfs_unreadable')
|
||||
|
||||
def test_romfs_of_another_size_fails(self):
|
||||
(self.dest / 'romfs' / 'romfs.bin').write_bytes(b'short')
|
||||
self.assertEqual(self.run_cleanup(), 'cleanup.romfs_unreadable')
|
||||
|
||||
|
||||
class BaseGameCheckTest(unittest.TestCase):
|
||||
"""Issue #261: an update NSP chosen as the base game was reported as "not Ocarina of Time"."""
|
||||
BASE, UPDATE = '0100F2C0115B6000', '0100F2C0115B6800'
|
||||
|
||||
@staticmethod
|
||||
def nca(content_type, title_id):
|
||||
return ({'name': 'x.nca', 'nca_offset': 0, 'size': 0xc00, 'content_type': content_type, 'title_id': title_id}, b'')
|
||||
|
||||
def test_update_package_names_the_update(self):
|
||||
# An update NSP keeps the base title id on its Program NCA; only its Meta NCA carries the update id.
|
||||
records = [self.nca(0, self.BASE), self.nca(1, self.UPDATE)]
|
||||
with self.assertRaises(ValueError):
|
||||
steps.program_nca(records)
|
||||
self.assertEqual(steps.base_misuse(records), 'check.base_is_update')
|
||||
|
||||
def test_combined_package_is_named(self):
|
||||
records = [self.nca(0, self.BASE), self.nca(1, self.BASE), self.nca(1, self.UPDATE)]
|
||||
self.assertEqual(steps.base_misuse(records), 'check.base_combined')
|
||||
|
||||
def test_plain_base_game_passes(self):
|
||||
records = [self.nca(0, self.BASE), self.nca(1, self.BASE), self.nca(3, self.BASE)]
|
||||
self.assertIsNone(steps.base_misuse(records))
|
||||
self.assertEqual(steps.program_nca(records)[0]['title_id'], self.BASE)
|
||||
|
||||
|
||||
class FontsTest(unittest.TestCase):
|
||||
"""The fonts step: keyed by language and firmware, a no-op without firmware, writes dest/shared-fonts."""
|
||||
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
base = Path(self.tmp.name)
|
||||
self.dest, temp, self.firmware = base / 'dest', base / 'temp', base / 'firmware'
|
||||
temp.mkdir()
|
||||
self.firmware.mkdir()
|
||||
(self.firmware / 'a.nca').write_bytes(b'nca')
|
||||
self.keys = base / 'prod.keys'
|
||||
self.keys.write_text('header_key = 00\n')
|
||||
self.ctx = SimpleNamespace(dest=self.dest, temp=temp, lang='zh-Hans', state={},
|
||||
layout=SimpleNamespace(tools=ROOT / 'tools', root=ROOT),
|
||||
reporter=Reporter(out=io.StringIO()),
|
||||
options=SimpleNamespace(lang='zh-Hans', firmware=self.firmware, keys=str(self.keys)))
|
||||
self.saved_run_tool = steps.run_tool
|
||||
|
||||
def tearDown(self):
|
||||
steps.run_tool = self.saved_run_tool
|
||||
self.ctx.reporter.close()
|
||||
self.tmp.cleanup()
|
||||
|
||||
def fake_tool(self, files=None, code=0):
|
||||
"""Stand-in for extract_shared_fonts.py: writes the requested files where --output points."""
|
||||
def run(ctx, step, script, args):
|
||||
out = Path(args[args.index('--output') + 1])
|
||||
out.mkdir(parents=True)
|
||||
for name in steps.SHARED_FONT_FILES if files is None else files:
|
||||
(out / name).write_bytes(b'bfttf')
|
||||
(out / 'manifest.json').write_text('{}')
|
||||
log = ctx.temp / 'fonts.log'
|
||||
log.write_text('error line\n')
|
||||
return code, log
|
||||
steps.run_tool = run
|
||||
|
||||
def test_all_steps_order(self):
|
||||
self.assertEqual([s.id for s in steps.all_steps()],
|
||||
['check', 'fonts', 'extract', 'lift', 'compile', 'prepare', 'dlss', 'cleanup'])
|
||||
|
||||
def test_extracts_into_the_install_folder(self):
|
||||
self.fake_tool()
|
||||
self.assertEqual(steps.Fonts().run(self.ctx), {'files': 6})
|
||||
self.assertTrue(steps.shared_fonts_ready(self.dest))
|
||||
self.assertEqual(sorted(p.name for p in (self.dest / 'shared-fonts').iterdir()),
|
||||
sorted((*steps.SHARED_FONT_FILES, 'manifest.json')))
|
||||
|
||||
def test_noop_without_firmware(self):
|
||||
self.ctx.options.firmware = None
|
||||
self.fake_tool()
|
||||
self.assertEqual(steps.Fonts().run(self.ctx), {})
|
||||
self.assertFalse((self.dest / 'shared-fonts').exists())
|
||||
|
||||
def test_noop_for_other_languages(self):
|
||||
self.ctx.lang, self.ctx.options.lang = 'en-US', 'en-US'
|
||||
self.fake_tool()
|
||||
self.assertEqual(steps.Fonts().run(self.ctx), {})
|
||||
self.assertFalse((self.dest / 'shared-fonts').exists())
|
||||
|
||||
def test_failed_extraction_keeps_the_old_folder(self):
|
||||
self.fake_tool()
|
||||
steps.Fonts().run(self.ctx)
|
||||
self.fake_tool(code=1)
|
||||
with self.assertRaises(steps.InstallError) as e:
|
||||
steps.Fonts().run(self.ctx)
|
||||
self.assertEqual(e.exception.id, 'fonts.failed')
|
||||
self.assertTrue(steps.shared_fonts_ready(self.dest))
|
||||
|
||||
def test_incomplete_output_is_an_error(self):
|
||||
self.fake_tool(files=steps.SHARED_FONT_FILES[:1])
|
||||
with self.assertRaises(steps.InstallError) as e:
|
||||
steps.Fonts().run(self.ctx)
|
||||
self.assertEqual(e.exception.id, 'fonts.incomplete')
|
||||
|
||||
def test_current_tracks_language_and_output(self):
|
||||
fonts = steps.Fonts()
|
||||
self.assertFalse(fonts.current(self.ctx, {}))
|
||||
self.fake_tool()
|
||||
fonts.run(self.ctx)
|
||||
self.assertTrue(fonts.current(self.ctx, {}))
|
||||
self.ctx.options.lang = 'en-US'
|
||||
self.assertTrue(fonts.current(self.ctx, {}))
|
||||
self.ctx.options.lang = 'zh-Hans'
|
||||
(self.dest / 'shared-fonts' / '0-standard.bfttf').unlink()
|
||||
self.assertFalse(fonts.current(self.ctx, {}))
|
||||
|
||||
def test_key_changes_with_language_and_firmware(self):
|
||||
fonts = steps.Fonts()
|
||||
zh = fonts.key(self.ctx)
|
||||
self.ctx.options.lang = 'en-US'
|
||||
self.assertNotEqual(zh, fonts.key(self.ctx))
|
||||
self.ctx.options.lang = 'zh-Hans'
|
||||
(self.firmware / 'b.nca').write_bytes(b'other')
|
||||
self.assertNotEqual(zh, fonts.key(self.ctx))
|
||||
|
||||
def test_registered_child_is_used(self):
|
||||
nested = self.firmware / 'registered'
|
||||
nested.mkdir()
|
||||
(nested / 'c.nca').write_bytes(b'nca')
|
||||
self.assertEqual(steps.firmware_dir(self.firmware), nested)
|
||||
|
||||
|
||||
class EventsTest(unittest.TestCase):
|
||||
def test_lines_are_json_and_the_log_is_english(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
out = io.StringIO()
|
||||
rep = Reporter(out=out, log_path=Path(d) / 'x.log')
|
||||
rep.step('check', 'started')
|
||||
rep.progress('extract', 'copy', 0, 10, 'bytes')
|
||||
rep.progress('extract', 'copy', 5, 10, 'bytes') # throttled
|
||||
rep.progress('extract', 'copy', 10, 10, 'bytes') # last one always goes out
|
||||
rep.error('check.disk_space', {'path': 'D:\\x', 'need_gb': 40, 'free_gb': 3}, step='check')
|
||||
rep.close()
|
||||
events = [json.loads(line) for line in out.getvalue().splitlines()]
|
||||
self.assertEqual([e['type'] for e in events], ['step', 'progress', 'progress', 'error'])
|
||||
self.assertEqual([e.get('done') for e in events[1:3]], [0, 10])
|
||||
self.assertTrue(all('t' in e for e in events))
|
||||
log = (Path(d) / 'x.log').read_text(encoding='utf-8')
|
||||
self.assertIn('40 GB needed, 3 GB free', log)
|
||||
self.assertIn('[check.disk_space]', log)
|
||||
|
||||
def test_catalog_has_every_id_the_code_uses(self):
|
||||
catalog = json.loads((ROOT / 'installer' / 'text' / 'en.json').read_text(encoding='utf-8'))
|
||||
used = set()
|
||||
for path in (ROOT / 'installer').glob('*.py'):
|
||||
text = path.read_text(encoding='utf-8')
|
||||
used |= set(re.findall(r"InstallError\('([a-z_.]+)'", text))
|
||||
used |= set(re.findall(r"warning\('([a-z_.]+)'", text))
|
||||
used |= {f'phase.{s}.{p}' for s, p in re.findall(r"progress\('(\w+)', '(\w+)'[,)]", text)}
|
||||
self.assertEqual(sorted(i for i in used if i not in catalog), [])
|
||||
|
||||
|
||||
@unittest.skipUnless(steps.os.name == 'nt', 'DLSS is Windows-only')
|
||||
class DlssTest(unittest.TestCase):
|
||||
"""The DLSS step: off without --dlss, needs an RTX GPU, keeps a player's library, verifies the download."""
|
||||
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
base = Path(self.tmp.name)
|
||||
self.dest, temp = base / 'dest', base / 'temp'
|
||||
self.dest.mkdir()
|
||||
temp.mkdir()
|
||||
self.src = base / 'nvngx_dlss.dll'
|
||||
self.src.write_bytes(b'dlss' * 1000)
|
||||
self.saved = steps.DLSS_URL, steps.DLSS_SIZE, steps.DLSS_SHA256
|
||||
steps.DLSS_URL = self.src.as_uri() # urllib reads file: URLs as it reads https: ones
|
||||
steps.DLSS_SIZE, steps.DLSS_SHA256 = 4000, steps.file_sha256(self.src)
|
||||
rtx = {'devices': [{'name': 'NVIDIA GeForce RTX 4080 SUPER', 'api': [1, 4, 0], 'type': 2}]}
|
||||
self.ctx = SimpleNamespace(dest=self.dest, temp=temp, host=rtx, reporter=Reporter(out=io.StringIO()),
|
||||
options=SimpleNamespace(dlss=True))
|
||||
|
||||
def tearDown(self):
|
||||
steps.DLSS_URL, steps.DLSS_SIZE, steps.DLSS_SHA256 = self.saved
|
||||
self.ctx.reporter.close()
|
||||
self.tmp.cleanup()
|
||||
|
||||
def warnings(self):
|
||||
return [json.loads(line)['id'] for line in self.ctx.reporter.out.getvalue().splitlines()
|
||||
if json.loads(line)['type'] == 'warning']
|
||||
|
||||
def test_off_without_the_option(self):
|
||||
self.ctx.options.dlss = False
|
||||
self.assertEqual(steps.Dlss().run(self.ctx), {})
|
||||
self.assertFalse((self.dest / 'nvngx_dlss.dll').exists())
|
||||
self.assertNotEqual(steps.Dlss().key(self.ctx), steps.Dlss().key(SimpleNamespace(options=SimpleNamespace(dlss=True))))
|
||||
|
||||
def test_downloads_and_checks_the_library(self):
|
||||
rec = steps.Dlss().run(self.ctx)
|
||||
self.assertEqual(rec, {'installed': steps.DLSS_VERSION})
|
||||
self.assertEqual((self.dest / 'nvngx_dlss.dll').read_bytes(), self.src.read_bytes())
|
||||
self.assertFalse(list(self.ctx.temp.iterdir()))
|
||||
self.assertTrue(steps.Dlss().current(self.ctx, rec))
|
||||
(self.dest / 'nvngx_dlss.dll').unlink()
|
||||
self.assertFalse(steps.Dlss().current(self.ctx, rec))
|
||||
|
||||
def test_a_wrong_file_is_not_installed_and_tried_again(self):
|
||||
steps.DLSS_SHA256 = '0' * 64
|
||||
rec = steps.Dlss().run(self.ctx)
|
||||
self.assertEqual(rec, {'failed': True})
|
||||
self.assertFalse((self.dest / 'nvngx_dlss.dll').exists())
|
||||
self.assertEqual(self.warnings(), ['warn.dlss_download_failed'])
|
||||
self.assertFalse(steps.Dlss().current(self.ctx, rec))
|
||||
|
||||
def test_a_cut_connection_warns(self):
|
||||
class Cut(io.BytesIO): # a response whose connection drops mid-file
|
||||
def read(self, n=-1):
|
||||
raise http.client.IncompleteRead(b'dl', 3998)
|
||||
|
||||
with mock.patch.object(steps.urllib.request, 'urlopen', return_value=Cut()):
|
||||
self.assertEqual(steps.Dlss().run(self.ctx), {'failed': True})
|
||||
self.assertFalse((self.dest / 'nvngx_dlss.dll').exists())
|
||||
self.assertFalse(list(self.ctx.temp.iterdir()))
|
||||
self.assertEqual(self.warnings(), ['warn.dlss_download_failed'])
|
||||
|
||||
def test_keeps_the_players_library(self):
|
||||
(self.dest / 'nvngx_dlss.dll').write_bytes(b'newer')
|
||||
self.assertEqual(steps.Dlss().run(self.ctx), {'kept': True})
|
||||
self.assertEqual((self.dest / 'nvngx_dlss.dll').read_bytes(), b'newer')
|
||||
self.assertEqual(self.warnings(), ['warn.dlss_kept'])
|
||||
|
||||
def test_skipped_without_an_rtx_gpu(self):
|
||||
self.ctx.host = {'devices': [{'name': 'AMD Radeon RX 7800 XT', 'api': [1, 3, 0], 'type': 2}]}
|
||||
self.assertEqual(steps.Dlss().run(self.ctx), {'skipped': 'no_rtx'})
|
||||
self.assertEqual(self.warnings(), [])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
809
tests/jumptables_check.py
Normal file
809
tests/jumptables_check.py
Normal file
@@ -0,0 +1,809 @@
|
||||
"""Check of recomp/jumptables.py on every `br` in the game's modules against an emulation oracle.
|
||||
|
||||
The oracle is LAS Recomp's jump-table method: find `cmp #N; b.hi/b.hs` before the br (or a `b.ls/b.lo` to the
|
||||
dispatch block), replay ADRP/ADR/ADD-imm from the function entry, then single-step Unicorn from the bounds check to
|
||||
the br once per index. Functions are [seed, next seed) over the module's seeds plus BL targets.
|
||||
|
||||
Reports per module: br sites, resolved by the resolver / by the oracle, agreement, `is_computed` sites left
|
||||
unresolved, tables whose targets leave the function, an independent check of hoisted table bases (hoisted_base),
|
||||
PLT stubs (`br x17` after the GOT load) that must stay a plain call, and resolver time. A disagreement is counted
|
||||
as 'oracle wrong' only with independent evidence (oracle_wrong). Samples of every disagreement, unverified base and
|
||||
unresolved computed site are printed with disassembly. Output also goes to build/logs/jumptables-check.log, and one
|
||||
row per table-like br to build/logs/jumptables-<module>.tsv.
|
||||
|
||||
The oracle is adapted from LAS Recomp tools/gen.py @ da35b83 (MIT, Copyright (c) 2026 Peiikai).
|
||||
|
||||
Exits with 1 if any disagreement is unexplained, a hoisted base is not verified, or a PLT stub counts as computed.
|
||||
|
||||
Usage: jumptables_check.py [--exefs DIR] [--modules main,sdk,subsdk0] [--samples N] [--no-oracle]
|
||||
"""
|
||||
import argparse
|
||||
import bisect
|
||||
import sys
|
||||
import time
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
import capstone
|
||||
import numpy as np
|
||||
import unicorn as uc
|
||||
from unicorn import arm64_const as A
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'recomp'))
|
||||
|
||||
import jumptables # noqa: E402
|
||||
import layout # noqa: E402
|
||||
from lifter import bits, sext # noqa: E402
|
||||
from module import Module # noqa: E402
|
||||
|
||||
SCRATCH_MEM = 0x7F00000000
|
||||
LOG = ROOT / 'build' / 'logs' / 'jumptables-check.log'
|
||||
|
||||
_md = capstone.Cs(capstone.CS_ARCH_ARM64, capstone.CS_MODE_ARM)
|
||||
|
||||
|
||||
class Oracle:
|
||||
"""LAS Recomp's Unicorn jump-table resolver (`Module.jump_table` and helpers in tools/gen.py)."""
|
||||
|
||||
def __init__(self, mod):
|
||||
self.m = mod
|
||||
mu = uc.Uc(uc.UC_ARCH_ARM64, uc.UC_MODE_ARM)
|
||||
mu.mem_map(mod.base, len(mod.image))
|
||||
mu.mem_write(mod.base, mod.image)
|
||||
mu.mem_map(SCRATCH_MEM, 0x100000)
|
||||
mu.reg_write(A.UC_ARM64_REG_CPACR_EL1, 0x300000)
|
||||
self.mu = mu
|
||||
self.set_at = self.regs = None
|
||||
self.start = self.cmp = None # the first instruction of the bounds-check block, the bounds cmp
|
||||
self.guard = None # _taken: the b.ls/b.lo into the dispatch block (set_at)
|
||||
self.count = None # indices the bounds check lets through
|
||||
self.exclude = frozenset() # indices the emulation skips (oracle_wrong: excluded on every way in)
|
||||
self.watch = self.seen = None # (address, register): record the register when the emulation reaches address
|
||||
|
||||
def word(self, a):
|
||||
return self.m.word(a)
|
||||
|
||||
def kind(self, a):
|
||||
return self.m.lift(a).kind
|
||||
|
||||
def jump_table(self, br_pc, entry, end):
|
||||
"""Targets, or None. Leaves the index set point and the registers set there in self.set_at, self.regs."""
|
||||
self.set_at = self.regs = self.start = self.cmp = self.guard = self.count = None
|
||||
r = self._fallthrough(br_pc, entry, end)
|
||||
if r is None:
|
||||
r = self._taken(br_pc, entry, end)
|
||||
return r
|
||||
|
||||
def _index_regs(self, start, at, idx):
|
||||
def copy_src(w):
|
||||
if (w & 0x7F800000) == 0x12000000: # AND (imm)
|
||||
return bits(w, 5, 5)
|
||||
if (w & 0x7FE0FFE0) == 0x2A0003E0: # MOV (ORR with zr)
|
||||
return bits(w, 16, 5)
|
||||
if (w & 0x7F800000) in (0x53000000, 0x13000000): # UBFM/SBFM
|
||||
return bits(w, 5, 5)
|
||||
return None
|
||||
regs = {idx}
|
||||
changed = True
|
||||
while changed:
|
||||
changed = False
|
||||
for a in range(start, at, 4):
|
||||
w = self.word(a)
|
||||
rd, src = w & 31, copy_src(w)
|
||||
if src is None or src >= 31 or rd >= 31:
|
||||
continue
|
||||
for x, y in ((rd, src), (src, rd)):
|
||||
if x in regs and y not in regs:
|
||||
regs.add(y)
|
||||
changed = True
|
||||
return sorted(regs)
|
||||
|
||||
def _reset(self, entry, start):
|
||||
mu = self.mu
|
||||
for r in range(29):
|
||||
mu.reg_write(A.UC_ARM64_REG_X0 + r, SCRATCH_MEM + 0x80000)
|
||||
mu.reg_write(A.UC_ARM64_REG_SP, SCRATCH_MEM + 0x80000)
|
||||
a = entry
|
||||
while a < start:
|
||||
w = self.word(a)
|
||||
if (w & 0x1F000000) == 0x10000000 or (w & 0xFF800000) == 0x91000000:
|
||||
mu.emu_start(a, 0, count=1)
|
||||
a += 4
|
||||
|
||||
def _step_to(self, frm, to):
|
||||
mu, pc = self.mu, frm
|
||||
for _ in range(64):
|
||||
if self.watch and pc == self.watch[0] and self.seen is None:
|
||||
self.seen = self._read(self.watch[1])
|
||||
if pc == to:
|
||||
return True
|
||||
mu.emu_start(pc, 0, count=1)
|
||||
pc = mu.reg_read(A.UC_ARM64_REG_PC)
|
||||
return False
|
||||
|
||||
def _read(self, rn):
|
||||
return self.mu.reg_read(A.UC_ARM64_REG_X0 + rn if rn < 29 else
|
||||
(A.UC_ARM64_REG_X29 if rn == 29 else A.UC_ARM64_REG_X30))
|
||||
|
||||
def _emulate(self, br_pc, entry, end, start, regs, count, set_at):
|
||||
rn = bits(self.word(br_pc), 5, 5)
|
||||
targets = []
|
||||
try:
|
||||
for i in range(count):
|
||||
if i in self.exclude:
|
||||
continue
|
||||
self._reset(entry, start)
|
||||
if set_at != start and not self._step_to(start, set_at):
|
||||
return None
|
||||
for r in regs:
|
||||
self.mu.reg_write(A.UC_ARM64_REG_X0 + r if r < 29 else A.UC_ARM64_REG_X29, i)
|
||||
if not self._step_to(set_at, br_pc):
|
||||
return None
|
||||
t = self._read(rn)
|
||||
if not (entry <= t < end) or t % 4:
|
||||
return None
|
||||
targets.append(t)
|
||||
except uc.UcError:
|
||||
return None
|
||||
return sorted(set(targets))
|
||||
|
||||
def _taken(self, br_pc, entry, end):
|
||||
s = br_pc
|
||||
while s - 4 >= entry and br_pc - (s - 4) <= 4 * 12 and self.kind(s - 4) in ('op', 'asm'):
|
||||
s -= 4
|
||||
for g in range(entry, end, 4):
|
||||
w = self.word(g)
|
||||
if (w & 0xFF000010) != 0x54000000 or bits(w, 0, 4) not in (9, 3):
|
||||
continue
|
||||
if g + sext(bits(w, 5, 19), 19) * 4 != s:
|
||||
continue
|
||||
cw = self.word(g - 4) if g - 4 >= entry else 0
|
||||
if (cw & 0x7F80001F) != 0x7100001F or bits(cw, 22, 1):
|
||||
continue
|
||||
limit = bits(cw, 10, 12)
|
||||
count = limit + 1 if bits(w, 0, 4) == 9 else limit
|
||||
if count <= 0 or count > 4096:
|
||||
continue
|
||||
self.set_at, self.regs, self.guard, self.cmp, self.count = s, [bits(cw, 5, 5)], g, g - 4, count
|
||||
b = g - 4
|
||||
while b - 4 >= entry and g - 4 - (b - 4) <= 4 * 12 and self.kind(b - 4) in ('op', 'asm'):
|
||||
b -= 4
|
||||
self.start = b
|
||||
return self._emulate(br_pc, entry, end, s, [bits(cw, 5, 5)], count, s)
|
||||
return None
|
||||
|
||||
def _fallthrough(self, br_pc, entry, end):
|
||||
a = br_pc - 4
|
||||
guard = cmp = None
|
||||
while a >= entry and br_pc - a <= 4 * 16:
|
||||
w = self.word(a)
|
||||
if (w & 0xFF000010) == 0x54000000 and bits(w, 0, 4) in (8, 2):
|
||||
guard = a
|
||||
break
|
||||
if self.kind(a) in ('b', 'bl', 'blr', 'br', 'ret', 'trap'):
|
||||
return None
|
||||
a -= 4
|
||||
if guard is None:
|
||||
return None
|
||||
a = guard - 4
|
||||
while a >= entry and guard - a <= 4 * 8:
|
||||
if (self.word(a) & 0x7F80001F) == 0x7100001F:
|
||||
cmp = a
|
||||
break
|
||||
if self.kind(a) != 'op':
|
||||
return None
|
||||
a -= 4
|
||||
if cmp is None:
|
||||
return None
|
||||
w = self.word(cmp)
|
||||
if bits(w, 22, 1):
|
||||
return None
|
||||
limit = bits(w, 10, 12)
|
||||
count = limit + 1 if bits(self.word(guard), 0, 4) == 8 else limit
|
||||
if count <= 0 or count > 4096:
|
||||
return None
|
||||
start = cmp
|
||||
while start - 4 >= entry and cmp - (start - 4) <= 4 * 12 and self.kind(start - 4) in ('op', 'asm'):
|
||||
start -= 4
|
||||
regs = self._index_regs(start, cmp, bits(w, 5, 5))
|
||||
self.set_at, self.regs, self.start, self.cmp, self.count = cmp, regs, start, cmp, count
|
||||
rn = bits(self.word(br_pc), 5, 5)
|
||||
targets = []
|
||||
try:
|
||||
for i in range(count):
|
||||
if i in self.exclude:
|
||||
continue
|
||||
self._reset(entry, start)
|
||||
if not self._step_to(start, cmp):
|
||||
return None
|
||||
for r in regs:
|
||||
self.mu.reg_write(A.UC_ARM64_REG_X0 + r if r < 29 else A.UC_ARM64_REG_X29, i)
|
||||
if not self._step_to(cmp, br_pc):
|
||||
return None
|
||||
t = self._read(rn)
|
||||
if not (entry <= t < end) or t % 4:
|
||||
return None
|
||||
targets.append(t)
|
||||
except uc.UcError:
|
||||
return None
|
||||
return sorted(set(targets))
|
||||
|
||||
|
||||
_mdd = capstone.Cs(capstone.CS_ARCH_ARM64, capstone.CS_MODE_ARM)
|
||||
_mdd.detail = True
|
||||
CALL_CLOBBERED = set(range(19)) | {30}
|
||||
|
||||
|
||||
def _insn(mod, a):
|
||||
return next(_mdd.disasm(mod.image[a - mod.base:a - mod.base + 4], a), None)
|
||||
|
||||
|
||||
def _regno(name):
|
||||
"""X register number of a capstone register name (w and x views), 31 for sp/zr, None for others."""
|
||||
if name in ('fp', 'x29', 'w29'):
|
||||
return 29
|
||||
if name in ('lr', 'x30', 'w30'):
|
||||
return 30
|
||||
if name in ('sp', 'wsp', 'xzr', 'wzr'):
|
||||
return 31
|
||||
return int(name[1:]) if name[0] in 'xw' and name[1:].isdigit() else None
|
||||
|
||||
|
||||
def _writes(ins):
|
||||
"""X register numbers an instruction writes, by capstone (w and x views folded), plus the call clobbers."""
|
||||
if ins is None:
|
||||
return set(range(31))
|
||||
if ins.mnemonic in ('cmp', 'cmn', 'tst', 'ccmp', 'ccmn'):
|
||||
return set() # capstone lists the first operand of these aliases as written
|
||||
try:
|
||||
_, w = ins.regs_access()
|
||||
except capstone.CsError:
|
||||
return set(range(31))
|
||||
out = set()
|
||||
for r in w:
|
||||
n = _regno(ins.reg_name(r))
|
||||
if n is not None and n < 31:
|
||||
out.add(n)
|
||||
if ins.mnemonic in ('bl', 'blr'):
|
||||
out |= CALL_CLOBBERED
|
||||
return out
|
||||
|
||||
|
||||
class Reaching:
|
||||
"""Reaching definitions of single registers over the resolver's blocks, with capstone's write sets."""
|
||||
|
||||
def __init__(self, mod, cfg):
|
||||
self.mod, self.cfg, self.cache = mod, cfg, {}
|
||||
self.writes = {}
|
||||
for b, stop in zip(cfg.leaders, cfg.stops):
|
||||
for x in range(b, stop, 4):
|
||||
self.writes[x] = _writes(_insn(mod, x))
|
||||
|
||||
def at_blocks(self, reg):
|
||||
"""{block start: definitions of reg reaching it}; 'entry' for a value from outside the function."""
|
||||
if reg in self.cache:
|
||||
return self.cache[reg]
|
||||
cfg = self.cfg
|
||||
gen = {}
|
||||
for b, stop in zip(cfg.leaders, cfg.stops):
|
||||
gen[b] = next((x for x in range(stop - 4, b - 4, -4) if reg in self.writes[x]), None)
|
||||
ins = {b: {'entry'} if b in cfg.unknown else set() for b in cfg.leaders}
|
||||
changed = True
|
||||
while changed:
|
||||
changed = False
|
||||
for b in cfg.leaders:
|
||||
new = set(ins[b])
|
||||
for src, _ in cfg.preds[b]:
|
||||
p = cfg.block(src)
|
||||
new |= {gen[p]} if gen[p] is not None else ins[p]
|
||||
if new != ins[b]:
|
||||
ins[b], changed = new, True
|
||||
self.cache[reg] = ins
|
||||
return ins
|
||||
|
||||
def at(self, reg, a):
|
||||
"""Definitions of reg reaching instruction a."""
|
||||
b = self.cfg.block(a)
|
||||
last = next((x for x in range(a - 4, b - 4, -4) if reg in self.writes[x]), None)
|
||||
return {last} if last is not None else self.at_blocks(reg)[b]
|
||||
|
||||
def values(self, reg, a, active=None):
|
||||
"""Constants reg holds at a, following adrp/add/adr and register copies; None if one is not constant.
|
||||
|
||||
A definition already being evaluated (a cycle of copies) adds no value of its own.
|
||||
"""
|
||||
active = set() if active is None else active
|
||||
out = set()
|
||||
for d in self.at(reg, a):
|
||||
if d == 'entry':
|
||||
return None
|
||||
if (reg, d) in active:
|
||||
continue
|
||||
active.add((reg, d))
|
||||
ins = _insn(self.mod, d)
|
||||
ops = ins.operands
|
||||
if ins.mnemonic in ('adrp', 'adr'):
|
||||
out.add(ops[1].imm)
|
||||
elif ins.mnemonic == 'add' and len(ops) == 3 and ops[2].type == capstone.arm64.ARM64_OP_IMM \
|
||||
and _regno(ins.reg_name(ops[1].reg)) == reg:
|
||||
prev = self.values(reg, d, active)
|
||||
if prev is None:
|
||||
return None
|
||||
imm = ops[2].imm << (ops[2].shift.value if ops[2].shift.type else 0)
|
||||
out |= {v + imm for v in prev}
|
||||
elif ins.mnemonic == 'mov' and len(ops) == 2 and ops[1].type == capstone.arm64.ARM64_OP_REG \
|
||||
and ins.reg_name(ops[1].reg)[0] in 'xlf' and (_regno(ins.reg_name(ops[1].reg)) or 0) < 31:
|
||||
prev = self.values(_regno(ins.reg_name(ops[1].reg)), d, active)
|
||||
if prev is None:
|
||||
return None
|
||||
out |= prev
|
||||
else:
|
||||
return None
|
||||
return out
|
||||
|
||||
|
||||
class FunctionCheck:
|
||||
"""The resolver's final tables and blocks for one function, with reaching definitions on demand."""
|
||||
|
||||
def __init__(self, mod, entry, end):
|
||||
self.mod, self.entry, self.end = mod, entry, end
|
||||
self.tables = {a: t for a, t in jumptables._function(mod, entry, end).items() if t is not None}
|
||||
self.cfg = jumptables._Cfg(mod, entry, end, self.tables, {})
|
||||
self._reaching = self._facts = None
|
||||
|
||||
@property
|
||||
def reaching(self):
|
||||
if self._reaching is None:
|
||||
self._reaching = Reaching(self.mod, self.cfg)
|
||||
return self._reaching
|
||||
|
||||
@property
|
||||
def facts(self):
|
||||
if self._facts is None:
|
||||
self._facts = jumptables._facts(self.cfg) or {}
|
||||
return self._facts
|
||||
|
||||
|
||||
def table_load(mod, s, br):
|
||||
"""Last register-offset load in [s, br): the table read of a dispatch block."""
|
||||
a = br
|
||||
while a > s:
|
||||
a -= 4
|
||||
ins = _insn(mod, a)
|
||||
if ins and ins.mnemonic.startswith('ldr') and len(ins.operands) == 2 \
|
||||
and ins.operands[1].type == capstone.arm64.ARM64_OP_MEM and ins.operands[1].mem.index:
|
||||
return ins
|
||||
return None
|
||||
|
||||
|
||||
def hoisted_base(fc, br):
|
||||
"""(base register, its single constant at the dispatch block or None, verdict) of the table read before br.
|
||||
|
||||
The verdict is 'local' when the dispatch block writes the base, 'verified' when every definition reaching the
|
||||
block (over the resolver's blocks and table edges, with capstone's write sets, through register copies) is an
|
||||
adrp/add or adr of one and the same constant and the resolver's fact does not name another, else a reason.
|
||||
"""
|
||||
mod, s = fc.mod, fc.cfg.block(br)
|
||||
load = table_load(mod, s, br)
|
||||
if load is None:
|
||||
return None, None, 'no table load'
|
||||
reg = _regno(load.reg_name(load.operands[1].mem.base))
|
||||
if reg is None or reg == 31 or any(reg in _writes(_insn(mod, x)) for x in range(s, load.address, 4)):
|
||||
return reg, None, 'local'
|
||||
values = fc.reaching.values(reg, s)
|
||||
if values is None or len(values) != 1:
|
||||
return reg, None, 'not one constant' if values else 'not constant'
|
||||
value = values.pop()
|
||||
fact = fc.facts.get(s, {}).get(reg)
|
||||
if fact is not None and fact != 'z' and fact[0] == 'c' and fact[1] != value:
|
||||
return reg, value, 'facts disagree'
|
||||
return reg, value, 'verified'
|
||||
|
||||
|
||||
def _reads(ins):
|
||||
try:
|
||||
r, _ = ins.regs_access()
|
||||
except capstone.CsError:
|
||||
return set(range(32))
|
||||
return {n for n in (_regno(ins.reg_name(x)) for x in r) if n is not None}
|
||||
|
||||
|
||||
def _branch_target(ins):
|
||||
"""Target of a direct b, b.cond, cbz/cbnz or tbz/tbnz, else None."""
|
||||
if ins is None or not ins.operands or not (ins.mnemonic in ('b', 'cbz', 'cbnz', 'tbz', 'tbnz')
|
||||
or ins.mnemonic.startswith('b.')):
|
||||
return None
|
||||
op = ins.operands[-1]
|
||||
return op.imm if op.type == capstone.arm64.ARM64_OP_IMM else None
|
||||
|
||||
|
||||
def _ways_in(fc, lo, hi, tables):
|
||||
"""Direct branches of the function into [lo, hi], or None when the range is entered otherwise too: by falling
|
||||
through into lo, at the function entry, or as a target of another table."""
|
||||
mod = fc.mod
|
||||
if lo <= fc.entry or any(lo <= t <= hi for ts in tables.values() if ts != 'all' for t in ts):
|
||||
return None
|
||||
prev = _insn(mod, lo - 4)
|
||||
if prev is None or prev.mnemonic not in ('b', 'br', 'ret'):
|
||||
return None
|
||||
out = []
|
||||
for a in range(fc.entry, fc.end, 4):
|
||||
ins = _insn(mod, a)
|
||||
t = _branch_target(ins)
|
||||
if t is not None and lo <= t <= hi:
|
||||
out.append((a, ins, t))
|
||||
return out
|
||||
|
||||
|
||||
# Instructions an index slice may hold: integer arithmetic that reads only general registers and writes the flags at
|
||||
# most (the flags-reading forms adc/sbc/csel/ccmp/... stay out).
|
||||
_FLAG_SETTERS = {'cmp', 'cmn', 'tst', 'ands', 'bics', 'adds', 'subs', 'negs'}
|
||||
_PURE_ALU = {'mov', 'movz', 'movn', 'movk', 'add', 'sub', 'and', 'orr', 'eor', 'bic', 'orn', 'eon', 'lsl', 'lsr',
|
||||
'asr', 'ror', 'ubfx', 'sbfx', 'ubfiz', 'sbfiz', 'bfi', 'bfxil', 'uxtb', 'uxth', 'uxtw', 'sxtb', 'sxth',
|
||||
'sxtw', 'neg', 'mvn', 'mul', 'madd', 'msub', 'udiv', 'sdiv', 'ubfm', 'sbfm', 'bfm'} | _FLAG_SETTERS
|
||||
_DEPTH = 4 # predecessor blocks above the dispatch region
|
||||
|
||||
|
||||
def _ends_block(ins):
|
||||
return ins is None or _branch_target(ins) is not None or ins.mnemonic in ('br', 'ret', 'brk', 'udf')
|
||||
|
||||
|
||||
class _Ways:
|
||||
"""Direct branches and fall-throughs of one function, with the addresses control may enter from elsewhere."""
|
||||
|
||||
def __init__(self, fc, tables):
|
||||
mod = self.mod = fc.mod
|
||||
self.entry, self.into = fc.entry, {}
|
||||
for a in range(fc.entry, fc.end, 4):
|
||||
ins = _insn(mod, a)
|
||||
t = _branch_target(ins)
|
||||
if t is not None and fc.entry <= t < fc.end:
|
||||
self.into.setdefault(t, []).append((a, ins))
|
||||
self.roots = {fc.entry} | set(fc.cfg.unknown)
|
||||
self.roots |= {t for ts in tables.values() if ts != 'all' for t in ts}
|
||||
self.leaders = self.roots | set(self.into)
|
||||
|
||||
def block_start(self, a):
|
||||
s = a
|
||||
while s > self.entry and s not in self.leaders and not _ends_block(_insn(self.mod, s - 4)):
|
||||
s -= 4
|
||||
return s
|
||||
|
||||
def preds(self, s):
|
||||
"""(branch or last instruction, insn, taken) for every way into s."""
|
||||
out = [(a, ins, True) for a, ins in self.into.get(s, ())]
|
||||
if s > self.entry:
|
||||
p = _insn(self.mod, s - 4)
|
||||
if p is not None and p.mnemonic not in ('b', 'br', 'ret', 'brk', 'udf'):
|
||||
out.append((s - 4, p, False))
|
||||
return out
|
||||
|
||||
|
||||
def _edge_holds(oracle, ways, start, src, ins, taken, reg, k):
|
||||
"""Whether the way out of the block [start, src] over the branch at src can be taken with x<reg> = k at src:
|
||||
False only when the branch condition is a function of x<reg> alone (its slice, back to the last write of x<reg>,
|
||||
is plain arithmetic and flag setting on x<reg>), evaluated by Unicorn; True otherwise."""
|
||||
if _branch_target(ins) is None or ins.mnemonic == 'b':
|
||||
return True
|
||||
mod = ways.mod
|
||||
|
||||
def gp_reads(i): # general registers read, zr dropped; None when sp or a non-general register is read
|
||||
try:
|
||||
r, _ = i.regs_access()
|
||||
except capstone.CsError:
|
||||
return None
|
||||
if i.mnemonic in ('cmp', 'cmn', 'tst'): # capstone lists their first operand as written, not read
|
||||
r = list(r) + [op.reg for op in i.operands if op.type == capstone.arm64.ARM64_OP_REG]
|
||||
out = set()
|
||||
for x in r:
|
||||
name = i.reg_name(x)
|
||||
if name in ('xzr', 'wzr', 'nzcv'):
|
||||
continue
|
||||
n = _regno(name)
|
||||
if n is None or n == 31:
|
||||
return None
|
||||
out.add(n)
|
||||
return out
|
||||
|
||||
def sets_flags(i):
|
||||
try:
|
||||
return i.mnemonic in _FLAG_SETTERS or 'nzcv' in (i.reg_name(r) for r in i.regs_access()[1])
|
||||
except capstone.CsError:
|
||||
return True
|
||||
|
||||
if ins.mnemonic.startswith('b.'):
|
||||
setter = next((x for x in range(src - 4, start - 4, -4) if sets_flags(_insn(mod, x))), None)
|
||||
if setter is None or _insn(mod, setter).mnemonic not in _FLAG_SETTERS or any(reg in _writes(_insn(mod, x)) for x in range(setter, src, 4)):
|
||||
return True
|
||||
need, top, sl = gp_reads(_insn(mod, setter)), setter, [setter]
|
||||
else:
|
||||
need, top, sl = gp_reads(ins), src, []
|
||||
if need is None:
|
||||
return True
|
||||
for x in range(top - 4, start - 4, -4):
|
||||
i = _insn(mod, x)
|
||||
w = _writes(i)
|
||||
if reg in w:
|
||||
break # x<reg> is k from here on
|
||||
if not w & need:
|
||||
continue
|
||||
r = gp_reads(i)
|
||||
if i.mnemonic not in _PURE_ALU or r is None:
|
||||
return True
|
||||
sl.append(x)
|
||||
need = (need - w) | r
|
||||
if need - {reg}:
|
||||
return True
|
||||
mu = oracle.mu
|
||||
try:
|
||||
for r in range(29):
|
||||
mu.reg_write(A.UC_ARM64_REG_X0 + r, SCRATCH_MEM + 0x80000)
|
||||
mu.reg_write(A.UC_ARM64_REG_X0 + reg if reg < 29 else A.UC_ARM64_REG_X29, k)
|
||||
for x in reversed(sl):
|
||||
mu.emu_start(x, 0, count=1)
|
||||
mu.emu_start(src, 0, count=1)
|
||||
went = mu.reg_read(A.UC_ARM64_REG_PC) != src + 4
|
||||
except uc.UcError:
|
||||
return True
|
||||
return went == taken
|
||||
|
||||
|
||||
def _way_live(oracle, ways, src, ins, taken, reg, k, depth, seen):
|
||||
"""Whether the way over src can carry x<reg> = k: its edge holds (_edge_holds) and its block either writes x<reg>
|
||||
or is entered with k (_reaches)."""
|
||||
start = ways.block_start(src)
|
||||
if not _edge_holds(oracle, ways, start, src, ins, taken, reg, k):
|
||||
return False
|
||||
if any(reg in _writes(_insn(ways.mod, x)) for x in range(start, src + 4, 4)):
|
||||
return True
|
||||
return _reaches(oracle, ways, start, reg, k, depth, seen)
|
||||
|
||||
|
||||
def _reaches(oracle, ways, s, reg, k, depth, seen):
|
||||
"""Whether control can enter s with x<reg> = k, over at most `depth` predecessor blocks."""
|
||||
if s in ways.roots or depth == 0 or s in seen:
|
||||
return True
|
||||
seen = seen | {s}
|
||||
return any(_way_live(oracle, ways, src, ins, taken, reg, k, depth - 1, seen)
|
||||
for src, ins, taken in ways.preds(s))
|
||||
|
||||
|
||||
def excluded_indices(fc, br, oracle, tables, count):
|
||||
"""Indices < count that cannot reach br: no way into the dispatch region [oracle.start, br] (its first instruction
|
||||
and every branch target inside it) can carry them. The index is the register of the oracle's bounds cmp, which no
|
||||
instruction from such an entry to br may write. Ways in are the module's direct branches and fall-throughs by
|
||||
capstone; the function entry, exception landing pads and every resolver table target count as entries from
|
||||
anywhere. Branch conditions are evaluated only where they depend on the index alone (_edge_holds). When the
|
||||
oracle enters the dispatch block by the taken bounds branch (_taken), that branch must be its only way in."""
|
||||
mod, lo = fc.mod, oracle.start
|
||||
if lo is None:
|
||||
return frozenset()
|
||||
if oracle.guard is not None:
|
||||
ways = _ways_in(fc, oracle.set_at, br, tables)
|
||||
if ways is None or [a for a, _, _ in ways] != [oracle.guard]:
|
||||
return frozenset()
|
||||
reg = bits(oracle.word(oracle.cmp), 5, 5)
|
||||
ways = _Ways(fc, tables)
|
||||
# The region falls through from lo to br (_fallthrough), or is the bounds-check block whose b.ls/b.lo is the only
|
||||
# way into the dispatch block (_taken).
|
||||
hi = br if oracle.guard is None else oracle.cmp
|
||||
if oracle.guard is not None and any(reg in _writes(_insn(mod, x)) for x in range(oracle.set_at, br, 4)):
|
||||
return frozenset()
|
||||
entries = [lo] + [t for t in range(lo + 4, hi + 4, 4) if t in ways.leaders]
|
||||
for e in entries:
|
||||
if e in ways.roots or any(reg in _writes(_insn(mod, x)) for x in range(e, hi, 4)):
|
||||
return frozenset()
|
||||
out = set()
|
||||
for k in range(count):
|
||||
live = False
|
||||
for e in entries:
|
||||
for src, ins, taken in ways.preds(e):
|
||||
if lo <= src < hi:
|
||||
continue # inside the region: its own conditions are ignored
|
||||
if _way_live(oracle, ways, src, ins, taken, reg, k, _DEPTH, frozenset()):
|
||||
live = True
|
||||
break
|
||||
if live:
|
||||
break
|
||||
if not live:
|
||||
out.add(k)
|
||||
return frozenset(out)
|
||||
|
||||
|
||||
def oracle_wrong(fc, br, oracle, ours=None):
|
||||
"""Why the oracle's targets for br are not the table's, from evidence independent of the resolver; or None.
|
||||
|
||||
- the index the table is read with does not depend on the registers the oracle sets (it comes from another
|
||||
register or is reloaded from memory, which the oracle leaves at its scratch value);
|
||||
- the table base the oracle uses (its replay of ADRP/ADR/ADD-imm from the entry) is not the one constant that
|
||||
every reaching definition gives;
|
||||
- the oracle enumerates an index that no way into the dispatch region lets through (excluded_indices: a case
|
||||
peeled off before the table by `cmp wI, #k; b.ne`, `cbnz wI` or a range check on wI, possibly some blocks up),
|
||||
and without it the oracle gives the resolver's targets.
|
||||
"""
|
||||
mod = fc.mod
|
||||
s = fc.cfg.block(br)
|
||||
load = table_load(mod, s, br)
|
||||
if load is None or oracle.set_at is None:
|
||||
return None
|
||||
live = {_regno(load.reg_name(load.operands[1].mem.index))}
|
||||
a = load.address
|
||||
while a > oracle.set_at:
|
||||
a -= 4
|
||||
ins = _insn(mod, a)
|
||||
w = _writes(ins)
|
||||
if live & w:
|
||||
live = (live - w) | _reads(ins)
|
||||
if not live & set(oracle.regs):
|
||||
return 'index not set by the oracle (live-in ' + ', '.join(f'x{r}' for r in sorted(live)) + ')'
|
||||
reg, value, verdict = hoisted_base(fc, br)
|
||||
if verdict == 'verified':
|
||||
oracle.watch = (load.address, reg)
|
||||
oracle.jump_table(br, fc.entry, fc.end)
|
||||
seen, oracle.watch, oracle.seen = oracle.seen, None, None
|
||||
if seen is not None and seen != value:
|
||||
return f'oracle base x{reg} = {seen:#x}, every reaching definition gives {value:#x}'
|
||||
if ours is not None:
|
||||
oracle.jump_table(br, fc.entry, fc.end)
|
||||
ex = excluded_indices(fc, br, oracle, fc.tables, oracle.count) if oracle.count else frozenset()
|
||||
if ex:
|
||||
oracle.exclude = ex
|
||||
try:
|
||||
again = oracle.jump_table(br, fc.entry, fc.end)
|
||||
finally:
|
||||
oracle.exclude = frozenset()
|
||||
if again == ours:
|
||||
return 'index ' + ', '.join(str(k) for k in sorted(ex)) + ' unreachable on every way in'
|
||||
return None
|
||||
|
||||
|
||||
def plt_stubs(mod):
|
||||
"""Addresses of `br x17` in `adrp x16; ldr x17, [x16, #o]; add x16, x16, #o; br x17` stubs."""
|
||||
w = mod.words
|
||||
m = ((w[3:] == 0xD61F0220) & ((w[:-3] & 0x9F00001F) == 0x90000010) & ((w[1:-2] & 0xFFC003FF) == 0xF9400211)
|
||||
& ((w[2:-1] & 0xFFC003FF) == 0x91000210))
|
||||
return (mod.lo + 4 * (np.nonzero(m)[0] + 3)).tolist()
|
||||
|
||||
|
||||
def function_starts(mod):
|
||||
"""Seeds plus BL targets in .text."""
|
||||
w = mod.words.astype(np.int64)
|
||||
pc = mod.lo + 4 * np.arange(len(w), dtype=np.int64)
|
||||
bl = (w & 0xFC000000) == 0x94000000
|
||||
t = pc[bl] + 4 * (((w[bl] & 0x3FFFFFF) ^ 0x2000000) - 0x2000000)
|
||||
starts = set(mod.seeds()) | {a for a in t.tolist() if mod.in_text(a)}
|
||||
return sorted(starts)
|
||||
|
||||
|
||||
def disasm(mod, lo, hi, marks=()):
|
||||
out = []
|
||||
for ins in _md.disasm(mod.image[lo - mod.base:hi - mod.base], lo):
|
||||
tag = '>' if ins.address in marks else ' '
|
||||
out.append(f' {tag}{ins.address - mod.base:8x}: {ins.mnemonic} {ins.op_str}')
|
||||
return out
|
||||
|
||||
|
||||
def window(mod, br, entry, n=24):
|
||||
"""Code before the br, and the bounds check that branches into its block if it sits elsewhere."""
|
||||
lo = max(entry, br - 4 * n)
|
||||
lines = disasm(mod, lo, br + 4, {br})
|
||||
labels = jumptables.branch_sources(mod, lo, br + 4)
|
||||
far = sorted({s for d, ss in labels.items() for s in ss if not lo <= s <= br})
|
||||
for s in far[:3]:
|
||||
lines.append(f' ... reached from {s - mod.base:x}:')
|
||||
lines += disasm(mod, max(entry, s - 16), s + 4, {s})
|
||||
return lines
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument('--exefs', default=str(ROOT / 'build' / 'extracted' / 'v121' / 'exefs'))
|
||||
ap.add_argument('--modules', default='main,sdk,subsdk0')
|
||||
ap.add_argument('--samples', type=int, default=8)
|
||||
ap.add_argument('--no-oracle', action='store_true')
|
||||
args = ap.parse_args()
|
||||
LOG.parent.mkdir(parents=True, exist_ok=True)
|
||||
log = open(LOG, 'w', encoding='utf-8')
|
||||
|
||||
def out(s=''):
|
||||
print(s)
|
||||
log.write(s + '\n')
|
||||
|
||||
bases = {n: b for n, b, _ in layout.layout(args.exefs)}
|
||||
failed = 0
|
||||
for name in args.modules.split(','):
|
||||
mod = Module(name, Path(args.exefs) / name, bases[name])
|
||||
starts = function_starts(mod)
|
||||
brs = (mod.lo + 4 * np.nonzero((mod.words & 0xFFFFFC1F) == 0xD61F0000)[0]).tolist()
|
||||
oracle = None if args.no_oracle else Oracle(mod)
|
||||
stats, t_res, t_orc = Counter(), 0.0, 0.0
|
||||
samples, fc = {}, None
|
||||
rows = open(LOG.with_name(f'jumptables-{name}.tsv'), 'w', encoding='utf-8')
|
||||
rows.write('br\tentry\tend\tcomputed\tresolver\toracle\tcategory\n')
|
||||
for br in brs:
|
||||
i = bisect.bisect_right(starts, br) - 1
|
||||
entry, end = starts[i], starts[i + 1] if i + 1 < len(starts) else mod.hi
|
||||
t0 = time.perf_counter()
|
||||
raw = jumptables.table_targets(mod, br, entry, end)
|
||||
ours = jumptables.resolve(mod, br, entry, end)
|
||||
comp = jumptables.is_computed(mod, br, entry)
|
||||
t1 = time.perf_counter()
|
||||
orc = oracle.jump_table(br, entry, end) if oracle else None
|
||||
t_res += t1 - t0
|
||||
t_orc += time.perf_counter() - t1
|
||||
stats['br'] += 1
|
||||
stats['computed'] += comp
|
||||
if raw is not None and ours is None:
|
||||
stats['targets outside function'] += 1
|
||||
samples.setdefault('targets outside function', []).append((br, entry, end, raw, orc))
|
||||
if (ours is not None or orc is not None) and (fc is None or (fc.entry, fc.end) != (entry, end)):
|
||||
fc = FunctionCheck(mod, entry, end)
|
||||
why = None
|
||||
if ours is not None:
|
||||
stats['resolved'] += 1
|
||||
stats['entries'] += len(ours)
|
||||
reg, _, verdict = hoisted_base(fc, br)
|
||||
stats[f'base {verdict}' if verdict in ('local', 'verified') else 'base not verified'] += 1
|
||||
if verdict not in ('local', 'verified'):
|
||||
samples.setdefault('base not verified', []).append((br, entry, end, ours, orc))
|
||||
rows.write(f'{br - mod.base:x}\t{entry - mod.base:x}\t{end - mod.base:x}\t-\t-\t-\t'
|
||||
f'base x{reg}: {verdict}\n')
|
||||
if not comp:
|
||||
stats['resolved, not computed'] += 1
|
||||
samples.setdefault('resolved, not computed', []).append((br, entry, end, ours, orc))
|
||||
if orc is not None:
|
||||
stats['oracle resolved'] += 1
|
||||
if ours is not None and orc is not None:
|
||||
key = 'agree' if ours == orc else 'disagree'
|
||||
elif ours is not None:
|
||||
key = 'only resolver'
|
||||
elif orc is not None:
|
||||
key = 'only oracle'
|
||||
else:
|
||||
key = 'computed, unresolved' if comp else None
|
||||
if key in ('disagree', 'only oracle'):
|
||||
why = oracle_wrong(fc, br, oracle, ours)
|
||||
if why:
|
||||
key = 'oracle wrong' if key == 'disagree' else 'only oracle, oracle wrong'
|
||||
if key or raw is not None:
|
||||
rows.write(f'{br - mod.base:x}\t{entry - mod.base:x}\t{end - mod.base:x}\t{int(comp)}\t'
|
||||
f'{"-" if raw is None else len(raw)}\t{"-" if orc is None else len(orc)}\t{key}'
|
||||
f'{": " + why if why else ""}\n')
|
||||
if key:
|
||||
stats[key] += 1
|
||||
if key != 'agree':
|
||||
samples.setdefault(key, []).append((br, entry, end, ours if ours is not None else raw, orc, why))
|
||||
rows.close()
|
||||
plt = plt_stubs(mod)
|
||||
stats['PLT stubs'] = len(plt)
|
||||
stats['PLT stubs computed or resolved'] = sum(
|
||||
jumptables.is_computed(mod, a, a - 12) or jumptables.table_targets(mod, a, a - 12, a + 4) is not None
|
||||
for a in plt)
|
||||
out(f'== {name}: {len(starts)} functions')
|
||||
for k in ('br', 'computed', 'resolved', 'entries', 'oracle resolved', 'agree', 'disagree', 'oracle wrong',
|
||||
'only resolver', 'only oracle', 'only oracle, oracle wrong', 'computed, unresolved',
|
||||
'resolved, not computed', 'targets outside function', 'base local', 'base verified',
|
||||
'base not verified', 'PLT stubs', 'PLT stubs computed or resolved'):
|
||||
out(f' {k:32s} {stats[k]}')
|
||||
failed += stats['disagree'] + stats['base not verified'] + stats['PLT stubs computed or resolved']
|
||||
out(f' {"resolver time":32s} {t_res:.2f} s')
|
||||
if oracle:
|
||||
out(f' {"oracle time":32s} {t_orc:.2f} s')
|
||||
for key in ('disagree', 'only oracle', 'base not verified', 'oracle wrong', 'only oracle, oracle wrong',
|
||||
'computed, unresolved', 'resolved, not computed', 'targets outside function', 'only resolver'):
|
||||
for br, entry, end, ours, orc, *why in samples.get(key, [])[:args.samples]:
|
||||
out(f'-- {key}: br {br - mod.base:x} in [{entry - mod.base:x}, {end - mod.base:x})'
|
||||
+ (f': {why[0]}' if why and why[0] else ''))
|
||||
fmt = lambda t: 'None' if t is None else f'{len(t)} [' + ' '.join(
|
||||
f'{a - mod.base:x}' for a in t[:12]) + (' ...' if len(t) > 12 else '') + ']'
|
||||
out(f' resolver {fmt(ours)}')
|
||||
out(f' oracle {fmt(orc)}')
|
||||
for line in window(mod, br, entry):
|
||||
out(line)
|
||||
log.close()
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
178
tests/mods_romfs_check.py
Normal file
178
tests/mods_romfs_check.py
Normal file
@@ -0,0 +1,178 @@
|
||||
"""The mods folder served as RomFS overlays (runtime/romfs.c, runtime/mods.c): load order, added files, case, Unicode.
|
||||
|
||||
.venv\\Scripts\\python tests\\mods_romfs_check.py --exe build\\v121-<tag>\\nx2navi.exe --version v121
|
||||
|
||||
Builds a mods folder in a temp directory with files of its own bytes at real RomFS paths (from
|
||||
build/extracted/<version>/romfs/index.json), runs `nx2navi.exe --probe-fs` on it and reads the `mods:` lines of the log.
|
||||
--probe-fs compares the bytes it reads through the RomFS with every overlay file, so a wrong winner fails it too.
|
||||
A second folder checks the merged resource size table the start writes to the profile (runtime/romfs.c
|
||||
restbl_overlay): a grown file's entry, an added file's, a shipped table's and another version's table's.
|
||||
Needs the extracted RomFS of the version.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
|
||||
import zstandard
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
UNI = 'Ж мод'
|
||||
|
||||
|
||||
def pick_paths(version):
|
||||
index = json.loads((ROOT / 'build/extracted' / version / 'romfs/index.json').read_text())
|
||||
paths = [f['path'] for f in index['files'] if 1024 <= f['size'] <= 65536]
|
||||
have = set(f['path'] for f in index['files'])
|
||||
p1 = next(p for p in paths if any(c.isalpha() for c in p.rsplit('/', 1)[-1]))
|
||||
p2 = next(p for p in paths if p != p1)
|
||||
i = next(k for k, c in enumerate(p1) if c.isalpha() and k > p1.rfind('/'))
|
||||
p1case = p1[:i] + p1[i].swapcase() + p1[i + 1:]
|
||||
assert p1case not in have
|
||||
return p1, p2, p1case
|
||||
|
||||
|
||||
def put(root, mod, rel, data, part='romfs', times=64):
|
||||
f = Path(root) / mod / part / rel
|
||||
f.parent.mkdir(parents=True, exist_ok=True)
|
||||
f.write_bytes(data * times)
|
||||
|
||||
|
||||
def run(exe, mods, order, disabled=''):
|
||||
"""--probe-fs with a profile of its own beside mods (the merged resource size table is written there)."""
|
||||
env = dict(os.environ, NX2NAVI_MODS_DIR=str(mods), NX2NAVI_MODS_ORDER=order, NX2NAVI_MODS_DISABLED=disabled)
|
||||
profile = Path(mods).parent / 'profile'
|
||||
r = subprocess.run([str(exe), '--profile', str(profile), '--probe-fs'], env=env, capture_output=True, timeout=600)
|
||||
log = (r.stdout + r.stderr).decode('utf-8', 'replace')
|
||||
return r.returncode, log
|
||||
|
||||
|
||||
# ---- the resource size table (RESTBL: "RESTBL", u32 version, u32 name length, u32 hashed count, u32 named count,
|
||||
# {u32 crc32, u32 size} sorted, then {char[name length], u32 size})
|
||||
def restbl_read(data):
|
||||
raw = zstandard.ZstdDecompressor().decompress(data)
|
||||
assert raw[:6] == b'RESTBL'
|
||||
_, name_len, hashed, named = struct.unpack_from('<4I', raw, 6)
|
||||
out = dict(struct.unpack_from('<II', raw, 22 + i * 8) for i in range(hashed))
|
||||
hashes = [struct.unpack_from('<I', raw, 22 + i * 8)[0] for i in range(hashed)]
|
||||
assert hashes == sorted(hashes), 'hashed entries out of order'
|
||||
at = 22 + hashed * 8
|
||||
for _ in range(named):
|
||||
out[raw[at:at + name_len].split(b'\0')[0].decode()] = struct.unpack_from('<I', raw, at + name_len)[0]
|
||||
at += name_len + 4
|
||||
return out, raw
|
||||
|
||||
|
||||
def restbl_pack(entries, raw):
|
||||
"""raw (a table) with the sizes of its hashed entries set from entries (crc32 -> size, existing hashes only)."""
|
||||
out = bytearray(raw)
|
||||
hashed = struct.unpack_from('<I', raw, 14)[0]
|
||||
for i in range(hashed):
|
||||
h = struct.unpack_from('<I', raw, 22 + i * 8)[0]
|
||||
if h in entries:
|
||||
struct.pack_into('<I', out, 26 + i * 8, entries[h])
|
||||
return zstandard.ZstdCompressor(write_content_size=True).compress(bytes(out))
|
||||
|
||||
|
||||
def key(path):
|
||||
return zlib.crc32((path[:-3] if path.endswith(('.zs', '.mc')) else path).encode())
|
||||
|
||||
|
||||
def restbl_case(exe, version, check, tmp):
|
||||
romfs = ROOT / 'build/extracted' / version / 'romfs'
|
||||
index = json.loads((romfs / 'index.json').read_text())
|
||||
files = {f['path']: f for f in index['files']}
|
||||
table_path = next(p for p in files if p.startswith('System/Resource/ResourceSizeTable.') and
|
||||
p.endswith('.rsizetable.zs'))
|
||||
with (romfs / 'romfs.bin').open('rb') as image:
|
||||
def read(path, n=None):
|
||||
image.seek(files[path]['offset'])
|
||||
return image.read(n or files[path]['size'])
|
||||
game, game_raw = restbl_read(read(table_path))
|
||||
packs = [p for p in files if p.startswith('Pack/Actor/') and p.endswith('.pack.zs') and key(p) in game]
|
||||
grown, shipped_key, old_key, old_other = packs[0], packs[1], packs[2], packs[3]
|
||||
was = zstandard.get_frame_parameters(read(grown, 18)).content_size
|
||||
zs = zstandard.ZstdCompressor(write_content_size=True)
|
||||
mods = Path(tmp) / 'restbl' / 'mods'
|
||||
put(mods, 'Grow', grown, zs.compress(bytes(was + 100000)), times=1)
|
||||
put(mods, 'Grow', 'Pack/Actor/ModTest_Added.pack.zs', zs.compress(bytes(50000)), times=1)
|
||||
put(mods, 'Ships', table_path, restbl_pack({key(shipped_key): game[key(shipped_key)] + 12345}, game_raw), times=1)
|
||||
put(mods, 'Old', old_key, b'old', times=1) # smaller than the original: only its table raises it
|
||||
put(mods, 'Old', table_path.replace('.Product.', '.Product.999', 1) if '.Product.' in table_path else
|
||||
'System/Resource/ResourceSizeTable.Product.999.Nin_NX_NVN.rsizetable.zs',
|
||||
restbl_pack({key(old_key): game[key(old_key)] + 7777, key(old_other): game[key(old_other)] + 5555},
|
||||
game_raw), times=1)
|
||||
code, log = run(exe, mods, 'Grow|Ships|Old')
|
||||
check(code == 0, 'probe-fs passes with the resource size table mods', log)
|
||||
merged_file = mods.parent / 'profile' / 'mods' / table_path.rsplit('/', 1)[1]
|
||||
check(merged_file.is_file(), f'the merged table is written to the profile ({merged_file.name})', log)
|
||||
if not merged_file.is_file():
|
||||
return
|
||||
merged, _ = restbl_read(merged_file.read_bytes())
|
||||
g = game[key(grown)]
|
||||
check(merged[key(grown)] >= g + 100000, f'a file grown by 100000 bytes: entry {g} -> {merged[key(grown)]}')
|
||||
added = merged.get(key('Pack/Actor/ModTest_Added.pack.zs'), 0)
|
||||
check(added >= 50000 + 256, f'an added 50000-byte pack gets an entry ({added})')
|
||||
check(merged[key(shipped_key)] == game[key(shipped_key)] + 12345, 'a shipped table\'s larger entry is merged')
|
||||
check(merged[key(old_key)] == game[key(old_key)] + 7777, 'another version\'s table raises its own mod\'s file')
|
||||
check(merged[key(old_other)] == game[key(old_other)], 'another version\'s table leaves other files alone')
|
||||
lower = [h for h, s in game.items() if merged.get(h, 0) < s]
|
||||
check(not lower, f'no entry of the game\'s table is lowered ({len(lower)})')
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument('--exe', type=Path, required=True)
|
||||
ap.add_argument('--version', choices=('v121', 'v143'), required=True)
|
||||
args = ap.parse_args()
|
||||
p1, p2, p1case = pick_paths(args.version)
|
||||
fails = 0
|
||||
|
||||
def check(ok, what, log=None):
|
||||
nonlocal fails
|
||||
print(('PASS ' if ok else 'FAIL ') + what)
|
||||
if not ok:
|
||||
fails += 1
|
||||
if log:
|
||||
print('\n'.join(l for l in log.splitlines() if 'mods' in l or 'overlay' in l.lower())[-3000:])
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
mods = Path(tmp) / 'mods'
|
||||
put(mods, 'Top', p1, b'mod-A')
|
||||
put(mods, 'Low', p1, b'mod-B')
|
||||
put(mods, 'Low', p2, b'mod-C')
|
||||
put(mods, 'Top', 'New/added.bin', b'mod-D')
|
||||
put(mods, 'Low', 'New/added.bin', b'mod-E')
|
||||
put(mods, 'Case', p1case, b'mod-F')
|
||||
put(mods, UNI, p2, b'mod-G')
|
||||
# a file of another game version's (no RomFS has a Product.999 table): counted, never served
|
||||
put(mods, 'Low', 'System/Resource/ResourceSizeTable.Product.999.rsizetable.zs', b'mod-H')
|
||||
put(mods, 'Lite', 'Lite/added.bin', b'mod-I', part='romfslite') # a mod manager's romfslite folder
|
||||
|
||||
code, log = run(args.exe, mods, f'Top|Low|{UNI}|Case')
|
||||
check(code == 0, 'probe-fs passes with five mods (served bytes equal the winners\' files)', log)
|
||||
for line in ('mods: Top: 2 files served (1 replaced, 1 added), 0 overridden',
|
||||
'mods: Low: 1 files served (1 replaced, 0 added), 2 overridden by Top, 1 for another game version',
|
||||
'mods: Case: 0 files served (0 replaced, 0 added), 1 overridden by Top',
|
||||
f'mods: {UNI}: 0 files served (0 replaced, 0 added), 1 overridden by Low',
|
||||
'mods: Lite: 1 files served (0 replaced, 1 added), 0 overridden'):
|
||||
check(line in log, line, log)
|
||||
check(p1case not in log, 'the case-flipped path is served as the RomFS path, not added', log)
|
||||
|
||||
code, log = run(args.exe, mods, f'Top|Low|{UNI}|Case', disabled='Top')
|
||||
check(code == 0, 'probe-fs passes with Top switched off', log)
|
||||
check('mods: Low: 3 files served (2 replaced, 1 added), 0 overridden' in log, 'Low serves P1 and the added file', log)
|
||||
check('mods: Top: ' not in log.replace('mods: Top: off', ''), 'a switched-off mod serves nothing', log)
|
||||
|
||||
restbl_case(args.exe, args.version, check, tmp)
|
||||
print('mods_romfs_check:', 'FAIL' if fails else 'ok')
|
||||
return 1 if fails else 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
311
tests/mouse_direct_check.py
Normal file
311
tests/mouse_direct_check.py
Normal file
@@ -0,0 +1,311 @@
|
||||
"""Check the direct camera hook's register contract and raw-count routing without game data.
|
||||
|
||||
Usage: python tests/mouse_direct_check.py --cc <clang.exe>
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
HARNESS = r'''
|
||||
#include <assert.h>
|
||||
#include <stdio.h>
|
||||
#include "mouse_direct.c"
|
||||
static int raw_x, raw_y, takes, captured=1, foreground=1, allowed=1, mode=1, patched=1, selecting;
|
||||
static float ratio=0.5f;
|
||||
static uint64_t now=1000000000;
|
||||
static GuestFn hook, activity_hook, begin_hook, angles_hook, hand_hook;
|
||||
const char *cfg_get(const char *key) { (void)key; return mode ? "direct" : "stick"; }
|
||||
const char *cfg_value(const char *key) { return cfg_get(key); }
|
||||
double cfg_double(const char *key) { (void)key; return 1; }
|
||||
int rt_patch_set(const char *s) { (void)s; return 0; }
|
||||
int rt_patch_enabled(int s) { (void)s; return patched; }
|
||||
int ikb_captured(void) { return captured; }
|
||||
int ikb_foreground(void) { return foreground; }
|
||||
int input_cursor_allowed(void) { return allowed; }
|
||||
int input_direct_mouse_selector_active(void) { return selecting; }
|
||||
void ikb_mouse_take(int32_t *x, int32_t *y) { ++takes; *x=raw_x; *y=raw_y; raw_x=raw_y=0; }
|
||||
void input_direct_mouse_sample(int enabled, int32_t *x, int32_t *y) {
|
||||
*x=*y=0;
|
||||
if(enabled && captured && foreground && allowed) ikb_mouse_take(x,y);
|
||||
}
|
||||
uint64_t plat_now_ns(void) { return now; }
|
||||
int frame_step_realtime(void) { return 1; }
|
||||
double frame_step_limit_hz(void) { return 60; }
|
||||
void *rt_patch_page(void) { return ∶ }
|
||||
void rt_register_host(uint64_t addr, GuestFn fn) {
|
||||
if(addr==HOST_MOUSE_DIRECT) hook=fn; else if(addr==HOST_MOUSE_ACTIVITY) activity_hook=fn;
|
||||
else if(addr==HOST_MOUSE_ORBIT_BEGIN) begin_hook=fn; else if(addr==HOST_MOUSE_ORBIT_ANGLES) angles_hook=fn;
|
||||
else if(addr==HOST_MOUSE_HAND) hand_hook=fn; else abort();
|
||||
}
|
||||
void plat_mutex_lock(PlatMutex *m) { (void)m; }
|
||||
void plat_mutex_unlock(PlatMutex *m) { (void)m; }
|
||||
void rt_log(const char *fmt, ...) { (void)fmt; }
|
||||
static float scalar(__uint128_t v) { float f; memcpy(&f,&v,4); return f; }
|
||||
int main(void) {
|
||||
assert(hook && activity_hook);
|
||||
Cpu c, expected;
|
||||
unsigned char target[0x500] = {0};
|
||||
memset(&c, 0x5a, sizeof c);
|
||||
c.x[20]=(uintptr_t)target; c.x[21]=0; c.v[9]=c.v[10]=0;
|
||||
expected=c;
|
||||
raw_x=100; raw_y=-50;
|
||||
hook(&c);
|
||||
assert(fabsf(scalar(c.v[9])-2)<1e-6f);
|
||||
assert(fabsf(scalar(c.v[10])-1)<1e-6f);
|
||||
float stored; memcpy(&stored,target+0x420,4); assert(stored==scalar(c.v[9]));
|
||||
expected.v[9]=c.v[9]; expected.v[10]=c.v[10]; assert(!memcmp(&expected,&c,sizeof c));
|
||||
assert(input_direct_mouse_active());
|
||||
// The next read sees no repeated movement. Gamepad/keyboard input is preserved.
|
||||
expected=c; hook(&c); assert(!memcmp(&expected,&c,sizeof c));
|
||||
// Equal mouse distance produces equal integrated input at 30, 60 and 144 Hz; there is no unit-circle clamp.
|
||||
float rates[]={30,60,144};
|
||||
for(int i=0;i<3;i++) {
|
||||
ratio=30/rates[i]; raw_x=10000; raw_y=10000; c.v[9]=c.v[10]=0; hook(&c);
|
||||
assert(fabsf(scalar(c.v[9])*ratio-100)<0.001f);
|
||||
assert(fabsf(scalar(c.v[10])*ratio+100)<0.001f);
|
||||
}
|
||||
// One count is not discarded by a dead zone.
|
||||
ratio=0.5f; raw_x=1; raw_y=0; c.v[9]=c.v[10]=0; hook(&c); assert(scalar(c.v[9])>0);
|
||||
uint64_t stack[4]={0,0x123456789abcdef0ull,0x456789abcdef0123ull,0}; c.sp=(uintptr_t)stack;
|
||||
// The flags are in w8 at the site: the game's next instruction stores w8 to +0x418, so memory is not touched.
|
||||
uint32_t inactive=0x28, mem=0x77; memcpy(target+0x418,&mem,4);
|
||||
c.x[8]=inactive; activity_hook(&c);
|
||||
assert((uint32_t)c.x[8]==0x11);
|
||||
uint32_t after; memcpy(&after,target+0x418,4); assert(after==mem);
|
||||
assert(c.v[9]==stack[1] && c.v[8]==stack[2]);
|
||||
// A controller-only update keeps the game's activity bits.
|
||||
c.x[8]=inactive; activity_hook(&c); assert((uint32_t)c.x[8]==inactive);
|
||||
// Disabled game camera input leaves raw counts for the menu's legacy mouse-to-stick path.
|
||||
c.x[21]=1; raw_x=42; int before=takes; hook(&c);
|
||||
assert(takes==before && raw_x==42 && !input_direct_mouse_active()); c.x[21]=0;
|
||||
// Focus loss and host menus do not move the camera.
|
||||
captured=0; before=takes; hook(&c); assert(takes==before); captured=1;
|
||||
foreground=0; hook(&c); assert(takes==before); foreground=1;
|
||||
allowed=0; hook(&c); assert(takes==before); allowed=1;
|
||||
mode=0; hook(&c); assert(takes==before && !input_direct_mouse_active()); mode=1;
|
||||
patched=0; hook(&c); assert(takes==before && !input_direct_mouse_active()); patched=1;
|
||||
now+=150000001; assert(!input_direct_mouse_active());
|
||||
// Orbit input removes the mouse from the stick path and writes equal degrees per count at the final angles.
|
||||
unsigned char camera[0x300]={0}, frame[0x100]={0}, hand[0x300]={0};
|
||||
uint64_t input_address=(uintptr_t)target; memcpy(camera+0x20,&input_address,8);
|
||||
for(int rate=0;rate<3;rate++) {
|
||||
memset(orbit_seen,0,sizeof orbit_seen);
|
||||
ratio=30/rates[rate]; c.x[20]=(uintptr_t)target; c.x[21]=0; c.v[9]=c.v[10]=0;
|
||||
raw_x=100; raw_y=-50; hook(&c);
|
||||
guest_float_store((uintptr_t)camera+0x108,30); guest_float_store((uintptr_t)camera+0x120,40);
|
||||
guest_float_store((uintptr_t)frame+0x44,30); guest_float_store((uintptr_t)frame+0x48,40);
|
||||
c.x[0]=(uintptr_t)camera; begin_hook(&c);
|
||||
assert(guest_float((uintptr_t)target+0x420)==0 && guest_float((uintptr_t)target+0x424)==0);
|
||||
c.x[20]=(uintptr_t)camera; c.sp=(uintptr_t)frame; angles_hook(&c);
|
||||
assert(fabsf(guest_float((uintptr_t)frame+0x44)-26)<0.0001f);
|
||||
assert(fabsf(guest_float((uintptr_t)frame+0x48)-32)<0.0001f);
|
||||
assert(fabsf(guest_float((uintptr_t)camera+0x15c)-26)<0.0001f);
|
||||
c.x[0]=(uintptr_t)camera; begin_hook(&c); angles_hook(&c);
|
||||
assert(fabsf(guest_float((uintptr_t)frame+0x48)-32)<0.0001f); // one sample cannot repeat next update
|
||||
// The next input frame is stationary and the game's parameter update rewrites the old default target.
|
||||
// Restoring the mouse's desired target makes the camera filter stay at the new pitch, not spring to zero.
|
||||
c.x[20]=(uintptr_t)target; c.v[9]=c.v[10]=0; raw_x=raw_y=0; hook(&c);
|
||||
guest_float_store((uintptr_t)camera+0x15c,0);
|
||||
c.x[0]=(uintptr_t)camera; begin_hook(&c);
|
||||
float current=guest_float((uintptr_t)camera+0x108), goal=guest_float((uintptr_t)camera+0x15c);
|
||||
assert(fabsf(goal-26)<0.0001f && fabsf((current+goal)*0.5f-26)<0.0001f);
|
||||
// A later game filter overwrites the pitch after the begin hook. Neither a stationary frame nor the next
|
||||
// tiny mouse delta may integrate from that recentered result.
|
||||
guest_float_store((uintptr_t)camera+0x108,0);
|
||||
guest_float_store((uintptr_t)camera+0x11c,5);
|
||||
guest_float_store((uintptr_t)camera+0x10c,20);
|
||||
guest_float_store((uintptr_t)camera+0x88,0.3f);
|
||||
guest_float_store((uintptr_t)frame+0x44,0);
|
||||
c.x[20]=(uintptr_t)camera; angles_hook(&c);
|
||||
assert(fabsf(guest_float((uintptr_t)frame+0x44)-26)<0.0001f);
|
||||
assert(fabsf(guest_float((uintptr_t)camera+0x15c)-7)<0.0001f);
|
||||
assert(fabsf(guest_float((uintptr_t)camera+0x108)-12)<0.0001f);
|
||||
for(int i=0;i<100;i++) {
|
||||
guest_float_store((uintptr_t)frame+0x44,0); angles_hook(&c);
|
||||
assert(fabsf(guest_float((uintptr_t)camera+0x108)-12)<0.0001f);
|
||||
}
|
||||
c.x[20]=(uintptr_t)target; c.v[9]=c.v[10]=0; raw_y=1; hook(&c);
|
||||
c.x[0]=(uintptr_t)camera; begin_hook(&c);
|
||||
guest_float_store((uintptr_t)frame+0x44,0);
|
||||
c.x[20]=(uintptr_t)camera; angles_hook(&c);
|
||||
assert(fabsf(guest_float((uintptr_t)frame+0x44)-26.08f)<0.0001f);
|
||||
// An explicit controller recenter still hands pitch back to the game.
|
||||
uint64_t recenter=0x20; memcpy(target+0x148,&recenter,8);
|
||||
c.x[20]=(uintptr_t)target; c.v[9]=c.v[10]=0; hook(&c);
|
||||
c.x[0]=(uintptr_t)camera; begin_hook(&c);
|
||||
guest_float_store((uintptr_t)frame+0x44,5);
|
||||
c.x[20]=(uintptr_t)camera; angles_hook(&c);
|
||||
assert(guest_float((uintptr_t)frame+0x44)==5);
|
||||
memset(target+0x148,0,8);
|
||||
memset(camera+0x88,0,4); memset(camera+0x10c,0,4); memset(camera+0x11c,0,4);
|
||||
}
|
||||
// A gamepad-only step (no mouse motion, no pitch hold) leaves the camera input and the angles as the game made them.
|
||||
memset(orbit_seen,0,sizeof orbit_seen);
|
||||
c.x[20]=(uintptr_t)target; c.x[21]=0; raw_x=raw_y=0; c.v[9]=c.v[10]=0;
|
||||
{ float px=0.25f, py=-0.5f; memcpy(&c.v[9],&px,4); memcpy(&c.v[10],&py,4); }
|
||||
hook(&c);
|
||||
guest_float_store((uintptr_t)target+0x420,0.25f); guest_float_store((uintptr_t)target+0x424,0.75f);
|
||||
guest_float_store((uintptr_t)frame+0x44,11); guest_float_store((uintptr_t)frame+0x48,22);
|
||||
guest_float_store((uintptr_t)camera+0x15c,3);
|
||||
c.x[0]=(uintptr_t)camera; begin_hook(&c);
|
||||
assert(guest_float((uintptr_t)target+0x420)==0.25f && guest_float((uintptr_t)target+0x424)==0.75f);
|
||||
c.x[20]=(uintptr_t)camera; c.sp=(uintptr_t)frame; angles_hook(&c);
|
||||
assert(guest_float((uintptr_t)frame+0x44)==11 && guest_float((uintptr_t)frame+0x48)==22);
|
||||
assert(guest_float((uintptr_t)camera+0x15c)==3);
|
||||
guest_float_store((uintptr_t)camera+0x108,30); guest_float_store((uintptr_t)camera+0x120,40);
|
||||
for(int count=0;count<100;count++) {
|
||||
c.x[20]=(uintptr_t)target; c.x[21]=0; c.v[9]=c.v[10]=0; raw_x=1; raw_y=0; hook(&c);
|
||||
c.x[0]=(uintptr_t)camera; begin_hook(&c); c.x[20]=(uintptr_t)camera;
|
||||
guest_float_store((uintptr_t)frame+0x44,guest_float((uintptr_t)camera+0x108));
|
||||
guest_float_store((uintptr_t)frame+0x48,guest_float((uintptr_t)camera+0x120));
|
||||
angles_hook(&c);
|
||||
}
|
||||
assert(fabsf(guest_float((uintptr_t)camera+0x120)-32)<0.001f);
|
||||
// A held object receives mouse deltas after its own curves, once per sample, including with camera input off.
|
||||
c.x[20]=(uintptr_t)target; c.v[9]=c.v[10]=0; raw_x=100; raw_y=50; ratio=0.5f; hook(&c);
|
||||
c.x[19]=(uintptr_t)hand; c.x[8]=0x12345678; c.v[15]=0;
|
||||
guest_float_store((uintptr_t)frame+0x60,0); hand_hook(&c);
|
||||
assert(fabsf(guest_float((uintptr_t)frame+0x60)-2)<0.00001f);
|
||||
assert(fabsf(scalar(c.v[15])+1)<0.00001f);
|
||||
hand_hook(&c); assert(fabsf(scalar(c.v[15])+1)<0.00001f);
|
||||
c.x[21]=1; raw_x=5; raw_y=6; hook(&c); assert(raw_x==0 && raw_y==0 && input_direct_mouse_active());
|
||||
// A radial selector retains its virtual cursor between mouse packets, including repeated HID polls.
|
||||
float cursor[2]={0}; int32_t selector[2], held_selector[2];
|
||||
input_direct_mouse_radial(100,50,1,cursor,selector);
|
||||
held_selector[0]=selector[0]; held_selector[1]=selector[1];
|
||||
for(int i=0;i<100;i++) {
|
||||
input_direct_mouse_radial(0,0,1,cursor,selector);
|
||||
assert(selector[0]==held_selector[0] && selector[1]==held_selector[1]);
|
||||
}
|
||||
input_direct_mouse_radial(-100,-50,1,cursor,selector); assert(!selector[0] && !selector[1]);
|
||||
input_direct_mouse_radial(100000,100000,1,cursor,selector);
|
||||
assert((int64_t)selector[0]*selector[0]+(int64_t)selector[1]*selector[1]<=32768ll*32768);
|
||||
// Quick lists must not latch a wheel direction or automatically drain a large flick after the mouse stops.
|
||||
InputMouseRow row={0}; uint64_t row_time=0;
|
||||
input_direct_mouse_row(10000,1,row_time,&row,selector); assert(selector[0]==32767 && !selector[1]);
|
||||
input_direct_mouse_row(0,1,row_time+10000000,&row,selector); assert(selector[0]==32767);
|
||||
for(int i=1;i<=100;i++) {
|
||||
input_direct_mouse_row(0,1,row_time+i*100000000ull,&row,selector); assert(!selector[0] && !selector[1]);
|
||||
}
|
||||
// Repeated HID polls do not rearm a pulse; a frame that skips its deadline must still observe neutral first.
|
||||
row=(InputMouseRow){0};
|
||||
for(int i=0;i<31;i++) { input_direct_mouse_row(1,1,0,&row,selector); assert(!selector[0]); }
|
||||
input_direct_mouse_row(1,1,0,&row,selector); assert(selector[0]==32767);
|
||||
input_direct_mouse_row(1000,1,100000000,&row,selector); assert(!selector[0]);
|
||||
input_direct_mouse_row(1000,1,110000000,&row,selector); assert(!selector[0]);
|
||||
input_direct_mouse_row(1000,1,140000000,&row,selector); assert(selector[0]==32767);
|
||||
input_direct_mouse_row(-1000,1,141000000,&row,selector); assert(!selector[0]);
|
||||
input_direct_mouse_row(-1,1,180000000,&row,selector); assert(selector[0]==-32767);
|
||||
selecting=1; c.x[20]=(uintptr_t)target; raw_x=42; raw_y=0; before=takes; hook(&c);
|
||||
assert(takes==before && raw_x==42); selecting=0; raw_x=0;
|
||||
puts("direct mouse: register contract, routing, linearity, frame ratio and capture checks passed");
|
||||
puts("direct mouse: final orbit angles, single-count motion, equal axes and held-object routing passed");
|
||||
puts("direct mouse: persistent pitch target, radial cursor hold and gamepad-only steps untouched passed");
|
||||
puts("direct mouse: final-pitch recenter rejection, explicit handoff and bounded quick-list steps passed");
|
||||
}
|
||||
'''
|
||||
|
||||
INPUT_HARNESS = r'''
|
||||
#include <assert.h>
|
||||
#include "input.c"
|
||||
GfxStats gfx_counters;
|
||||
InputMap input_map;
|
||||
// Other input entry points are linked by PE unwind metadata; reaching them is outside this script-only test.
|
||||
void ikb_cursor_update(void) { abort(); }
|
||||
void ikb_menu_closed(void) { abort(); }
|
||||
int cfg_bool(const char *key) { (void)key; abort(); }
|
||||
int input_menu_gate(int *h,int open,uint64_t *b,int32_t s[4]) { (void)h;(void)open;(void)b;(void)s;abort(); }
|
||||
int input_overlay_gate(int *h,uint64_t *b) { (void)h;(void)b;abort(); }
|
||||
int menu_is_open(void) { abort(); }
|
||||
void menu_toggle(void) { abort(); }
|
||||
void menu_perf_cycle(void) { abort(); }
|
||||
const char *input_action_name(int action) { (void)action;abort(); }
|
||||
void input_map_configure(void) { abort(); }
|
||||
int input_direct_mouse_active(void) { return 1; }
|
||||
int input_direct_mouse_enabled(void) { return 1; }
|
||||
static int row_calls, radial_calls, live_mouse;
|
||||
void input_direct_mouse_row(int32_t dx,float sensitivity,uint64_t now,InputMouseRow *row,int32_t out[2]) {
|
||||
(void)dx;(void)sensitivity;(void)now; row_calls++; out[0]=123;out[1]=0;row->direction=1;
|
||||
}
|
||||
void input_direct_mouse_radial(int32_t dx,int32_t dy,float sensitivity,float cursor[2],int32_t out[2]) {
|
||||
(void)dx;(void)dy;(void)sensitivity;radial_calls++;cursor[0]=1;out[0]=456;out[1]=789;
|
||||
}
|
||||
static const char *script_path;
|
||||
static uint64_t test_tick;
|
||||
static int64_t test_world=-1;
|
||||
const char *cfg_get(const char *key) {
|
||||
if (!strcmp(key,"NX2NAVI_INPUT_SCRIPT")) return script_path;
|
||||
if (!strcmp(key,"NX2NAVI_INPUT")) return "none";
|
||||
return NULL;
|
||||
}
|
||||
unsigned cfg_version(void) { return 0; }
|
||||
double cfg_double(const char *key) { (void)key; return 1; }
|
||||
uint64_t frame_step_ticks(void) { return test_tick; }
|
||||
int64_t frame_step_world_us(void) { return test_world; }
|
||||
void rt_log(const char *fmt, ...) { (void)fmt; }
|
||||
void plat_mutex_lock(PlatMutex *m) { (void)m; }
|
||||
void plat_mutex_unlock(PlatMutex *m) { (void)m; }
|
||||
int ikb_captured(void) { return live_mouse; }
|
||||
int ikb_foreground(void) { return 0; }
|
||||
void ikb_mouse_take(int32_t *x,int32_t *y) { *x=*y=0; }
|
||||
uint64_t plat_now_ns(void) { return 1000000000ull; }
|
||||
int main(int argc,char **argv) {
|
||||
assert(argc==2); script_path=argv[1];
|
||||
uint64_t buttons=0; int32_t left[2]={0},right[2]={0},dx,dy;
|
||||
test_tick=10;
|
||||
for(int i=0;i<3;i++) script_apply(&buttons,left,right);
|
||||
input_direct_mouse_sample(1,&dx,&dy);
|
||||
assert(dx==123 && dy==-45 && !buttons && !right[0] && !right[1]);
|
||||
input_direct_mouse_sample(1,&dx,&dy); assert(!dx && !dy);
|
||||
test_tick=20; script_apply(&buttons,left,right);
|
||||
g_menu=1; input_direct_mouse_sample(1,&dx,&dy); assert(!dx && !dy);
|
||||
g_menu=0; input_direct_mouse_sample(1,&dx,&dy); assert(!dx && !dy);
|
||||
test_world=100000;
|
||||
for(int i=0;i<3;i++) script_apply(&buttons,left,right);
|
||||
rec=tmpfile(); assert(rec); rec_state=1;
|
||||
input_direct_mouse_sample(1,&dx,&dy); assert(dx==-100 && dy==50);
|
||||
rewind(rec); char line[128]; assert(fgets(line,sizeof line,rec));
|
||||
assert(strstr(line,"MOUSE -100 50")); assert(!fgets(line,sizeof line,rec));
|
||||
fclose(rec); rec_state=-1;
|
||||
live_mouse=1;
|
||||
mouse_camera(right,1); assert(radial_calls==1 && !row_calls && right[0]==456 && right[1]==789);
|
||||
assert(input_direct_mouse_selector_active());
|
||||
mouse_camera(right,2); assert(row_calls==1 && radial_calls==1 && right[0]==123 && !right[1]);
|
||||
mouse_camera(right,0); assert(!input_direct_mouse_selector_active() && !right[0] && !right[1]);
|
||||
mouse_camera(right,3); assert(row_calls==2 && right[0]==123 && !right[1]);
|
||||
live_mouse=0; mouse_camera(right,3);
|
||||
assert(!input_direct_mouse_selector_active() && !right[0] && !right[1]);
|
||||
puts("direct mouse: one-shot script replay, host-menu blocking and raw-count recording passed");
|
||||
puts("direct mouse: separate wheel/list routing, selector ownership and release passed");
|
||||
}
|
||||
'''
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument('--cc', required=True)
|
||||
ap.add_argument('--deps', type=Path, help='dependency include directory (SDL3/Vulkan), to also check input scripts')
|
||||
args = ap.parse_args()
|
||||
with tempfile.TemporaryDirectory(prefix='nx2navi-mouse-') as tmp:
|
||||
folder = Path(tmp)
|
||||
(folder / 'patches.h').write_text('#define RT_PATCH_PAGE 0x7003adf000ull\n')
|
||||
source = folder / 'check.c'
|
||||
source.write_text(HARNESS)
|
||||
exe = folder / ('check.exe' if os.name == 'nt' else 'check')
|
||||
link = [] if os.name == 'nt' else ['-lm']
|
||||
subprocess.run([args.cc, '-O2', '-Wall', '-Wextra', '-I', str(folder), '-I', str(ROOT/'runtime'),
|
||||
str(source), *link, '-o', str(exe)], check=True)
|
||||
subprocess.run([str(exe)], check=True)
|
||||
if args.deps:
|
||||
source.write_text(INPUT_HARNESS)
|
||||
script = folder / 'input.txt'
|
||||
script.write_text('10 MOUSE 123 -45\n20 MOUSE 1000 0\n@100 MOUSE -100 50\n')
|
||||
subprocess.run([args.cc, '-O2', '-flto', '-ffunction-sections', '-fdata-sections', '-Wl,--gc-sections', *(['-fuse-ld=lld', '-Wl,--exclude-all-symbols'] if os.name == 'nt' else []),
|
||||
'-I', str(ROOT/'runtime'), '-I', str(args.deps), '-I', str(args.deps/'vma'),
|
||||
str(source), *link, '-o', str(exe)], check=True)
|
||||
subprocess.run([str(exe), str(script)], check=True)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
173
tests/ncz_test.py
Normal file
173
tests/ncz_test.py
Normal file
@@ -0,0 +1,173 @@
|
||||
"""tools/ncz.py and the installer's unpacking of NSZ input (installer/steps.py, Check.unpack) on small made-up files:
|
||||
an NCA with an AES-CTR section and a plain one, stored as an NCZ with one zstd stream or as NCZBLOCK blocks.
|
||||
|
||||
Run: .venv-py314\\Scripts\\python tests/ncz_test.py
|
||||
"""
|
||||
import hashlib
|
||||
import io
|
||||
import os
|
||||
import struct
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from compression import zstd
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(ROOT))
|
||||
sys.path.insert(0, str(ROOT / 'tools'))
|
||||
import ncz # noqa: E402
|
||||
from installer import steps # noqa: E402
|
||||
from installer.events import Reporter # noqa: E402
|
||||
|
||||
KEY, COUNTER = bytes(range(16)), bytes(range(16, 32))
|
||||
# (offset, size, crypto type): an AES-CTR section, then a plain one
|
||||
SECTIONS = ((0x4000, 0x9000, 3), (0xD000, 0x3200, 1))
|
||||
|
||||
|
||||
def pfs0(files):
|
||||
"""An NSP of (name, data) pairs."""
|
||||
strings = b''.join(name.encode() + b'\0' for name, _ in files)
|
||||
strings += bytes(-(16 + 0x18 * len(files) + len(strings)) % 0x20)
|
||||
head = b'PFS0' + struct.pack('<III', len(files), len(strings), 0)
|
||||
offset = name_at = 0
|
||||
for name, data in files:
|
||||
head += struct.pack('<QQII', offset, len(data), name_at, 0)
|
||||
offset += len(data)
|
||||
name_at += len(name) + 1
|
||||
return head + strings + b''.join(data for _, data in files)
|
||||
|
||||
|
||||
def made_up_nca():
|
||||
"""(the NCA as stored on the console, its body after the header with the sections decrypted)."""
|
||||
rand = os.urandom
|
||||
plain = rand(0x2000) + bytes(0x7000) # half random, half compressible
|
||||
rest = bytes(0x1200) + rand(0x2000)
|
||||
counter = COUNTER[:8] + struct.pack('>Q', 0x4000 >> 4)
|
||||
secret = Cipher(algorithms.AES(KEY), modes.CTR(counter)).encryptor().update(plain)
|
||||
header = rand(0x4000)
|
||||
return header + secret + rest, plain + rest
|
||||
|
||||
|
||||
def ncz_file(nca, body, blocks=False):
|
||||
out = nca[:0x4000] + b'NCZSECTN' + struct.pack('<Q', len(SECTIONS))
|
||||
for offset, size, crypto in SECTIONS:
|
||||
out += struct.pack('<QQQQ', offset, size, crypto, 0) + KEY + COUNTER
|
||||
if not blocks:
|
||||
return out + zstd.compress(body)
|
||||
exponent = 14
|
||||
pieces = [body[i:i + (1 << exponent)] for i in range(0, len(body), 1 << exponent)]
|
||||
stored = [zstd.compress(p) if p.count(0) > len(p) // 2 else p for p in pieces] # random blocks stay as they are
|
||||
out += b'NCZBLOCK' + bytes([2, 1, 0, exponent]) + struct.pack('<IQ', len(pieces), len(body))
|
||||
out += struct.pack(f'<{len(stored)}I', *map(len, stored))
|
||||
return out + b''.join(stored)
|
||||
|
||||
|
||||
class UnpackTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.dir = Path(self.tmp.name)
|
||||
self.nca, self.body = made_up_nca()
|
||||
self.name = hashlib.sha256(self.nca).hexdigest()[:32]
|
||||
self.tik = b'ticket' * 40
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def nsz(self, blocks=False, damage=None):
|
||||
data = ncz_file(self.nca, self.body, blocks)
|
||||
if damage:
|
||||
data = damage(bytearray(data))
|
||||
path = self.dir / 'game.nsz'
|
||||
path.write_bytes(pfs0([(self.name + '.ncz', bytes(data)), ('a.tik', self.tik)]))
|
||||
return path
|
||||
|
||||
def expected(self):
|
||||
return pfs0([(self.name + '.nca', self.nca), ('a.tik', self.tik)])
|
||||
|
||||
def test_stream_comes_back_byte_for_byte(self):
|
||||
src = self.nsz()
|
||||
self.assertEqual(ncz.compressed(src)[0], 'nsz')
|
||||
self.assertEqual(ncz.unpacked_size(src), len(self.expected()))
|
||||
seen = []
|
||||
self.assertEqual(ncz.unpack(src, self.dir / 'out.nsp', lambda d, t: seen.append((d, t))), 'nsz')
|
||||
self.assertEqual((self.dir / 'out.nsp').read_bytes(), self.expected())
|
||||
self.assertEqual(seen[-1], (len(self.expected()), len(self.expected())))
|
||||
self.assertFalse((self.dir / 'out.nsp.part').exists())
|
||||
|
||||
def test_blocks_come_back_byte_for_byte(self):
|
||||
ncz.unpack(self.nsz(blocks=True), self.dir / 'out.nsp')
|
||||
self.assertEqual((self.dir / 'out.nsp').read_bytes(), self.expected())
|
||||
|
||||
def test_plain_nsp_is_not_compressed(self):
|
||||
path = self.dir / 'plain.nsp'
|
||||
path.write_bytes(self.expected())
|
||||
self.assertIsNone(ncz.compressed(path))
|
||||
|
||||
def test_changed_byte_fails_the_hash(self):
|
||||
def flip(data):
|
||||
data[0x4000 + 16 + 0x40 * len(SECTIONS) + 0x100] ^= 1 # inside a stored (random) block
|
||||
return data
|
||||
with self.assertRaises(ncz.DamagedFile) as e:
|
||||
ncz.unpack(self.nsz(blocks=True, damage=flip), self.dir / 'out.nsp')
|
||||
self.assertIn('hash mismatch', e.exception.detail)
|
||||
|
||||
def test_cut_stream_is_damaged(self):
|
||||
with self.assertRaises(ValueError):
|
||||
ncz.unpack(self.nsz(damage=lambda d: d[:-40]), self.dir / 'out.nsp')
|
||||
self.assertFalse((self.dir / 'out.nsp').exists())
|
||||
|
||||
|
||||
class CheckUnpackTest(unittest.TestCase):
|
||||
"""Check.unpack: an NSZ base becomes <temp>/unpacked/base.nsp, a plain NSP update is read in place, a second run
|
||||
reuses the NSP."""
|
||||
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
base = Path(self.tmp.name)
|
||||
nca, body = made_up_nca()
|
||||
name = hashlib.sha256(nca).hexdigest()[:32]
|
||||
self.base, self.update = base / 'base.nsz', base / 'update.nsp'
|
||||
self.base.write_bytes(pfs0([(name + '.ncz', ncz_file(nca, body))]))
|
||||
self.update.write_bytes(pfs0([('a.tik', b'x' * 64)]))
|
||||
self.expected = pfs0([(name + '.nca', nca)])
|
||||
self.out = io.StringIO()
|
||||
self.ctx = SimpleNamespace(options=SimpleNamespace(base=str(self.base), update=str(self.update)),
|
||||
temp=base / 'temp', reporter=Reporter(out=self.out, log_path=base / 'x.log'))
|
||||
|
||||
def tearDown(self):
|
||||
self.ctx.reporter.close()
|
||||
self.tmp.cleanup()
|
||||
|
||||
def test_unpacks_the_base_and_reuses_it(self):
|
||||
steps.Check().unpack(self.ctx)
|
||||
target = self.ctx.temp / 'unpacked' / 'base.nsp'
|
||||
self.assertEqual(target.read_bytes(), self.expected)
|
||||
self.assertEqual(steps.game_file(self.ctx, 'base'), target)
|
||||
self.assertEqual(steps.game_file(self.ctx, 'update'), self.update)
|
||||
self.assertIn('"phase": "unpack_base"', self.out.getvalue())
|
||||
mtime = target.stat().st_mtime_ns
|
||||
steps.Check().unpack(self.ctx)
|
||||
self.assertEqual(target.stat().st_mtime_ns, mtime)
|
||||
|
||||
def test_content_decides_not_the_name(self):
|
||||
renamed = self.base.with_name('base.nsp')
|
||||
self.base.rename(renamed)
|
||||
self.ctx.options.base = str(renamed)
|
||||
steps.Check().unpack(self.ctx)
|
||||
self.assertEqual((self.ctx.temp / 'unpacked' / 'base.nsp').read_bytes(), self.expected)
|
||||
|
||||
def test_damaged_base_names_the_chosen_file(self):
|
||||
data = bytearray(self.base.read_bytes())
|
||||
data[-100] ^= 0xFF
|
||||
self.base.write_bytes(bytes(data))
|
||||
with self.assertRaises(steps.InstallError) as e:
|
||||
steps.Check().unpack(self.ctx)
|
||||
self.assertIn(e.exception.id, ('check.input_damaged', 'check.unpack_failed'))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
111
tests/pack_shader_cache_test.py
Normal file
111
tests/pack_shader_cache_test.py
Normal file
@@ -0,0 +1,111 @@
|
||||
"""tools/pack_shader_cache.py on a fake shader cache: the pack layout gfx_shader.c reads (sorted index, bounds, data),
|
||||
loose files folded in and deleted, a second run merging new loose files, a loose file winning over its packed copy,
|
||||
malformed files skipped and --keep. No game files; stdlib only.
|
||||
|
||||
Run: python tests/pack_shader_cache_test.py
|
||||
"""
|
||||
import hashlib
|
||||
import struct
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(ROOT / 'tools'))
|
||||
import pack_shader_cache as psc # noqa: E402
|
||||
|
||||
USE = struct.pack('<5I', 0x4d554b54, 1, 3, 0, 1) # gfx_shader.c use_store
|
||||
|
||||
|
||||
def name(seed):
|
||||
return hashlib.sha256(seed.encode()).hexdigest()
|
||||
|
||||
|
||||
def read_pack(path):
|
||||
"""{(kind, name hex): bytes}, checking what gfx_shader.c pack_open checks."""
|
||||
b = path.read_bytes()
|
||||
assert b[:8] == psc.MAGIC
|
||||
count = struct.unpack_from('<I', b, 8)[0]
|
||||
out, last = {}, None
|
||||
for i in range(count):
|
||||
nm, kind, size, offset = struct.unpack_from('<32sIIQ', b, 16 + 48 * i)
|
||||
assert offset + size <= len(b) and offset >= 16 + 48 * count
|
||||
assert last is None or (last[0], last[1]) < (kind, nm), 'index not sorted by (kind, name)'
|
||||
last = (kind, nm)
|
||||
out[(kind, nm.hex())] = b[offset:offset + size]
|
||||
return out
|
||||
|
||||
|
||||
class PackTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.cache = Path(self.tmp.name) / 'shader-cache'
|
||||
(self.cache / 'in').mkdir(parents=True)
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def shader(self, seed, spv=b'\x03\x02\x23\x07spirv', res=b'{"x":1}'):
|
||||
k = name(seed)
|
||||
(self.cache / f'{k}.spv').write_bytes(spv + seed.encode())
|
||||
(self.cache / f'{k}.res.json').write_bytes(res)
|
||||
(self.cache / f'{name("use" + seed)}.use').write_bytes(USE)
|
||||
(self.cache / 'in').mkdir(exist_ok=True) # in_store makes it again after a pack removed it
|
||||
(self.cache / 'in' / f'{name("in" + seed)}.in').write_bytes(b'TKSI' + seed.encode())
|
||||
return k
|
||||
|
||||
def test_pack_holds_every_file_and_deletes_the_loose_ones(self):
|
||||
keys = [self.shader(str(i)) for i in range(50)]
|
||||
(self.cache / 'notes.txt').write_text('kept')
|
||||
(self.cache / f'{keys[0]}.code').write_bytes(b'kept too')
|
||||
n, new, skipped, _ = psc.pack(self.cache)
|
||||
self.assertEqual((n, new, skipped), (200, 200, 0))
|
||||
got = read_pack(self.cache / psc.PACK)
|
||||
self.assertEqual(got[(1, keys[3])], b'\x03\x02\x23\x07spirv3')
|
||||
self.assertEqual(got[(2, keys[3])], b'{"x":1}')
|
||||
self.assertEqual(got[(3, name('use3'))], USE)
|
||||
self.assertEqual(got[(4, name('in3'))], b'TKSI3')
|
||||
left = sorted(p.name for p in self.cache.iterdir())
|
||||
self.assertEqual(left, sorted([psc.PACK, 'notes.txt', f'{keys[0]}.code'])) # in/ removed when empty
|
||||
|
||||
def test_second_run_merges_new_loose_files_and_loose_wins(self):
|
||||
a = self.shader('a')
|
||||
psc.pack(self.cache)
|
||||
b = self.shader('b')
|
||||
(self.cache / f'{a}.res.json').write_bytes(b'{"newer":1}')
|
||||
n, new, _, _ = psc.pack(self.cache)
|
||||
got = read_pack(self.cache / psc.PACK)
|
||||
self.assertEqual(n, 8)
|
||||
self.assertEqual(new, 5)
|
||||
self.assertEqual(got[(2, a)], b'{"newer":1}')
|
||||
self.assertEqual(got[(1, b)], b'\x03\x02\x23\x07spirvb')
|
||||
|
||||
def test_malformed_files_are_skipped_or_replaced_by_the_packed_copy(self):
|
||||
a = self.shader('a')
|
||||
psc.pack(self.cache)
|
||||
(self.cache / f'{name("usea")}.use').write_bytes(b'short') # bad loose copy: the packed one stays
|
||||
(self.cache / f'{name("bad")}.use').write_bytes(b'short') # bad and nowhere else: skipped
|
||||
(self.cache / f'{name("empty")}.spv').write_bytes(b'')
|
||||
n, _, skipped, _ = psc.pack(self.cache)
|
||||
got = read_pack(self.cache / psc.PACK)
|
||||
self.assertEqual(got[(3, name('usea'))], USE)
|
||||
self.assertNotIn((3, name('bad')), got)
|
||||
self.assertEqual((n, skipped), (4, 2))
|
||||
self.assertTrue((self.cache / f'{name("bad")}.use').exists()) # never deleted unless packed
|
||||
|
||||
def test_keep_leaves_the_loose_files(self):
|
||||
k = self.shader('a')
|
||||
psc.pack(self.cache, keep=True)
|
||||
self.assertTrue((self.cache / f'{k}.spv').exists())
|
||||
self.assertEqual(len(read_pack(self.cache / psc.PACK)), 4)
|
||||
|
||||
def test_a_broken_pack_is_rebuilt_from_the_loose_files(self):
|
||||
self.shader('a')
|
||||
(self.cache / psc.PACK).write_bytes(b'garbage')
|
||||
n, _, _, _ = psc.pack(self.cache)
|
||||
self.assertEqual(n, 4)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
590
tests/romfs_prompts_test.py
Normal file
590
tests/romfs_prompts_test.py
Normal file
@@ -0,0 +1,590 @@
|
||||
"""tools/make_romfs_patches.py --prompts and tools/button_icons.py on a RomFS made here, with no game files: each prompt
|
||||
names the label of the Xbox button that the pad layout presses, the shoulder buttons, triggers, Plus, Minus and the
|
||||
d-pad are drawn as on an Xbox controller, the PC prompts draw every button as its key or mouse button (the defaults or
|
||||
a nx2navi.ini's bind_ lines), the resource size table grows with the fonts, every other file and member stays byte for
|
||||
byte, the same input gives the same output, and a RomFS the tool does not recognise fails before it writes anything. Both game versions: 1.2.1 (Common.Product.110, 72x72 button guide images) and 1.4.3
|
||||
(Common.Product.140, 108x108).
|
||||
|
||||
Run: .venv-py314\\Scripts\\python tests\\romfs_prompts_test.py
|
||||
"""
|
||||
import compression.zstd as zstd
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
import random
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
|
||||
from fontTools.fontBuilder import FontBuilder
|
||||
from fontTools.pens.recordingPen import RecordingPen
|
||||
from fontTools.pens.t2CharStringPen import T2CharStringPen
|
||||
from fontTools.ttLib import TTFont
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
TOOL = ROOT / 'tools' / 'make_romfs_patches.py'
|
||||
spec = importlib.util.spec_from_file_location('button_icons', ROOT / 'tools' / 'button_icons.py')
|
||||
icons = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(icons)
|
||||
|
||||
# The face button each Switch button is in a pad layout (docs/controls.md, runtime/input_map.c), the label an Xbox pad
|
||||
# prints on it, and the layout each overlay is made for (runtime/input_map.c input_prompts_folder).
|
||||
POSITION = {'nintendo': {'A': 'east', 'B': 'south', 'X': 'north', 'Y': 'west'},
|
||||
'xbox': {'A': 'south', 'B': 'east', 'X': 'west', 'Y': 'north'}}
|
||||
XBOX_LABEL = {'south': 'A', 'east': 'B', 'west': 'X', 'north': 'Y'}
|
||||
STYLE_LAYOUT = {'xbox': 'nintendo', 'xbox-label': 'xbox'}
|
||||
FACE = {'A': 0xE040, 'B': 0xE041, 'X': 0xE042, 'Y': 0xE043}
|
||||
# The glyphs drawn as Xbox icons, with their advance: LB RB LT RT, Menu and View, the d-pad.
|
||||
DRAWN = {0xE044: 820, 0xE045: 820, 0xE046: 760, 0xE047: 760, 0xE04B: 640, 0xE04C: 640, 0xE049: 640,
|
||||
**{cp: 640 for cp in range(0xE060, 0xE066)}}
|
||||
|
||||
# restbl: the resource size table's name after its version, in both forms the tool takes.
|
||||
VERSIONS = {'v121': dict(common='110', size=72, newest=121, restbl='Nin_NX_NVN.rsizetable.zs'),
|
||||
'v143': dict(common='140', size=108, newest=143, restbl='rsizetable.zs')}
|
||||
DEFAULT_FONT, REGIONAL_FONT = 'Font/Font.Nin_NX_NVN.bfarc.zs', 'Font/Font_KRko.Nin_NX_NVN.bfarc.zs'
|
||||
PANES = {'T_A_00': 'A', 'T_B_00': 'B', 'T_X_00': 'X', 'T_Y_00': 'Y', 'T_Name_00': 'A'}
|
||||
MESSAGES = {**PANES, 'Greeting': 'Hello'}
|
||||
R8_UNORM, BC1 = 0x201, 0x1a01
|
||||
IMAGE = 16384 # an R8 square up to 128 rows, two GOBs wide, block height 16: the size of the game's key images
|
||||
|
||||
|
||||
# ---- the formats, written here from their layout so the tools' readers are checked against an independent writer
|
||||
def sarc(members, align=0x80):
|
||||
names, sfnt, name_at = list(members), b'', {}
|
||||
for name in names:
|
||||
name_at[name] = len(sfnt) // 4
|
||||
sfnt += name.encode() + b'\0'
|
||||
sfnt += bytes(-len(sfnt) % 4)
|
||||
head = 0x14 + 12 + 16 * len(names) + 8 + len(sfnt)
|
||||
start = head + -head % align
|
||||
data, nodes = b'', b''
|
||||
for name in names:
|
||||
data += bytes(-len(data) % align)
|
||||
nodes += struct.pack('<4I', zlib.crc32(name.encode()), 0x01000000 | name_at[name], len(data), len(data) + len(members[name]))
|
||||
data += members[name]
|
||||
out = (b'SARC' + struct.pack('<HHIIHH', 0x14, 0xFEFF, start + len(data), start, 0x100, 0)
|
||||
+ b'SFAT' + struct.pack('<HHI', 12, len(names), 0x65) + nodes + b'SFNT' + struct.pack('<HH', 8, 0) + sfnt)
|
||||
return out + bytes(start - len(out)) + data
|
||||
|
||||
|
||||
def sarc_files(data):
|
||||
header, start = struct.unpack_from('<H', data, 4)[0], struct.unpack_from('<I', data, 12)[0]
|
||||
count, out = struct.unpack_from('<H', data, header + 6)[0], {}
|
||||
names = header + 12 + count * 16 + 8
|
||||
for i in range(count):
|
||||
_, attribute, lo, hi = struct.unpack_from('<4I', data, header + 12 + i * 16)
|
||||
at = names + (attribute & 0xffffff) * 4
|
||||
out[data[at:data.index(0, at)].decode()] = data[start + lo:start + hi]
|
||||
return out
|
||||
|
||||
|
||||
def bflan(panes):
|
||||
"""A FLAN whose pai1 section steps each pane's visibility from 0 at frame 0 to 1 at frame 1."""
|
||||
entries = [name.encode().ljust(28, b'\0') + bytes([1, 0, 0, 0]) + struct.pack('<I', 36)
|
||||
+ b'FLVI' + bytes([1, 0, 0, 0]) + struct.pack('<I', 12)
|
||||
+ bytes([0, 0, 1, 0]) + struct.pack('<HHI', 2, 0, 12)
|
||||
+ struct.pack('<fHxx', 0.0, 0) + struct.pack('<fHxx', 1.0, 1) for name in panes]
|
||||
table, body = 20, b''
|
||||
offsets = []
|
||||
for e in entries:
|
||||
offsets.append(table + 4 * len(entries) + len(body))
|
||||
body += e
|
||||
section = struct.pack('<HBxHHI', 0, 0, 0, len(entries), table) + struct.pack(f'<{len(entries)}I', *offsets) + body
|
||||
section = b'pai1' + struct.pack('<I', 8 + len(section)) + section
|
||||
return b'FLAN' + struct.pack('<HHIIHH', 0xFEFF, 0x14, 0x08000000, 0x14 + len(section), 1, 0) + section
|
||||
|
||||
|
||||
def visibility(data):
|
||||
"""pane -> [(frame, value)] of a FLAN made by bflan()."""
|
||||
pos, out = 0x14, {}
|
||||
count, table = struct.unpack_from('<H', data, pos + 14)[0], pos + struct.unpack_from('<I', data, pos + 16)[0]
|
||||
for e in range(count):
|
||||
eo = pos + struct.unpack_from('<I', data, table + 4 * e)[0]
|
||||
keys = eo + 36 + 12 + 12
|
||||
out[data[eo:eo + 28].split(b'\0')[0].decode()] = [struct.unpack_from('<fH', data, keys + 8 * k) for k in range(2)]
|
||||
return out
|
||||
|
||||
|
||||
def bntx(textures):
|
||||
"""A BNTX of single-mip square textures: name -> (format, size, image)."""
|
||||
out = bytearray(0x30)
|
||||
out[0:4], out[0x20:0x24] = b'BNTX', b'NX '
|
||||
table = len(out)
|
||||
out += bytes(8 * len(textures))
|
||||
struct.pack_into('<IQ', out, 0x24, len(textures), table)
|
||||
for k, (name, (fmt, size, image)) in enumerate(textures.items()):
|
||||
info = len(out)
|
||||
out += bytes(0x80)
|
||||
out[info:info + 4] = b'BRTI'
|
||||
struct.pack_into('<Q', out, table + 8 * k, info)
|
||||
struct.pack_into('<H', out, info + 0x16, 1)
|
||||
struct.pack_into('<I', out, info + 0x1c, fmt)
|
||||
struct.pack_into('<II', out, info + 0x24, size, size)
|
||||
struct.pack_into('<I', out, info + 0x34, 4) # block height 1 << 4 GOBs
|
||||
struct.pack_into('<I', out, info + 0x50, len(image))
|
||||
struct.pack_into('<Q', out, info + 0x60, len(out))
|
||||
out += struct.pack('<H', len(name)) + name.encode() + b'\0'
|
||||
out += bytes(-len(out) % 8)
|
||||
mips = len(out)
|
||||
struct.pack_into('<Q', out, info + 0x70, mips)
|
||||
out += bytes(8)
|
||||
out += bytes(-len(out) % 0x200)
|
||||
struct.pack_into('<Q', out, mips, len(out))
|
||||
out += image
|
||||
return bytes(out)
|
||||
|
||||
|
||||
def bntx_images(data):
|
||||
"""name without the ^ suffix -> (format, size, image) of a BNTX made by bntx()."""
|
||||
count, table = struct.unpack_from('<IQ', data, 0x24)
|
||||
out = {}
|
||||
for k in range(count):
|
||||
info = struct.unpack_from('<Q', data, table + 8 * k)[0]
|
||||
at = struct.unpack_from('<Q', data, info + 0x60)[0]
|
||||
name = data[at + 2:at + 2 + struct.unpack_from('<H', data, at)[0]].decode().split('^')[0]
|
||||
start = struct.unpack_from('<Q', data, struct.unpack_from('<Q', data, info + 0x70)[0])[0]
|
||||
out[name] = (struct.unpack_from('<I', data, info + 0x1c)[0], struct.unpack_from('<I', data, info + 0x24)[0],
|
||||
data[start:start + struct.unpack_from('<I', data, info + 0x50)[0]])
|
||||
return out
|
||||
|
||||
|
||||
def bflyt(panes):
|
||||
"""A FLYT with one txt1 section per pane (name -> text) after a pan1 section."""
|
||||
sections = b'pan1' + struct.pack('<I', 0x4c) + bytes(0x44)
|
||||
for name, text in panes.items():
|
||||
s = bytearray(0x80)
|
||||
s[0:4] = b'txt1'
|
||||
struct.pack_into('<I', s, 4, len(s))
|
||||
s[12:12 + len(name)] = name.encode()
|
||||
struct.pack_into('<H', s, 0x56, 2 * len(text) + 2)
|
||||
struct.pack_into('<I', s, 0x64, 0x70)
|
||||
s[0x70:0x70 + 2 * len(text) + 2] = text.encode('utf-16-le') + b'\0\0'
|
||||
sections += s
|
||||
return b'FLYT' + struct.pack('<HHIIHH', 0xFEFF, 0x14, 0x08000000, 0x14 + len(sections), len(panes) + 1, 0) + sections
|
||||
|
||||
|
||||
def pane_texts(data):
|
||||
pos, out = struct.unpack_from('<H', data, 6)[0], {}
|
||||
while pos + 8 <= len(data):
|
||||
magic, size = data[pos:pos + 4], struct.unpack_from('<I', data, pos + 4)[0]
|
||||
if magic == b'txt1':
|
||||
length, at = struct.unpack_from('<H', data, pos + 0x56)[0], pos + struct.unpack_from('<I', data, pos + 0x64)[0]
|
||||
out[data[pos + 12:pos + 36].split(b'\0')[0].decode()] = data[at:at + length - 2].decode('utf-16-le')
|
||||
pos += size
|
||||
return out
|
||||
|
||||
|
||||
def msbt(messages):
|
||||
"""A UTF-16 MSBT: one LBL1 group with every label, then TXT2 (label -> text)."""
|
||||
labels = b''.join(bytes([len(name)]) + name.encode() + struct.pack('<I', i) for i, name in enumerate(messages))
|
||||
lbl1 = struct.pack('<III', 1, len(messages), 12) + labels
|
||||
texts = [text.encode('utf-16-le') + b'\0\0' for text in messages.values()]
|
||||
offsets, body = [], b''
|
||||
for text in texts:
|
||||
offsets.append(4 + 4 * len(texts) + len(body))
|
||||
body += text
|
||||
txt2 = struct.pack(f'<I{len(texts)}I', len(texts), *offsets) + body
|
||||
|
||||
def section(magic, content):
|
||||
s = magic + struct.pack('<I', len(content)) + bytes(8) + content
|
||||
return s + bytes(-len(s) % 16)
|
||||
sections = section(b'LBL1', lbl1) + section(b'TXT2', txt2)
|
||||
return b'MsgStdBn' + struct.pack('<HHBBHHI', 0xFEFF, 0, 1, 3, 2, 0, 0x20 + len(sections)) + bytes(10) + sections
|
||||
|
||||
|
||||
def message_texts(data):
|
||||
sections, pos = {}, 0x20
|
||||
for _ in range(struct.unpack_from('<H', data, 0x0e)[0]):
|
||||
size = struct.unpack_from('<I', data, pos + 4)[0]
|
||||
sections[data[pos:pos + 4]] = (pos + 16, size)
|
||||
pos = (pos + 16 + size + 15) & ~15
|
||||
(base, _), labels = sections[b'LBL1'], {}
|
||||
for g in range(struct.unpack_from('<I', data, base)[0]):
|
||||
count, at = struct.unpack_from('<II', data, base + 4 + 8 * g)
|
||||
at += base
|
||||
for _ in range(count):
|
||||
n = data[at]
|
||||
labels[struct.unpack_from('<I', data, at + 1 + n)[0]] = data[at + 1:at + 1 + n].decode()
|
||||
at += 1 + n + 4
|
||||
base, size = sections[b'TXT2']
|
||||
count = struct.unpack_from('<I', data, base)[0]
|
||||
offsets = list(struct.unpack_from(f'<{count}I', data, base + 4)) + [size]
|
||||
return {labels[i]: data[base + offsets[i]:base + offsets[i + 1]].decode('utf-16-le').rstrip('\0') for i in range(count)}
|
||||
|
||||
|
||||
def restbl(entries):
|
||||
"""A RESTBL keyed by the CRC32 of each path without .zs (path -> size)."""
|
||||
crcs = sorted((zlib.crc32(path.removesuffix('.zs').encode()), size) for path, size in entries.items())
|
||||
return b'RESTBL' + struct.pack('<IIII', 1, 160, len(crcs), 0) + b''.join(struct.pack('<II', *c) for c in crcs)
|
||||
|
||||
|
||||
def restbl_sizes(table):
|
||||
count = struct.unpack_from('<I', table, 14)[0]
|
||||
return dict(struct.unpack_from('<II', table, 22 + 8 * i) for i in range(count))
|
||||
|
||||
|
||||
MAGIC = 0x18029a7f # MacNativeGames make_button_icons.py: a .bfotf's first word is its key XOR this
|
||||
|
||||
|
||||
def bfotf(otf, key):
|
||||
"""A .bfotf as MacNativeGames make_button_icons.py bfotf_encode writes it: 32-bit words XORed with the key."""
|
||||
size = len(otf)
|
||||
out = struct.pack('<II', MAGIC ^ key, struct.unpack('<I', size.to_bytes(4, 'big'))[0] ^ key)
|
||||
out += b''.join(struct.pack('<I', struct.unpack_from('<I', otf, i)[0] ^ key) for i in range(0, size - size % 4, 4))
|
||||
return out + otf[size - size % 4:]
|
||||
|
||||
|
||||
def unbfotf(blob):
|
||||
key = struct.unpack_from('<I', blob, 0)[0] ^ MAGIC
|
||||
size = int.from_bytes(struct.pack('<I', struct.unpack_from('<I', blob, 4)[0] ^ key), 'big')
|
||||
words = b''.join(struct.pack('<I', struct.unpack_from('<I', blob, 8 + i)[0] ^ key) for i in range(0, size - size % 4, 4))
|
||||
return words + blob[8 + size - size % 4:8 + size]
|
||||
|
||||
|
||||
def rect(pen, x0, y0, x1, y1, hole=False):
|
||||
"""A rectangle, counter-clockwise (filled, the CFF way) or clockwise (a counter)."""
|
||||
points = [(x0, y0), (x1, y0), (x1, y1), (x0, y1)]
|
||||
points = points[::-1] if hole else points
|
||||
pen.moveTo(points[0])
|
||||
for p in points[1:]:
|
||||
pen.lineTo(p)
|
||||
pen.closePath()
|
||||
|
||||
|
||||
def polygon(*points):
|
||||
def draw(pen):
|
||||
pen.moveTo(points[0])
|
||||
for p in points[1:]:
|
||||
pen.lineTo(p)
|
||||
pen.closePath()
|
||||
return draw
|
||||
|
||||
|
||||
def cff_font(name, glyphs):
|
||||
"""A CFF OpenType font with a 1000-unit em: code point -> (advance, draw(pen))."""
|
||||
names = {cp: f'uni{cp:04X}' for cp in glyphs}
|
||||
fb = FontBuilder(1000, isTTF=False)
|
||||
fb.setupGlyphOrder(['.notdef'] + list(names.values()))
|
||||
fb.setupCharacterMap(names)
|
||||
charstrings, metrics = {}, {}
|
||||
for gname, (advance, draw) in [('.notdef', (500, lambda pen: rect(pen, 50, 0, 450, 700)))] + \
|
||||
[(names[cp], glyphs[cp]) for cp in glyphs]:
|
||||
pen = T2CharStringPen(advance, None)
|
||||
draw(pen)
|
||||
charstrings[gname], metrics[gname] = pen.getCharString(), (advance, 0)
|
||||
fb.setupCFF(name, {'FullName': name}, charstrings, {})
|
||||
fb.setupHorizontalMetrics(metrics)
|
||||
fb.setupHorizontalHeader(ascent=800, descent=-200)
|
||||
fb.setupNameTable({'familyName': name, 'styleName': 'Regular'})
|
||||
fb.setupOS2(sTypoAscender=800, usWinAscent=800, usWinDescent=200)
|
||||
fb.setupPost()
|
||||
out = io.BytesIO()
|
||||
fb.save(out)
|
||||
return out.getvalue()
|
||||
|
||||
|
||||
# A distinct rectangle and advance for every button glyph, so a glyph that moved is found by its outline.
|
||||
BUTTONS = {cp: (600 + 4 * (cp - 0xE040), lambda pen, i=cp - 0xE040: rect(pen, 20 + 5 * i, 20, 300 + 5 * i, 400 + 3 * i))
|
||||
for cp in range(0xE040, 0xE069)}
|
||||
LETTERS_1 = {ord('L'): (560, polygon((60, 0), (480, 0), (480, 100), (160, 100), (160, 700), (60, 700))),
|
||||
ord('B'): (560, lambda pen: [rect(pen, 60, 0, 460, 700), rect(pen, 160, 100, 360, 300, True),
|
||||
rect(pen, 160, 400, 360, 600, True)]),
|
||||
ord('R'): (560, lambda pen: [rect(pen, 60, 0, 460, 700), rect(pen, 160, 400, 360, 600, True)])}
|
||||
LETTERS_2 = {ord('T'): (560, polygon((230, 0), (330, 0), (330, 600), (520, 600), (520, 700), (40, 700), (40, 600), (230, 600)))}
|
||||
# The other characters the PC prompts' keys show (button_icons.PC_LABELS of the defaults): a block with a counter, each
|
||||
# its own size.
|
||||
LETTERS_2.update({ord(ch): (560, lambda pen, i=i: [rect(pen, 60, 0, 300 + 10 * i, 700), rect(pen, 120, 100, 240, 600, True)])
|
||||
for i, ch in enumerate('ACDEFMQSW1234⇧G')})
|
||||
PC_PANES = {'A': 'E', 'B': '↑', 'X': '▬', 'Y': 'L'} # the diamond's letters with the default keys
|
||||
|
||||
|
||||
class Romfs:
|
||||
"""A RomFS (romfs.bin, index.json) holding the files make_romfs_patches.py reads, the originals kept to compare."""
|
||||
|
||||
def __init__(self, folder, version, text_fonts=True, restbl_skips=(), l_format=R8_UNORM):
|
||||
v, rnd = VERSIONS[version], random.Random(version)
|
||||
words = [b'Layout', b'Pane', b'Font', b'Msg', b'Texture', b'Anim', b'Group', b'Text', b'Base', b'Key']
|
||||
samples = [b' '.join(rnd.choice(words) + str(rnd.randrange(999)).encode() for _ in range(rnd.randrange(20, 80)))
|
||||
for _ in range(400)]
|
||||
self.dictionary = zstd.train_dict(samples, 4096)
|
||||
self.common = f'UI/LayoutArchive/Common.Product.{v["common"]}.Nin_NX_NVN.blarc.zs'
|
||||
self.mals = [f'Mals/EUen.Product.{v["newest"]}.sarc.zs', f'Mals/USen.Product.{v["newest"]}.sarc.zs']
|
||||
self.old_mals = 'Mals/USen.Product.110.sarc.zs'
|
||||
self.restbl = f'System/Resource/ResourceSizeTable.Product.{v["newest"]}.{v["restbl"]}'
|
||||
self.old_restbl = f'System/Resource/ResourceSizeTable.Product.100.{v["restbl"]}' # an older version's: left alone
|
||||
size = v['size']
|
||||
images = {f'Nt_KeyTex{b}_00': (R8_UNORM, size, rnd.randbytes(IMAGE)) for b in 'ABXY'}
|
||||
images.update({'Nt_KeyTexL_00^z': (l_format, size, rnd.randbytes(IMAGE)),
|
||||
'Nt_KeyTexZL_00': (R8_UNORM, size, rnd.randbytes(IMAGE)),
|
||||
'Nt_KeyTexLSt_00': (R8_UNORM, size, rnd.randbytes(IMAGE)),
|
||||
'Nt_KeyTexLSt_01': (R8_UNORM, size, rnd.randbytes(IMAGE))})
|
||||
self.text_fonts = [bfotf(cff_font('Text1', LETTERS_1), 0x01020304), bfotf(cff_font('Text2', LETTERS_2), 0x0a0b0c0d)]
|
||||
default = {'scft/External.bfotf': bfotf(cff_font('External', BUTTONS), 0x5a17c3e9), 'fnt/rest.bin': rnd.randbytes(700)}
|
||||
if text_fonts:
|
||||
default.update(zip(icons.TEXT_FONTS, self.text_fonts))
|
||||
mals = sarc({'LayoutMsg/DoCommand_00.msbt': msbt(MESSAGES), 'LayoutMsg/Other_00.msbt': msbt({'T_A_00': 'A'})})
|
||||
self.contents = { # RomFS path -> contents before compression
|
||||
self.common: sarc({'anim/ShortCutBG_00_In.bflan': bflan(('C_CaptureBG_00', 'C_CaptureClearBG_00',
|
||||
'P_CaptureClearBG_00', 'N_Other_00')),
|
||||
'anim/AutoBuilder_00_In.bflan': bflan(('C_CaptureBG_00',)),
|
||||
'blyt/DoCommand_00.bflyt': bflyt(PANES),
|
||||
'blyt/Title_00.bflyt': bflyt({'N_Title_00': 'Z'}),
|
||||
'timg/__Combined.bntx': bntx(images)}),
|
||||
DEFAULT_FONT: sarc(default),
|
||||
# The same font under another key: drawn once, stored under each archive's own key.
|
||||
REGIONAL_FONT: sarc({'scft/External.bfotf': bfotf(cff_font('External', BUTTONS), 0x77777777),
|
||||
'fnt/rest.bin': rnd.randbytes(500)}),
|
||||
**{path: mals for path in self.mals + [self.old_mals]},
|
||||
}
|
||||
sizes = {path: len(data) + 256 for path, data in self.contents.items() if path not in restbl_skips}
|
||||
self.contents[self.restbl] = restbl({**sizes, **{f'Other/{i}.bin': 1000 + i for i in range(20)}})
|
||||
self.contents[self.old_restbl] = restbl({DEFAULT_FONT: 1})
|
||||
plain = (REGIONAL_FONT, self.restbl, self.old_restbl) # stored without a dictionary
|
||||
self.files = {'Pack/ZsDic.pack.zs': zstd.compress(sarc({'zs.zsdic': self.dictionary.dict_content}))}
|
||||
self.files.update({path: zstd.compress(data, zstd_dict=None if path in plain else self.dictionary)
|
||||
for path, data in self.contents.items()})
|
||||
self.files['Other/file.bin'] = rnd.randbytes(100)
|
||||
image, entries = bytearray(), []
|
||||
for path, data in self.files.items():
|
||||
image += bytes(-len(image) % 16)
|
||||
entries.append(dict(path=path, offset=len(image), size=len(data)))
|
||||
image += data
|
||||
folder.mkdir(parents=True)
|
||||
(folder / 'romfs.bin').write_bytes(image)
|
||||
(folder / 'index.json').write_text(json.dumps(dict(status='pass', files=entries)))
|
||||
self.folder, self.version = folder, version
|
||||
|
||||
def run(self, out, *flags):
|
||||
return subprocess.run([sys.executable, '-I', str(TOOL), '--version', self.version, '--romfs', str(self.folder),
|
||||
'--out', str(out), *flags], capture_output=True, text=True, encoding='utf-8')
|
||||
|
||||
def read(self, out, path):
|
||||
"""The overlay's file, decompressed."""
|
||||
raw = (out / path).read_bytes()
|
||||
return zstd.decompress(raw, zstd_dict=self.dictionary if zstd.get_frame_info(raw).dictionary_id else None)
|
||||
|
||||
|
||||
def font(blob):
|
||||
return TTFont(io.BytesIO(unbfotf(blob)))
|
||||
|
||||
|
||||
def glyph(f, cp):
|
||||
"""(outline, advance) of the glyph of code point cp."""
|
||||
name = f.getBestCmap()[cp]
|
||||
pen = RecordingPen()
|
||||
f.getGlyphSet()[name].draw(pen)
|
||||
return pen.value, f['hmtx'][name][0]
|
||||
|
||||
|
||||
def manifest(out):
|
||||
return {line.split('\t')[0]: line.split('\t')[1:] for line in (out / 'manifest.tsv').read_text().splitlines()
|
||||
if not line.startswith('#')}
|
||||
|
||||
|
||||
class PromptsTest(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.tmp = tempfile.TemporaryDirectory(prefix='nx2navi-prompts-')
|
||||
cls.base = Path(cls.tmp.name)
|
||||
cls.romfs = {v: Romfs(cls.base / v / 'romfs', v) for v in VERSIONS}
|
||||
cls.outs = {}
|
||||
for v, romfs in cls.romfs.items():
|
||||
for style in ('plain', 'xbox', 'xbox-label', 'pc'):
|
||||
out = cls.base / v / style
|
||||
proc = romfs.run(out, *(() if style == 'plain' else ('--prompts', style)))
|
||||
assert proc.returncode == 0, proc.stderr
|
||||
cls.outs[v, style] = (out, json.loads(proc.stdout.strip().splitlines()[-1]))
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
cls.tmp.cleanup()
|
||||
|
||||
def each(self, styles=('xbox', 'xbox-label')):
|
||||
"""(RomFS, style, overlay folder, summary) of every run of these styles."""
|
||||
return [(self.romfs[v], style, out, summary) for (v, style), (out, summary) in self.outs.items()
|
||||
if style in styles]
|
||||
|
||||
def test_plain_overlay_has_only_the_frame_rate_patch(self):
|
||||
for romfs, _, out, summary in self.each(('plain',)):
|
||||
with self.subTest(version=romfs.version, style="plain"):
|
||||
self.assertEqual(set(manifest(out)), {romfs.common})
|
||||
self.assertIsNone(summary['prompts'])
|
||||
keys = visibility(sarc_files(romfs.read(out, romfs.common))['anim/ShortCutBG_00_In.bflan'])
|
||||
self.assertEqual(keys['C_CaptureBG_00'], [(0.0, 0), (0.0, 1)])
|
||||
self.assertEqual(keys['N_Other_00'], [(0.0, 0), (1.0, 1)])
|
||||
|
||||
def test_face_buttons_name_the_xbox_label_of_the_button_pressed(self):
|
||||
for romfs, style, out, _ in self.each():
|
||||
with self.subTest(version=romfs.version, style=style):
|
||||
layout = STYLE_LAYOUT[style]
|
||||
label = {b: XBOX_LABEL[POSITION[layout][b]] for b in FACE} # Switch button -> the label of its button
|
||||
before, after = sarc_files(romfs.contents[romfs.common]), sarc_files(romfs.read(out, romfs.common))
|
||||
old, new = bntx_images(before['timg/__Combined.bntx']), bntx_images(after['timg/__Combined.bntx'])
|
||||
panes, messages = pane_texts(after['blyt/DoCommand_00.bflyt']), {}
|
||||
for path in romfs.mals if layout == 'nintendo' else []:
|
||||
messages[path] = message_texts(sarc_files(romfs.read(out, path))['LayoutMsg/DoCommand_00.msbt'])
|
||||
original = font(sarc_files(romfs.contents[DEFAULT_FONT])['scft/External.bfotf'])
|
||||
for path in (DEFAULT_FONT, REGIONAL_FONT):
|
||||
drawn = font(sarc_files(romfs.read(out, path))['scft/External.bfotf'])
|
||||
for b, cp in FACE.items():
|
||||
self.assertEqual(glyph(drawn, cp), glyph(original, FACE[label[b]]), f'{path}: glyph of {b}')
|
||||
for b in FACE:
|
||||
self.assertEqual(new[f'Nt_KeyTex{b}_00'], old[f'Nt_KeyTex{label[b]}_00'], f'action prompt image of {b}')
|
||||
self.assertEqual(panes[f'T_{b}_00'], label[b], f'diamond letter of {b}')
|
||||
for path, texts in messages.items():
|
||||
self.assertEqual(texts[f'T_{b}_00'], label[b], f'{path}: diamond message of {b}')
|
||||
# Text that only looks like a letter pane, and every other message, stays.
|
||||
self.assertEqual((panes['T_Name_00'], pane_texts(after['blyt/Title_00.bflyt'])), ('A', {'N_Title_00': 'Z'}))
|
||||
for path, texts in messages.items():
|
||||
self.assertEqual((texts['T_Name_00'], texts['Greeting']), ('A', 'Hello'))
|
||||
self.assertEqual(sarc_files(romfs.read(out, path))['LayoutMsg/Other_00.msbt'], msbt({'T_A_00': 'A'}))
|
||||
|
||||
def test_other_buttons_are_drawn_as_xbox_icons(self):
|
||||
for romfs, style, out, _ in self.each():
|
||||
with self.subTest(version=romfs.version, style=style):
|
||||
original = font(sarc_files(romfs.contents[DEFAULT_FONT])['scft/External.bfotf'])
|
||||
default = sarc_files(romfs.read(out, DEFAULT_FONT))
|
||||
drawn = font(default['scft/External.bfotf'])
|
||||
for cp in range(0xE040, 0xE069):
|
||||
outline, advance = glyph(drawn, cp)
|
||||
if cp in DRAWN:
|
||||
self.assertNotEqual(outline, glyph(original, cp)[0], f'U+{cp:04X} drawn')
|
||||
self.assertEqual(advance, DRAWN[cp], f'U+{cp:04X} advance')
|
||||
elif cp not in FACE.values():
|
||||
self.assertEqual((outline, advance), glyph(original, cp), f'U+{cp:04X} kept')
|
||||
# The letters are holes in the key: LB is the key, L, and B with its two counters.
|
||||
moves = sum(op == 'moveTo' for op, _ in glyph(drawn, 0xE044)[0])
|
||||
self.assertEqual(moves, 5)
|
||||
self.assertEqual(sum(op == 'moveTo' for op, _ in glyph(drawn, 0xE047)[0]), 1 + 1 + 1 + 1) # RT: key, R + counter, T
|
||||
self.assertEqual(sum(op == 'moveTo' for op, _ in glyph(drawn, 0xE060)[0]), 2) # d-pad up: cross, arrow
|
||||
# The text fonts and the archives' other members stay byte for byte; both archives hold the same font.
|
||||
before = sarc_files(romfs.contents[DEFAULT_FONT])
|
||||
self.assertEqual({k: v for k, v in default.items() if k != 'scft/External.bfotf'},
|
||||
{k: v for k, v in before.items() if k != 'scft/External.bfotf'})
|
||||
regional = sarc_files(romfs.read(out, REGIONAL_FONT))
|
||||
self.assertEqual(regional['fnt/rest.bin'], sarc_files(romfs.contents[REGIONAL_FONT])['fnt/rest.bin'])
|
||||
self.assertEqual(unbfotf(regional['scft/External.bfotf']), unbfotf(default['scft/External.bfotf']))
|
||||
self.assertEqual(struct.unpack_from('<I', regional['scft/External.bfotf'])[0] ^ MAGIC, 0x77777777)
|
||||
|
||||
def test_button_guide_images_show_the_same_icons(self):
|
||||
for romfs, style, out, _ in self.each():
|
||||
with self.subTest(version=romfs.version, style=style):
|
||||
size = VERSIONS[romfs.version]['size']
|
||||
before = sarc_files(romfs.contents[romfs.common])
|
||||
after = sarc_files(romfs.read(out, romfs.common))
|
||||
self.assertEqual(len(after['timg/__Combined.bntx']), len(before['timg/__Combined.bntx']))
|
||||
old, new = bntx_images(before['timg/__Combined.bntx']), bntx_images(after['timg/__Combined.bntx'])
|
||||
glyphs = icons.xbox_icons(icons.text_glyphs(romfs.text_fonts))
|
||||
for name, cp in icons.HUD_TEXTURES.items():
|
||||
fmt, width, image = new[name]
|
||||
pixels = bytes(image[icons.gob_offset(x, y, (size + 63) // 64, 16)] for y in range(size) for x in range(size))
|
||||
self.assertEqual(pixels, icons.rasterize(glyphs[cp][1], size), name)
|
||||
self.assertEqual({pixels[0], pixels[-1], max(pixels)}, {0, 255}, name) # a margin around a white key
|
||||
self.assertEqual(new['Nt_KeyTexLSt_00'], old['Nt_KeyTexLSt_00'])
|
||||
for member in before: # the 60 fps patch and the prompts touch only these
|
||||
if member not in ('timg/__Combined.bntx', 'blyt/DoCommand_00.bflyt') and not member.startswith('anim/'):
|
||||
self.assertEqual(after[member], before[member], member)
|
||||
|
||||
def test_resource_size_table_grows_with_the_fonts(self):
|
||||
for romfs, style, out, summary in self.each(('xbox', 'xbox-label', 'pc')):
|
||||
with self.subTest(version=romfs.version, style=style):
|
||||
table = restbl_sizes(romfs.read(out, romfs.restbl))
|
||||
original = restbl_sizes(romfs.contents[romfs.restbl])
|
||||
grown = {}
|
||||
for path in manifest(out):
|
||||
if path != romfs.restbl:
|
||||
grown[path] = len(romfs.read(out, path)) - len(romfs.contents[path])
|
||||
self.assertTrue(grown[DEFAULT_FONT] > 0 and grown[REGIONAL_FONT] > 0, grown)
|
||||
self.assertEqual(summary['prompts']['grown'], {p: g for p, g in grown.items() if g > 0})
|
||||
for crc, entry in original.items():
|
||||
paths = [p for p in grown if zlib.crc32(p.removesuffix('.zs').encode()) == crc and grown[p] > 0]
|
||||
self.assertEqual(table[crc], entry + (grown[paths[0]] if paths else 0))
|
||||
self.assertEqual(set(table), set(original))
|
||||
|
||||
def test_manifest_lists_what_changed_with_both_hashes(self):
|
||||
for romfs, style, out, _ in self.each(('plain', 'xbox', 'xbox-label', 'pc')):
|
||||
with self.subTest(version=romfs.version, style=style):
|
||||
expect = {romfs.common}
|
||||
if style != 'plain':
|
||||
expect |= {DEFAULT_FONT, REGIONAL_FONT, romfs.restbl}
|
||||
expect |= set(romfs.mals) if style in ('xbox', 'pc') else set()
|
||||
lines = manifest(out)
|
||||
self.assertEqual(set(lines), expect)
|
||||
for path, (size, sha, original) in lines.items():
|
||||
data = (out / path).read_bytes()
|
||||
self.assertEqual((int(size), sha), (len(data), hashlib.sha256(data).hexdigest()), path)
|
||||
self.assertEqual(original, hashlib.sha256(romfs.files[path]).hexdigest(), path)
|
||||
|
||||
def test_pc_prompts_draw_every_button_as_its_key(self):
|
||||
for romfs, _, out, _ in self.each(('pc',)):
|
||||
with self.subTest(version=romfs.version):
|
||||
original = font(sarc_files(romfs.contents[DEFAULT_FONT])['scft/External.bfotf'])
|
||||
drawn = font(sarc_files(romfs.read(out, DEFAULT_FONT))['scft/External.bfotf'])
|
||||
advances = icons.font_advances(sarc_files(romfs.contents[DEFAULT_FONT])['scft/External.bfotf'])
|
||||
pc = icons.pc_icons(icons.text_glyphs(romfs.text_fonts), advances)
|
||||
self.assertIn(0xE05E, pc) # the left stick's W A S D
|
||||
for cp in range(0xE040, 0xE069):
|
||||
outline, advance = glyph(drawn, cp)
|
||||
if cp in pc:
|
||||
self.assertNotEqual(outline, glyph(original, cp)[0], f'U+{cp:04X} drawn')
|
||||
else:
|
||||
self.assertEqual(outline, glyph(original, cp)[0], f'U+{cp:04X} kept')
|
||||
self.assertEqual(advance, glyph(original, cp)[1], f'U+{cp:04X} keeps its advance')
|
||||
# The key's label is a hole: E is the key, the block and its counter.
|
||||
self.assertEqual(sum(op == 'moveTo' for op, _ in glyph(drawn, 0xE040)[0]), 3)
|
||||
after = sarc_files(romfs.read(out, romfs.common))
|
||||
new = bntx_images(after['timg/__Combined.bntx'])
|
||||
size = VERSIONS[romfs.version]['size']
|
||||
for name, cp in icons.PC_HUD_TEXTURES.items():
|
||||
image = new[name][2]
|
||||
pixels = bytes(image[icons.gob_offset(x, y, (size + 63) // 64, 16)] for y in range(size) for x in range(size))
|
||||
self.assertEqual(pixels, icons.rasterize(pc[cp][1], size), name)
|
||||
panes = pane_texts(after['blyt/DoCommand_00.bflyt'])
|
||||
for b, letter in PC_PANES.items():
|
||||
self.assertEqual(panes[f'T_{b}_00'], letter, f'diamond letter of {b}')
|
||||
for path in romfs.mals:
|
||||
texts = message_texts(sarc_files(romfs.read(out, path))['LayoutMsg/DoCommand_00.msbt'])
|
||||
self.assertEqual((texts[f'T_{b}_00'], texts['T_Name_00']), (letter, 'A'), path)
|
||||
|
||||
def test_pc_prompts_follow_the_bind_lines(self):
|
||||
romfs = self.romfs['v143']
|
||||
ini = self.base / 'binds' / 'nx2navi.ini'
|
||||
ini.parent.mkdir(parents=True)
|
||||
ini.write_text('[input]\n# bind_A = key:q\nbind_A = pad:east, key:F\nbind_B = pad:south\nbind_X = key:default\n',
|
||||
encoding='utf-8')
|
||||
out = self.base / 'binds' / 'out'
|
||||
proc = romfs.run(out, '--prompts', 'pc', '--binds', str(ini))
|
||||
self.assertEqual(proc.returncode, 0, proc.stderr)
|
||||
panes = pane_texts(sarc_files(romfs.read(out, romfs.common))['blyt/DoCommand_00.bflyt'])
|
||||
self.assertEqual({b: panes[f'T_{b}_00'] for b in PC_PANES}, {**PC_PANES, 'A': 'F'})
|
||||
self.assertEqual(icons.pc_binds(ini.read_text(encoding='utf-8')), {**icons.PC_BINDS, 'A': 'f'})
|
||||
|
||||
def test_same_input_same_output_and_the_older_flag(self):
|
||||
romfs = self.romfs['v121']
|
||||
for flags in (('--prompts', 'xbox'), ('--xbox-prompts',)):
|
||||
out = self.base / 'again' / '-'.join(flags)
|
||||
proc = romfs.run(out, *flags)
|
||||
self.assertEqual(proc.returncode, 0, proc.stderr)
|
||||
self.assertEqual((out / 'manifest.tsv').read_bytes(), (self.outs['v121', 'xbox'][0] / 'manifest.tsv').read_bytes())
|
||||
|
||||
def test_unrecognised_romfs_fails_before_writing(self):
|
||||
cases = {'no text fonts': (dict(text_fonts=False), icons.TEXT_FONTS[0]),
|
||||
'no size entry': (dict(restbl_skips=(REGIONAL_FONT,)), 'has no entry in the resource size table'),
|
||||
'compressed L image': (dict(l_format=BC1), 'Nt_KeyTexL_00: format')}
|
||||
for i, (case, (options, message)) in enumerate(cases.items()):
|
||||
with self.subTest(case):
|
||||
romfs = Romfs(self.base / 'bad' / str(i) / 'romfs', 'v121', **options)
|
||||
out = self.base / 'bad' / str(i) / 'out'
|
||||
proc = romfs.run(out, '--prompts', 'xbox-label')
|
||||
self.assertNotEqual(proc.returncode, 0)
|
||||
self.assertIn(message, proc.stderr)
|
||||
self.assertFalse(out.exists(), 'a failed run writes nothing')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
80
tests/spin_waits_test.py
Normal file
80
tests/spin_waits_test.py
Normal file
@@ -0,0 +1,80 @@
|
||||
# Adapted from nxnative dm/tools/tests/test_spin_waits.py @ 6d34984 (MIT, Copyright (c) 2026 seedds).
|
||||
"""tools/spin_waits.py on hand-assembled AArch64 loops, without game files.
|
||||
|
||||
Run: .venv/Scripts/python tests/spin_waits_test.py
|
||||
"""
|
||||
import struct
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tools'))
|
||||
|
||||
from spin_waits import Search # noqa: E402
|
||||
|
||||
# A job queue poll (the shape of Breath of the Wild's main+0xef6354): take the queue's spin lock, look whether the wait
|
||||
# is over, release the lock, again.
|
||||
# 0x00 T: cbz x20, X 0x2c str w12, [x20, #0x100] 0x50 ldr w13, [x21, #0x90]
|
||||
# 0x04 str w11, [x20, #0x100] 0x30 ldrb w14, [x21, #0x98] 0x54 sub w8, w13, w25
|
||||
# 0x08 L0: ldr w14, [x9] 0x34 cmp w0, #0 0x58 add w15, w8, w25
|
||||
# 0x0c cmp w14, #1 0x38 ccmp w14, #0, #4, ne 0x5c str w15, [x21, #0x94]
|
||||
# 0x10 b.eq L0 0x3c ldr w25, [x21, #0x94] 0x60 cset w13, hs
|
||||
# 0x14 L1: ldaxr w14, [x9] 0x40 ccmp w25, #0, #0, ne 0x64 b U
|
||||
# 0x18 cbz w14, L2 0x44 cset w14, gt 0x68 C: clrex
|
||||
# 0x1c clrex 0x48 cmp w14, #1 0x6c U: ldxr w15, [x9] / cmp w15, #1 / b.ne C
|
||||
# 0x20 b L1 0x4c b.ne U 0x78 stlxr w15, wzr, [x9] / cbnz w15, U
|
||||
# 0x24 L2: stxr w14, w10, [x9] / cbnz w14, L1 0x80 cbz w14, T 0x84 X: ret
|
||||
# 0x50-0x64: the wait is over, take an item, store the new cursor, release and leave.
|
||||
POLL = [0xb4000434, 0xb901028b, 0xb940012e, 0x710005df, 0x54ffffc0, 0x885ffd2e, 0x3400006e, 0xd5033f5f, 0x17fffffd,
|
||||
0x880e7d2a, 0x35ffff6e, 0xb901028c, 0x394262ae, 0x7100001f, 0x7a4019c4, 0xb94096b9, 0x7a401b20, 0x1a9fd7ee,
|
||||
0x710005df, 0x54000101, 0xb94092ad, 0x4b1901a8, 0x0b19010f, 0xb90096af, 0x1a9f37ed, 0x14000002, 0xd5033f5f,
|
||||
0x885f7d2f, 0x710005ff, 0x54ffffa1, 0x880ffd3f, 0x35ffff8f, 0x34fffc0e, 0xd65f03c0]
|
||||
|
||||
|
||||
def region(words, T, bes):
|
||||
img = struct.pack('<' + 'I' * len(words), *words) + bytes(64)
|
||||
return Search(img, 4 * len(words), set()).region(T, bes)
|
||||
|
||||
|
||||
def patched(**at):
|
||||
w = list(POLL)
|
||||
for off, word in at.items():
|
||||
w[int(off[1:], 16) // 4] = word
|
||||
return w
|
||||
|
||||
|
||||
class SpinWaits(unittest.TestCase):
|
||||
def test_lock_poll_is_a_wait(self):
|
||||
g = region(POLL, 0, [0x80])
|
||||
self.assertIsNotNone(g)
|
||||
self.assertEqual(g['valid'], {0x80})
|
||||
self.assertNotIn(0x5c, g['R']) # the path that takes an item stores a new cursor: not part of the wait
|
||||
self.assertTrue(set(g['kinds']) >= {'acq', 'rel', 'tst', 'stc'})
|
||||
|
||||
def test_found_by_the_scan(self):
|
||||
self.assertIn(0, {g['T'] for g in Search(struct.pack('<' + 'I' * len(POLL), *POLL), 4 * len(POLL), set()).all()})
|
||||
|
||||
def test_patch_site_inside(self): # a site of the patch table in the region keeps it out
|
||||
img = struct.pack('<' + 'I' * len(POLL), *POLL) + bytes(64)
|
||||
self.assertIsNone(Search(img, 4 * len(POLL), {0x3c}).region(0, [0x80]))
|
||||
|
||||
def test_store_of_a_changing_value(self): # the wait path stores w14, which the pass writes
|
||||
self.assertIsNone(region(patched(o2c=0xb901028e), 0, [0x80]))
|
||||
|
||||
def test_lock_not_released(self): # the wait path skips the release (b.ne U -> b.ne to the back-edge)
|
||||
self.assertIsNone(region(patched(o4c=0x540001a1), 0, [0x80]))
|
||||
|
||||
def test_nothing_polled(self): # str xzr, [x8]; b back: stores of a constant and no load
|
||||
self.assertIsNone(region([0xf900011f, 0x17ffffff], 0, [4]))
|
||||
|
||||
def test_plain_test_loop(self): # ldr w8, [x0]; cbz w8, back
|
||||
g = region([0xb9400008, 0x34ffffe8], 0, [4])
|
||||
self.assertIsNotNone(g)
|
||||
self.assertEqual(g['valid'], {4})
|
||||
|
||||
def test_counted_loop(self): # ldr w8, [x0]; subs w1, w1, #1; b.ne back: a counter, it leaves by itself
|
||||
self.assertIsNone(region([0xb9400008, 0x71000421, 0x54ffffc1], 0, [8]))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
130
tests/worktree_test.py
Normal file
130
tests/worktree_test.py
Normal file
@@ -0,0 +1,130 @@
|
||||
"""tools/worktree.py against a scratch repo whose "main checkout" holds dummy ignored data. No game files needed.
|
||||
|
||||
Proves that marker files in the linked directories and in the linked file survive:
|
||||
1. tools/worktree.py remove,
|
||||
2. git worktree remove --force with the junctions still in place,
|
||||
3. git worktree prune after the worktree directory is deleted with its junctions in place.
|
||||
Also checks that git status in a worktree shows no untracked or typechanged entry for the links.
|
||||
Run with the repo's Python: tests/worktree_test.py
|
||||
"""
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(ROOT / 'tools'))
|
||||
import worktree # noqa: E402
|
||||
|
||||
IGNORED = '\n'.join('/' + d + '/' for d in worktree.LINK_DIRS) + '\n/' + worktree.LINK_FILES[0] + '\n/.worktrees/\n'
|
||||
|
||||
|
||||
def git(cwd, *a):
|
||||
r = subprocess.run(['git', '-C', str(cwd), *a], capture_output=True, text=True)
|
||||
assert r.returncode == 0, f'git {a}: {r.stderr}'
|
||||
return r.stdout
|
||||
|
||||
|
||||
def tool(scratch, *a):
|
||||
r = subprocess.run([sys.executable, str(scratch / 'tools' / 'worktree.py'), *a], capture_output=True, text=True)
|
||||
assert r.returncode == 0, f'worktree.py {a}: {r.stdout}{r.stderr}'
|
||||
return r.stdout
|
||||
|
||||
|
||||
def markers(main):
|
||||
return [main / d / 'deep' / 'marker.txt' for d in worktree.LINK_DIRS if (main / d).is_dir()] + \
|
||||
[main / f for f in worktree.LINK_FILES]
|
||||
|
||||
|
||||
def check(main, what):
|
||||
for m in markers(main):
|
||||
assert m.is_file() and m.read_text() == 'keep', f'{what}: {m} lost'
|
||||
print(f'ok: {what}: {len(markers(main))} markers intact')
|
||||
|
||||
|
||||
def make_repo(base):
|
||||
main = base / 'main'
|
||||
(main / 'tools').mkdir(parents=True)
|
||||
shutil.copy(ROOT / 'tools' / 'worktree.py', main / 'tools' / 'worktree.py')
|
||||
(main / '.gitignore').write_text(IGNORED)
|
||||
(main / 'README.md').write_text('x\n')
|
||||
git(main, 'init', '-b', 'main')
|
||||
git(main, 'config', 'user.email', 't@t')
|
||||
git(main, 'config', 'user.name', 't')
|
||||
git(main, 'add', '.')
|
||||
git(main, 'commit', '-m', 'init')
|
||||
for d in worktree.LINK_DIRS:
|
||||
if d == 'game':
|
||||
continue # absent in a real checkout too: link only what exists
|
||||
(main / d / 'deep').mkdir(parents=True)
|
||||
(main / d / 'deep' / 'marker.txt').write_text('keep')
|
||||
for f in worktree.LINK_FILES:
|
||||
(main / f).write_text('keep')
|
||||
return main
|
||||
|
||||
|
||||
def main():
|
||||
base = Path(tempfile.mkdtemp(prefix='wt-test-')).resolve()
|
||||
try:
|
||||
main_dir = make_repo(base)
|
||||
check(main_dir, 'setup')
|
||||
|
||||
# tools/worktree.py new / status / remove
|
||||
wt = Path(tool(main_dir, 'new', 'a').strip())
|
||||
assert wt.resolve() == (main_dir / '.worktrees' / 'a').resolve(), wt
|
||||
for d in worktree.LINK_DIRS:
|
||||
if (main_dir / d).is_dir():
|
||||
assert os.path.isjunction(wt / d), d
|
||||
assert (wt / 'build' / 'deep' / 'marker.txt').read_text() == 'keep'
|
||||
assert os.path.samefile(wt / 'config.local.toml', main_dir / 'config.local.toml')
|
||||
status = git(wt, 'status', '--porcelain', '--ignored=no')
|
||||
assert status == '', f'links show in git status: {status!r}'
|
||||
assert git(wt, 'ls-files', '--others', '--exclude-standard') == ''
|
||||
print('ok: git status in the worktree is clean, links are ignored')
|
||||
assert '.worktrees/a' in tool(main_dir, 'list').replace(os.sep, '/')
|
||||
(wt / 'dirty.txt').write_text('x')
|
||||
r = subprocess.run([sys.executable, str(main_dir / 'tools' / 'worktree.py'), 'remove', 'a'],
|
||||
capture_output=True, text=True)
|
||||
assert r.returncode != 0 and (wt / 'build').is_dir(), 'remove must refuse a dirty worktree'
|
||||
check(main_dir, 'dirty remove refused')
|
||||
(wt / 'dirty.txt').unlink()
|
||||
tool(main_dir, 'remove', 'a')
|
||||
assert not wt.exists()
|
||||
check(main_dir, 'tools/worktree.py remove')
|
||||
|
||||
# unmerged branch: --delete-branch refuses, --force deletes
|
||||
wt = Path(tool(main_dir, 'new', 'b').strip())
|
||||
(wt / 'f.txt').write_text('x')
|
||||
git(wt, 'add', 'f.txt')
|
||||
git(wt, 'commit', '-m', 'work')
|
||||
r = subprocess.run([sys.executable, str(main_dir / 'tools' / 'worktree.py'), 'remove', 'b', '--delete-branch'],
|
||||
capture_output=True, text=True)
|
||||
assert r.returncode != 0 and wt.is_dir(), 'unmerged branch must block --delete-branch'
|
||||
git(main_dir, 'merge', '--ff-only', 'task/b')
|
||||
tool(main_dir, 'remove', 'b', '--delete-branch')
|
||||
assert 'task/b' not in git(main_dir, 'branch', '--list')
|
||||
check(main_dir, 'remove --delete-branch after merge')
|
||||
|
||||
# raw git: worktree remove --force with the links in place
|
||||
wt = Path(tool(main_dir, 'new', 'c').strip())
|
||||
git(main_dir, 'worktree', 'remove', '--force', str(wt))
|
||||
check(main_dir, 'raw git worktree remove --force')
|
||||
|
||||
# raw: delete the worktree directory with the junctions in place, then prune
|
||||
wt = Path(tool(main_dir, 'new', 'e').strip())
|
||||
shutil.rmtree(wt)
|
||||
git(main_dir, 'worktree', 'prune')
|
||||
check(main_dir, 'rmtree of a linked worktree, then git worktree prune')
|
||||
print('PASS')
|
||||
finally:
|
||||
# never follow a junction while cleaning up
|
||||
for w in (base / 'main' / '.worktrees').glob('*'):
|
||||
for d in worktree.LINK_DIRS:
|
||||
worktree.drop_link(w / d)
|
||||
shutil.rmtree(base, ignore_errors=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user