Triforce v0.2.0

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
ruckus7382sfgsgdf
2026-11-10 19:32:48 -06:00
commit 34b3683441
1730 changed files with 554853 additions and 0 deletions

206
third_party/a64-kit/rt/a64.h vendored Normal file
View File

@@ -0,0 +1,206 @@
/* a64-kit guest model: what every lifted AArch64 function sees.
The instruction bodies come from suyu's translator (lift/suyu/arm64_to_c.h), which writes C against a
`GuestContext` and calls `recomp_load*` / `recomp_store*` / `recomp_ldxr` ...; this header supplies those
for Rexcomp: one flat guest memory below 4 GB (`A64_MEM + (uint32_t)addr`), no page table, no host bridge.
The FP helpers and the rest of suyu's runtime header are generated next to the lifted code
(a64_suyu.h, by lift/a64_rtgen.py) and included at the end. */
#ifndef A64_H
#define A64_H
#include <stdint.h>
#include <stddef.h>
#include <string.h>
typedef struct GuestContext {
uint64_t x[32]; /* x[31] is SP: suyu's translator spells SP as x[31] where the encoding means SP */
uint64_t pc; /* only written where the lifter stops a thread (traps, unhandled) */
uint8_t n, z, c, v; /* NZCV, one byte each */
int halted;
uint64_t vreg[32][2]; /* Q0-Q31 as two 64-bit halves */
uint64_t tpidr_el0; /* the guest's own thread pointer (C++ TLS) */
uint64_t tpidrro_el0; /* Horizon's thread-local region, set by the OS layer */
uint64_t fpcr, fpsr;
/* Exclusive monitor (LDXR/STXR): the address, size and value LDXR saw. STXR compare-and-swaps against
that value, so it works with real host threads (dynarmic's model). */
uint64_t excl_addr, excl_lo, excl_hi;
uint32_t excl_size;
int excl_valid;
void *thread; /* the OS layer's thread object */
} GuestContext;
typedef GuestContext Ctx;
/* A64_NOESC on every context parameter lifted code can call (the lifted functions, A64Fn, the runtime's and the OS
layer's entry points): none of them keeps the pointer past the call. Without it each call "captures" the context, and
LLVM must assume any guest load or store (an address made from an integer) might touch it: after every guest store
it reloaded guest registers, and it kept every store to them. It also needs the lifted code built with a higher
-capture-tracking-max-uses-to-explore (build.py): past 100 uses of the pointer LLVM stops looking and assumes
it escaped, and a large lifted function uses it thousands of times. (Its effect on one of TOTK's hot functions:
context loads 539 -> 278, stores 1068 -> 893, instructions -6%.) -DA64_NOESC= turns it off. */
#ifndef A64_NOESC
#if defined(__clang__)
#define A64_NOESC __attribute__((noescape))
#else
#define A64_NOESC
#endif
#endif
#if defined(_MSC_VER) && !defined(__clang__)
#define A64_INLINE static __forceinline
#else
#define A64_INLINE static inline __attribute__((always_inline))
#endif
/* Guest memory: the 4 GB guest window starts at a constant host address, so the base folds into every access
instead of being reloaded from a global (with -fno-strict-aliasing any guest byte store could have changed
that global). Natively the window is reserved at A64_HOST_BASE (hos_main.c); with A64_MEM_ZERO (web builds)
guest address = wasm address, which needs -fno-delete-null-pointer-checks because guest address 0 is real. */
#ifdef A64_MEM_ZERO
#define A64_MEM ((uint8_t *)0)
#else
#ifndef A64_HOST_BASE
#define A64_HOST_BASE 0x100000000000ULL /* 16 TB: far from anything Windows or glibc map by default */
#endif
#define A64_MEM ((uint8_t *)A64_HOST_BASE)
#endif
#define A64_PTR(a) (A64_MEM + (uint32_t)(a))
/* Browser builds (A64_WRITE_TRACK): wasm has no page write watch, so every guest store marks its 4 KB page (first
and last byte's: an unaligned store may cross a page); hos/web/win32.c's GetWriteWatch reads and clears the marks
for the renderer's buffer cache. */
#if defined(A64_NO_MARKS) /* (measurement only: build.py RX_NO_GUEST_MARKS) */
#define A64_MARK(a, n) ((void)0)
#elif defined(A64_WRITE_TRACK) && !defined(__EMSCRIPTEN__)
/* Native builds track the same way instead of with Windows' write watch (build.py: RX_SOFT_WRITE_WATCH=0), whose
GetWriteWatch over the ~2 GB the GPU sees took 20-50 ms while the game streamed (hitches at its fences). The mark is a
release store (a plain store on x86): the compiler may not move it above the data store, so a scan that clears a
mark has seen the data it marks. (The marks are ~1 ms of the castle's game-thread CPU a frame. Marking the last
byte's page only when a store crosses into it saved ~0.3 ms but made the code 20 MB bigger: not kept.) */
extern uint8_t a64_dirty[1u << 20];
/* (Tried 2026-10-21: reading a mark first and writing it only when clear, against cache lines moving between cores:
game threads +1.1 ms a frame, the code 18 MB bigger. Not kept.) */
#define A64_MARK(a, n) (__atomic_store_n(&a64_dirty[(uint32_t)(a) >> 12], 1, __ATOMIC_RELEASE), \
__atomic_store_n(&a64_dirty[(uint32_t)((a) + (n) - 1) >> 12], 1, __ATOMIC_RELEASE))
#elif defined(A64_WRITE_TRACK)
extern uint8_t a64_dirty[1u << 20];
#define A64_MARK(a, n) (a64_dirty[(uint32_t)(a) >> 12] = 1, a64_dirty[(uint32_t)((a) + (n) - 1) >> 12] = 1)
#else
#define A64_MARK(a, n) ((void)0)
#endif
A64_INLINE uint64_t recomp_load8(GuestContext *c, uint64_t a) { (void)c; return *(const uint8_t *)A64_PTR(a); }
A64_INLINE uint64_t recomp_load16(GuestContext *c, uint64_t a) { uint16_t v; (void)c; memcpy(&v, A64_PTR(a), 2); return v; }
A64_INLINE uint64_t recomp_load32(GuestContext *c, uint64_t a) { uint32_t v; (void)c; memcpy(&v, A64_PTR(a), 4); return v; }
A64_INLINE uint64_t recomp_load64(GuestContext *c, uint64_t a) { uint64_t v; (void)c; memcpy(&v, A64_PTR(a), 8); return v; }
A64_INLINE void recomp_store8(GuestContext *c, uint64_t a, uint64_t v) { (void)c; *(uint8_t *)A64_PTR(a) = (uint8_t)v; A64_MARK(a, 1); }
A64_INLINE void recomp_store16(GuestContext *c, uint64_t a, uint64_t v) { uint16_t t = (uint16_t)v; (void)c; memcpy(A64_PTR(a), &t, 2); A64_MARK(a, 2); }
A64_INLINE void recomp_store32(GuestContext *c, uint64_t a, uint64_t v) { uint32_t t = (uint32_t)v; (void)c; memcpy(A64_PTR(a), &t, 4); A64_MARK(a, 4); }
A64_INLINE void recomp_store64(GuestContext *c, uint64_t a, uint64_t v) { (void)c; memcpy(A64_PTR(a), &v, 8); A64_MARK(a, 8); }
/* Stores through SP (stack_stores in lift/a64_lift.py): the guest stack is never memory the GPU reads, so they mark no
page. They are a quarter of TOTK's store sites (prologue saves, nearly every STP). */
A64_INLINE void recomp_store8_sp(GuestContext *c, uint64_t a, uint64_t v) { (void)c; *(uint8_t *)A64_PTR(a) = (uint8_t)v; }
A64_INLINE void recomp_store16_sp(GuestContext *c, uint64_t a, uint64_t v) { uint16_t t = (uint16_t)v; (void)c; memcpy(A64_PTR(a), &t, 2); }
A64_INLINE void recomp_store32_sp(GuestContext *c, uint64_t a, uint64_t v) { uint32_t t = (uint32_t)v; (void)c; memcpy(A64_PTR(a), &t, 4); }
A64_INLINE void recomp_store64_sp(GuestContext *c, uint64_t a, uint64_t v) { (void)c; memcpy(A64_PTR(a), &v, 8); }
A64_INLINE void recomp_stp32_sp(GuestContext *c, uint64_t a, uint64_t v0, uint64_t v1) { recomp_store32_sp(c, a, v0); recomp_store32_sp(c, a + 4, v1); }
A64_INLINE void recomp_stp64_sp(GuestContext *c, uint64_t a, uint64_t v0, uint64_t v1) { recomp_store64_sp(c, a, v0); recomp_store64_sp(c, a + 8, v1); }
A64_INLINE void recomp_ldp32(GuestContext *c, uint64_t a, uint64_t *lo, uint64_t *hi) { *lo = recomp_load32(c, a); *hi = recomp_load32(c, a + 4); }
A64_INLINE void recomp_ldp64(GuestContext *c, uint64_t a, uint64_t *lo, uint64_t *hi) { *lo = recomp_load64(c, a); *hi = recomp_load64(c, a + 8); }
/* (a pair marks once, for its whole span) */
A64_INLINE void recomp_stp32(GuestContext *c, uint64_t a, uint64_t v0, uint64_t v1) { recomp_store32_sp(c, a, v0); recomp_store32_sp(c, a + 4, v1); A64_MARK(a, 8); }
A64_INLINE void recomp_stp64(GuestContext *c, uint64_t a, uint64_t v0, uint64_t v1) { recomp_store64_sp(c, a, v0); recomp_store64_sp(c, a + 8, v1); A64_MARK(a, 16); }
/* Everything is lifted at its final guest address, so the translator's module base is a constant 0. */
#define g_module_base ((uint64_t)0)
/* ---- control flow, as ppc-kit spells it -------------------------------------------------------------- */
#if defined(__wasm__) && !defined(__wasm_tail_call__)
#define TAILCALL(f) do { f(c); return; } while (0)
#else
#define TAILCALL(f) __attribute__((musttail)) return f(c)
#endif
/* ---- the lifter's runtime (rt/a64_rt.c) ------------------------------------------------------------- */
typedef void (*A64Fn)(Ctx *A64_NOESC);
A64Fn a64_lookup(uint64_t addr); /* function at a guest address, or NULL */
void a64_icall(Ctx *A64_NOESC c, uint64_t target); /* BLR: indirect call */
/* (hookshot) setjmp in lifted code: a call of a function that calls __setjmp is wrapped in a host __builtin_setjmp
on a slot (pushed per thread with that function's setjmp return addresses, popped after the call); the guest
longjmp's final BR to one of them comes back to the newest such slot through the host's a64_call_miss
(host/hos_setjmp.c), and the caller runs the function's code from that address (recomp_sj_target) */
void **recomp_sj_push(Ctx *A64_NOESC c, uint64_t r0, uint64_t r1, uint64_t r2, uint64_t r3);
uint64_t recomp_sj_target(void **slot);
void recomp_sj_pop(void **slot);
void a64_ijump(Ctx *A64_NOESC c, uint64_t target); /* BR that is not a jump table: indirect tail call */
/* The lifted BLR / BR: a cache per call site, filled by its first target (the guest address in the top 32 bits, the
function as an offset from a64_lookup in the low 32). A site calling that target again reads no shared table and
has an indirect branch of its own to predict; other targets go through a64_icall_site / a64_ijump_site, which try
the site's second slot (its second target) and else look them up as a64_icall does. A64_NO_SITE_CACHE: the plain
calls. */
void a64_icall_site(Ctx *A64_NOESC c, uint64_t target, uint64_t *site);
void a64_ijump_site(Ctx *A64_NOESC c, uint64_t target, uint64_t *site);
#ifdef A64_NO_SITE_CACHE
#define A64_ICALL(c, t) a64_icall(c, t)
#define A64_IJUMP(c, t) a64_ijump(c, t)
#else
#define A64_SITE_CALL(c, t, slow) do { \
static uint64_t a64_site_[2]; \
const uint64_t a64_site_t_ = (t), a64_site_e_ = __atomic_load_n(&a64_site_[0], __ATOMIC_RELAXED); \
if (__builtin_expect(a64_site_e_ >> 32 == a64_site_t_ && a64_site_e_ != 0, 1)) \
((A64Fn)((intptr_t)&a64_lookup + (int32_t)(uint32_t)a64_site_e_))(c); \
else slow(c, a64_site_t_, a64_site_); \
} while (0) /* (its names must not be ones the lifted code passes in, such as _t) */
#define A64_ICALL(c, t) A64_SITE_CALL(c, t, a64_icall_site)
#define A64_IJUMP(c, t) A64_SITE_CALL(c, t, a64_ijump_site)
#endif
void a64_svc(Ctx *A64_NOESC c, uint32_t imm); /* SVC #imm (sdk only) */
void a64_trap(Ctx *A64_NOESC c, uint64_t pc, uint32_t insn); /* BRK, HLT, UDF and words that are not code */
void a64_ret_mismatch(Ctx *A64_NOESC c, uint64_t pc); /* RET to something other than the caller (debug builds) */
#include "a64_suyu.h"
#include "a64_fpx.h"
/* ---- A64_INLINE_RT (experiment, off): LDXR/STXR, CLREX, DMB and the high multiplies inline in lifted code instead of
calls to a64_rt.c / a64_suyu.c. Measured 2026-10-21 (castle, 3 interleaved rounds at -O3): 69.5 -> 68.4 fps, game
threads +0.7 ms CPU a frame, though an LDXR/STXR pair becomes one inline lock cmpxchg: not kept. */
#if defined(A64_INLINE_RT) && !defined(A64_RT_IMPL)
A64_INLINE uint64_t a64_i_load_n(uint64_t a, uint32_t size) {
switch (size) {
case 1: return __atomic_load_n((uint8_t *)A64_PTR(a), __ATOMIC_SEQ_CST);
case 2: return __atomic_load_n((uint16_t *)A64_PTR(a), __ATOMIC_SEQ_CST);
case 4: return __atomic_load_n((uint32_t *)A64_PTR(a), __ATOMIC_SEQ_CST);
default: return __atomic_load_n((uint64_t *)A64_PTR(a), __ATOMIC_SEQ_CST);
}
}
A64_INLINE int a64_i_cas_n(uint64_t a, uint32_t size, uint64_t expect, uint64_t value) {
A64_MARK(a, size);
switch (size) {
case 1: { uint8_t e = (uint8_t)expect; return __atomic_compare_exchange_n((uint8_t *)A64_PTR(a), &e, (uint8_t)value, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); }
case 2: { uint16_t e = (uint16_t)expect; return __atomic_compare_exchange_n((uint16_t *)A64_PTR(a), &e, (uint16_t)value, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); }
case 4: { uint32_t e = (uint32_t)expect; return __atomic_compare_exchange_n((uint32_t *)A64_PTR(a), &e, (uint32_t)value, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); }
default: { uint64_t e = expect; return __atomic_compare_exchange_n((uint64_t *)A64_PTR(a), &e, value, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); }
}
}
A64_INLINE uint64_t a64_i_ldxr(GuestContext *c, uint64_t addr, uint32_t size) {
const uint64_t v = a64_i_load_n(addr, size);
c->excl_addr = addr; c->excl_size = size; c->excl_lo = v; c->excl_valid = 1;
return v;
}
A64_INLINE uint32_t a64_i_stxr(GuestContext *c, uint64_t addr, uint32_t size, uint64_t value) {
const int ok = c->excl_valid && c->excl_addr == addr && c->excl_size == size && a64_i_cas_n(addr, size, c->excl_lo, value);
c->excl_valid = 0;
return ok ? 0 : 1;
}
A64_INLINE void a64_i_clrex(GuestContext *c) { c->excl_valid = 0; }
A64_INLINE void a64_i_barrier(void) { __atomic_thread_fence(__ATOMIC_SEQ_CST); }
#if defined(__SIZEOF_INT128__)
A64_INLINE uint64_t a64_i_umulh(uint64_t a, uint64_t b) { return (uint64_t)((unsigned __int128)a * b >> 64); }
A64_INLINE uint64_t a64_i_smulh(uint64_t a, uint64_t b) { return (uint64_t)((__int128)(int64_t)a * (int64_t)b >> 64); }
#define recomp_umulh a64_i_umulh
#define recomp_smulh a64_i_smulh
#endif
#define recomp_ldxr a64_i_ldxr
#define recomp_stxr a64_i_stxr
#define recomp_clrex a64_i_clrex
#define recomp_barrier a64_i_barrier
#endif
#endif

333
third_party/a64-kit/rt/a64_rt.c vendored Normal file
View File

@@ -0,0 +1,333 @@
/* a64-kit runtime for lifted code: dispatch, indirect branches, traps, the exclusive monitor, the counter.
The OS layer (hos/) overrides the weak hooks: a64_svc, a64_on_trap, a64_cntpct_host. */
#define A64_RT_IMPL /* (the out-of-line exclusive monitor and barrier: rt/a64.h inlines them in lifted code) */
#include "a64.h"
#include <stdio.h>
#include <stdlib.h>
#if defined(_WIN32)
#define WIN32_LEAN_AND_MEAN
#include <windows.h>
#else
#include <time.h>
#endif
#if defined(_MSC_VER) && !defined(__clang__)
#define A64_WEAK
#else
#define A64_WEAK __attribute__((weak))
#endif
extern const uint32_t a64_fn_count;
extern const uint32_t a64_fn_addr[];
extern const A64Fn a64_fn_ptr[];
/* Indirect calls (C++ virtual calls above all) look their target up here. A direct-mapped cache in front of the
binary search over every function: each slot is one 64-bit word (address << 32 | the function's offset from a64_lookup), written and read
whole, so threads racing on a slot see an old entry or a new one, never a mix. */
static uint64_t lookup_cache[1 << 18];
#define A64_NO_FN 0x7FFFFFFF /* a cached slot's offset when the address has no lifted function */
/* Addresses the OS layer made at run time that are plain host functions (host calls without user data: every NVN proc)
are answered like lifted functions, so a64_lookup's cache and each call site's slots hold them and a call goes
straight to the handler. Before, every NVN call (tens of thousands a frame, on the game's recording threads) went
site_call -> a64_lookup -> a64_call_miss -> its handler. */
A64_WEAK A64Fn a64_lookup_host(uint64_t addr) { (void)addr; return NULL; }
A64Fn a64_lookup(uint64_t addr) {
if (addr >> 32 || !addr) return NULL;
uint32_t a = (uint32_t)addr, lo = 0, hi = a64_fn_count;
uint64_t *slot = &lookup_cache[(a >> 2 ^ a >> 20) & 0x3FFFF];
const uint64_t e = __atomic_load_n(slot, __ATOMIC_RELAXED);
const intptr_t anchor = (intptr_t)&a64_lookup;
if ((uint32_t)(e >> 32) == a) return (int32_t)(uint32_t)e == A64_NO_FN ? NULL : (A64Fn)(anchor + (int32_t)(uint32_t)e);
while (lo < hi) {
uint32_t mid = (lo + hi) >> 1;
if (a64_fn_addr[mid] < a) lo = mid + 1;
else hi = mid;
}
if (lo < a64_fn_count && a64_fn_addr[lo] == a) {
const intptr_t delta = (intptr_t)a64_fn_ptr[lo] - anchor;
if (delta == (int32_t)delta) __atomic_store_n(slot, (uint64_t)a << 32 | (uint32_t)(int32_t)delta, __ATOMIC_RELAXED);
return a64_fn_ptr[lo];
}
A64Fn h = a64_lookup_host(addr);
if (h) {
const intptr_t delta = (intptr_t)h - anchor;
if (delta == (int32_t)delta && (int32_t)delta != A64_NO_FN)
__atomic_store_n(slot, (uint64_t)a << 32 | (uint32_t)(int32_t)delta, __ATOMIC_RELAXED);
return h;
}
/* (remembered too: calls into the OS layer's other host calls searched the whole table each time) */
__atomic_store_n(slot, (uint64_t)a << 32 | (uint32_t)A64_NO_FN, __ATOMIC_RELAXED);
return NULL;
}
/* Called when a thread cannot continue (trap, missing function). The OS layer decides what that means. */
A64_WEAK void a64_on_trap(Ctx *c, uint64_t pc, uint32_t insn, const char *why) {
fprintf(stderr, "[a64] %s at %#llx (insn %08x), lr %#llx sp %#llx\n", why, (unsigned long long)pc, insn,
(unsigned long long)c->x[30], (unsigned long long)c->x[31]);
fflush(stderr);
abort();
}
void a64_trap(Ctx *c, uint64_t pc, uint32_t insn) {
c->pc = pc;
c->halted = 1;
a64_on_trap(c, pc, insn, insn ? "trap" : "branch to a non-function");
}
/* Addresses the OS layer made at run time (host calls: e.g. NVN procs handed out by nvnDeviceGetProcAddress) are
not in the lifted function table; the OS layer claims them here. Returns 1 if it ran the call. */
A64_WEAK int a64_call_miss(Ctx *c, uint64_t target) { (void)c; (void)target; return 0; }
/* (hookshot) every BR that is not a jump table and misses its site's cache asks this first: a longjmp's BR to a
setjmp return address (host/hos_setjmp.c), which is an entry as well, must unwind the host stack rather than
call that entry (it does not return when it is one). Targets it takes are never cached at the site. */
A64_WEAK void a64_ijump_hook(Ctx *c, uint64_t target) { (void)c; (void)target; }
void a64_icall(Ctx *c, uint64_t target) {
A64Fn f = a64_lookup(target);
if (!f) {
if (a64_call_miss(c, target)) return;
c->pc = target;
c->halted = 1;
a64_on_trap(c, target, 0, "indirect call to an address with no function");
return;
}
f(c);
}
void a64_ijump(Ctx *c, uint64_t target) {
a64_ijump_hook(c, target);
A64Fn f = a64_lookup(target);
if (!f) {
if (a64_call_miss(c, target)) return;
c->pc = target;
c->halted = 1;
a64_on_trap(c, target, 0, "indirect jump to an address with no function");
return;
}
f(c);
}
/* A64_SITE_PROFILE=<file> (profiling runs, for the lifter's A64_DEVIRT): call sites remember no targets, so every BLR
comes through here and counts as (call site, target), the site being the instruction before the return address the
lifted code put in x30. Every ~5 s the file is rewritten, "site target calls" a line (hex, hex, decimal);
scripts/devirt.py picks each site's dominant target from it. */
#define SITE_PROF_SLOTS (1u << 20)
static struct { uint64_t key, n; } *site_prof; /* key = site << 32 | target, open addressing */
static int site_prof_on = -1;
static volatile long site_prof_writing;
static uint64_t site_prof_last;
static uint64_t prof_ms(void) {
#if defined(_WIN32)
return GetTickCount64();
#else
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
return (uint64_t)ts.tv_sec * 1000 + (uint64_t)ts.tv_nsec / 1000000;
#endif
}
static void site_prof_write(void) {
if (__atomic_exchange_n(&site_prof_writing, 1, __ATOMIC_ACQUIRE)) return;
const uint64_t now = prof_ms();
if (now - site_prof_last >= 5000) {
site_prof_last = now;
FILE *f = fopen(getenv("A64_SITE_PROFILE"), "w");
if (f) {
for (uint32_t i = 0; i < SITE_PROF_SLOTS; i++) {
const uint64_t k = __atomic_load_n(&site_prof[i].key, __ATOMIC_RELAXED);
if (k) fprintf(f, "%x %x %llu\n", (uint32_t)(k >> 32), (uint32_t)k,
(unsigned long long)__atomic_load_n(&site_prof[i].n, __ATOMIC_RELAXED));
}
fclose(f);
}
}
__atomic_store_n(&site_prof_writing, 0, __ATOMIC_RELEASE);
}
static void site_prof_count(uint64_t site, uint64_t target) {
if (site >> 32 || target >> 32) return;
const uint64_t key = site << 32 | target;
uint32_t i = (uint32_t)((key * 0x9E3779B97F4A7C15ULL) >> 44);
for (uint32_t probe = 0; probe < 64; probe++, i = (i + 1) & (SITE_PROF_SLOTS - 1)) {
uint64_t k = __atomic_load_n(&site_prof[i].key, __ATOMIC_RELAXED);
if (!k && __atomic_compare_exchange_n(&site_prof[i].key, &k, key, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED)) k = key;
if (k == key) {
if ((__atomic_add_fetch(&site_prof[i].n, 1, __ATOMIC_RELAXED) & 0xFFFF) == 0) site_prof_write();
return;
}
}
}
static int site_profiling(void) {
if (site_prof_on < 0) {
int on = getenv("A64_SITE_PROFILE") != NULL;
if (on) {
site_prof = calloc(SITE_PROF_SLOTS, sizeof *site_prof);
on = site_prof != NULL;
site_prof_last = prof_ms();
}
site_prof_on = on;
}
return site_prof_on;
}
/* A64_ICALL / A64_IJUMP (a64.h) when the site's first slot does not hold the target: its second slot, else looked
up and remembered in a free slot (only the first two targets: a site calling many would otherwise keep rewriting a
line other threads read) */
static void site_call(Ctx *c, uint64_t target, uint64_t *site, int call, const char *why) {
if (site_prof_on && site_profiling()) { /* (A64_SITE_PROFILE: nothing remembered, every call counted) */
if (call) { /* (a BR's x30 is not its site) */
site_prof_count(c->x[30] - 4, target);
a64_icall(c, target);
} else a64_ijump(c, target);
return;
}
const uint64_t e = __atomic_load_n(&site[1], __ATOMIC_RELAXED);
if (e >> 32 == target && e) { /* (the site's second target) */
((A64Fn)((intptr_t)&a64_lookup + (int32_t)(uint32_t)e))(c);
return;
}
A64Fn f = a64_lookup(target);
if (!f) {
if (a64_call_miss(c, target)) return;
c->pc = target;
c->halted = 1;
a64_on_trap(c, target, 0, why);
return;
}
const intptr_t delta = (intptr_t)f - (intptr_t)&a64_lookup;
if (delta == (int32_t)delta) { /* (the first free slot of the two; a full site keeps its first two targets) */
const uint64_t entry = target << 32 | (uint32_t)(int32_t)delta;
if (!__atomic_load_n(&site[0], __ATOMIC_RELAXED)) __atomic_store_n(&site[0], entry, __ATOMIC_RELAXED);
else if (!e) __atomic_store_n(&site[1], entry, __ATOMIC_RELAXED);
}
f(c);
}
void a64_icall_site(Ctx *c, uint64_t target, uint64_t *site) {
site_call(c, target, site, 1, "indirect call to an address with no function");
}
void a64_ijump_site(Ctx *c, uint64_t target, uint64_t *site) {
a64_ijump_hook(c, target);
site_call(c, target, site, 0, "indirect jump to an address with no function");
}
A64_WEAK void a64_svc(Ctx *c, uint32_t imm) {
char why[32];
snprintf(why, sizeof why, "svc #%#x", imm);
a64_on_trap(c, c->pc, 0xD4000001u | (imm << 5), why);
}
void a64_ret_mismatch(Ctx *c, uint64_t pc) { a64_on_trap(c, pc, 0, "return to an address that is not the caller"); }
/* Generated import stubs (gen/a64_stubs.c) land here: log the first call, return 0. */
A64_WEAK void a64_unimplemented(Ctx *c, const char *sym, int *seen) {
if (!*seen) {
*seen = 1;
fprintf(stderr, "[a64] unimplemented import %s (lr %#llx)\n", sym, (unsigned long long)c->x[30]);
}
c->x[0] = 0;
}
void recomp_unhandled(GuestContext *c, uint32_t insn, uint64_t pc) { a64_trap(c, pc, insn); }
void recomp_ic_ivau(GuestContext *c, uint64_t address, int v) { (void)c; (void)address; (void)v; }
void recomp_barrier(void) { __atomic_thread_fence(__ATOMIC_SEQ_CST); }
/* ---- exclusive monitor --------------------------------------------------------------------------------
LDXR remembers (address, size, value). STXR succeeds only if the address and size match and memory still
holds that value, by compare-and-swap, so a lock taken by one host thread is respected by every other. */
static uint64_t load_n(uint64_t a, uint32_t size) {
switch (size) {
case 1: return __atomic_load_n((uint8_t *)A64_PTR(a), __ATOMIC_SEQ_CST);
case 2: return __atomic_load_n((uint16_t *)A64_PTR(a), __ATOMIC_SEQ_CST);
case 4: return __atomic_load_n((uint32_t *)A64_PTR(a), __ATOMIC_SEQ_CST);
default: return __atomic_load_n((uint64_t *)A64_PTR(a), __ATOMIC_SEQ_CST);
}
}
#if defined(A64_WRITE_TRACK)
uint8_t a64_dirty[1u << 20]; /* (rt/a64.h A64_MARK) */
#endif
static int cas_n(uint64_t a, uint32_t size, uint64_t expect, uint64_t value) {
A64_MARK(a, size); /* (marked whether or not it stores: harmless) */
switch (size) {
case 1: { uint8_t e = (uint8_t)expect; return __atomic_compare_exchange_n((uint8_t *)A64_PTR(a), &e, (uint8_t)value, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); }
case 2: { uint16_t e = (uint16_t)expect; return __atomic_compare_exchange_n((uint16_t *)A64_PTR(a), &e, (uint16_t)value, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); }
case 4: { uint32_t e = (uint32_t)expect; return __atomic_compare_exchange_n((uint32_t *)A64_PTR(a), &e, (uint32_t)value, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); }
default: { uint64_t e = expect; return __atomic_compare_exchange_n((uint64_t *)A64_PTR(a), &e, value, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); }
}
}
uint64_t recomp_ldxr(GuestContext *c, uint64_t addr, uint32_t size) {
uint64_t v = load_n(addr, size);
c->excl_addr = addr; c->excl_size = size; c->excl_lo = v; c->excl_valid = 1;
return v;
}
uint32_t recomp_stxr(GuestContext *c, uint64_t addr, uint32_t size, uint64_t value) {
int ok = c->excl_valid && c->excl_addr == addr && c->excl_size == size && cas_n(addr, size, c->excl_lo, value);
c->excl_valid = 0;
return ok ? 0 : 1;
}
void recomp_ldxp(GuestContext *c, uint64_t addr, uint32_t size, uint64_t *lo, uint64_t *hi) {
if (size == 4) {
uint64_t v = load_n(addr, 8);
*lo = (uint32_t)v; *hi = v >> 32;
} else {
/* 128-bit: two loads. Single-copy atomicity of the pair is not guaranteed here; the STXP below is. */
*lo = load_n(addr, 8); *hi = load_n(addr + 8, 8);
}
c->excl_addr = addr; c->excl_size = 2 * size; c->excl_lo = *lo; c->excl_hi = *hi; c->excl_valid = 1;
}
uint32_t recomp_stxp(GuestContext *c, uint64_t addr, uint32_t size, uint64_t lo, uint64_t hi) {
int ok = c->excl_valid && c->excl_addr == addr && c->excl_size == 2 * size;
if (ok && size == 4) {
ok = cas_n(addr, 8, (c->excl_lo & 0xFFFFFFFFu) | (c->excl_hi << 32), (lo & 0xFFFFFFFFu) | (hi << 32));
} else if (ok) {
#if defined(__SIZEOF_INT128__) && !defined(__wasm__)
unsigned __int128 e = ((unsigned __int128)c->excl_hi << 64) | c->excl_lo;
unsigned __int128 v = ((unsigned __int128)hi << 64) | lo;
ok = __atomic_compare_exchange_n((unsigned __int128 *)A64_PTR(addr), &e, v, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
#else
ok = load_n(addr, 8) == c->excl_lo && load_n(addr + 8, 8) == c->excl_hi;
if (ok) { recomp_store64(c, addr, lo); recomp_store64(c, addr + 8, hi); }
#endif
}
c->excl_valid = 0;
return ok ? 0 : 1;
}
void recomp_clrex(GuestContext *c) { c->excl_valid = 0; }
/* ---- CNTPCT_EL0: the 19.2 MHz counter ---------------------------------------------------------------- */
A64_WEAK uint64_t a64_cntpct_host(void) {
#if defined(_WIN32)
static LARGE_INTEGER base;
static uint64_t mult;
LARGE_INTEGER now;
if (!__atomic_load_n(&mult, __ATOMIC_ACQUIRE)) {
LARGE_INTEGER f, b;
QueryPerformanceFrequency(&f);
QueryPerformanceCounter(&b);
base = b;
__atomic_store_n(&mult, (19200000ULL << 32) / (uint64_t)f.QuadPart, __ATOMIC_RELEASE);
}
QueryPerformanceCounter(&now);
const uint64_t d = (uint64_t)(now.QuadPart - base.QuadPart);
return (uint64_t)(((unsigned __int128)d * mult) >> 32);
#else
static uint64_t base;
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
uint64_t ns = (uint64_t)ts.tv_sec * 1000000000ULL + ts.tv_nsec;
if (!base) base = ns;
return (ns - base) * 192ULL / 10000ULL;
#endif
}
uint64_t recomp_cntpct(GuestContext *c) { (void)c; return a64_cntpct_host(); }