mirror of
https://git.triforceproject.com/triforce/source
synced 2026-11-20 16:49:57 -04:00
Triforce v0.2.0
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
206
third_party/a64-kit/rt/a64.h
vendored
Normal file
206
third_party/a64-kit/rt/a64.h
vendored
Normal file
@@ -0,0 +1,206 @@
|
||||
/* a64-kit guest model: what every lifted AArch64 function sees.
|
||||
|
||||
The instruction bodies come from suyu's translator (lift/suyu/arm64_to_c.h), which writes C against a
|
||||
`GuestContext` and calls `recomp_load*` / `recomp_store*` / `recomp_ldxr` ...; this header supplies those
|
||||
for Rexcomp: one flat guest memory below 4 GB (`A64_MEM + (uint32_t)addr`), no page table, no host bridge.
|
||||
The FP helpers and the rest of suyu's runtime header are generated next to the lifted code
|
||||
(a64_suyu.h, by lift/a64_rtgen.py) and included at the end. */
|
||||
#ifndef A64_H
|
||||
#define A64_H
|
||||
#include <stdint.h>
|
||||
#include <stddef.h>
|
||||
#include <string.h>
|
||||
|
||||
typedef struct GuestContext {
|
||||
uint64_t x[32]; /* x[31] is SP: suyu's translator spells SP as x[31] where the encoding means SP */
|
||||
uint64_t pc; /* only written where the lifter stops a thread (traps, unhandled) */
|
||||
uint8_t n, z, c, v; /* NZCV, one byte each */
|
||||
int halted;
|
||||
uint64_t vreg[32][2]; /* Q0-Q31 as two 64-bit halves */
|
||||
uint64_t tpidr_el0; /* the guest's own thread pointer (C++ TLS) */
|
||||
uint64_t tpidrro_el0; /* Horizon's thread-local region, set by the OS layer */
|
||||
uint64_t fpcr, fpsr;
|
||||
/* Exclusive monitor (LDXR/STXR): the address, size and value LDXR saw. STXR compare-and-swaps against
|
||||
that value, so it works with real host threads (dynarmic's model). */
|
||||
uint64_t excl_addr, excl_lo, excl_hi;
|
||||
uint32_t excl_size;
|
||||
int excl_valid;
|
||||
void *thread; /* the OS layer's thread object */
|
||||
} GuestContext;
|
||||
typedef GuestContext Ctx;
|
||||
|
||||
/* A64_NOESC on every context parameter lifted code can call (the lifted functions, A64Fn, the runtime's and the OS
|
||||
layer's entry points): none of them keeps the pointer past the call. Without it each call "captures" the context, and
|
||||
LLVM must assume any guest load or store (an address made from an integer) might touch it: after every guest store
|
||||
it reloaded guest registers, and it kept every store to them. It also needs the lifted code built with a higher
|
||||
-capture-tracking-max-uses-to-explore (build.py): past 100 uses of the pointer LLVM stops looking and assumes
|
||||
it escaped, and a large lifted function uses it thousands of times. (Its effect on one of TOTK's hot functions:
|
||||
context loads 539 -> 278, stores 1068 -> 893, instructions -6%.) -DA64_NOESC= turns it off. */
|
||||
#ifndef A64_NOESC
|
||||
#if defined(__clang__)
|
||||
#define A64_NOESC __attribute__((noescape))
|
||||
#else
|
||||
#define A64_NOESC
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#if defined(_MSC_VER) && !defined(__clang__)
|
||||
#define A64_INLINE static __forceinline
|
||||
#else
|
||||
#define A64_INLINE static inline __attribute__((always_inline))
|
||||
#endif
|
||||
|
||||
/* Guest memory: the 4 GB guest window starts at a constant host address, so the base folds into every access
|
||||
instead of being reloaded from a global (with -fno-strict-aliasing any guest byte store could have changed
|
||||
that global). Natively the window is reserved at A64_HOST_BASE (hos_main.c); with A64_MEM_ZERO (web builds)
|
||||
guest address = wasm address, which needs -fno-delete-null-pointer-checks because guest address 0 is real. */
|
||||
#ifdef A64_MEM_ZERO
|
||||
#define A64_MEM ((uint8_t *)0)
|
||||
#else
|
||||
#ifndef A64_HOST_BASE
|
||||
#define A64_HOST_BASE 0x100000000000ULL /* 16 TB: far from anything Windows or glibc map by default */
|
||||
#endif
|
||||
#define A64_MEM ((uint8_t *)A64_HOST_BASE)
|
||||
#endif
|
||||
#define A64_PTR(a) (A64_MEM + (uint32_t)(a))
|
||||
|
||||
/* Browser builds (A64_WRITE_TRACK): wasm has no page write watch, so every guest store marks its 4 KB page (first
|
||||
and last byte's: an unaligned store may cross a page); hos/web/win32.c's GetWriteWatch reads and clears the marks
|
||||
for the renderer's buffer cache. */
|
||||
#if defined(A64_NO_MARKS) /* (measurement only: build.py RX_NO_GUEST_MARKS) */
|
||||
#define A64_MARK(a, n) ((void)0)
|
||||
#elif defined(A64_WRITE_TRACK) && !defined(__EMSCRIPTEN__)
|
||||
/* Native builds track the same way instead of with Windows' write watch (build.py: RX_SOFT_WRITE_WATCH=0), whose
|
||||
GetWriteWatch over the ~2 GB the GPU sees took 20-50 ms while the game streamed (hitches at its fences). The mark is a
|
||||
release store (a plain store on x86): the compiler may not move it above the data store, so a scan that clears a
|
||||
mark has seen the data it marks. (The marks are ~1 ms of the castle's game-thread CPU a frame. Marking the last
|
||||
byte's page only when a store crosses into it saved ~0.3 ms but made the code 20 MB bigger: not kept.) */
|
||||
extern uint8_t a64_dirty[1u << 20];
|
||||
/* (Tried 2026-10-21: reading a mark first and writing it only when clear, against cache lines moving between cores:
|
||||
game threads +1.1 ms a frame, the code 18 MB bigger. Not kept.) */
|
||||
#define A64_MARK(a, n) (__atomic_store_n(&a64_dirty[(uint32_t)(a) >> 12], 1, __ATOMIC_RELEASE), \
|
||||
__atomic_store_n(&a64_dirty[(uint32_t)((a) + (n) - 1) >> 12], 1, __ATOMIC_RELEASE))
|
||||
#elif defined(A64_WRITE_TRACK)
|
||||
extern uint8_t a64_dirty[1u << 20];
|
||||
#define A64_MARK(a, n) (a64_dirty[(uint32_t)(a) >> 12] = 1, a64_dirty[(uint32_t)((a) + (n) - 1) >> 12] = 1)
|
||||
#else
|
||||
#define A64_MARK(a, n) ((void)0)
|
||||
#endif
|
||||
|
||||
A64_INLINE uint64_t recomp_load8(GuestContext *c, uint64_t a) { (void)c; return *(const uint8_t *)A64_PTR(a); }
|
||||
A64_INLINE uint64_t recomp_load16(GuestContext *c, uint64_t a) { uint16_t v; (void)c; memcpy(&v, A64_PTR(a), 2); return v; }
|
||||
A64_INLINE uint64_t recomp_load32(GuestContext *c, uint64_t a) { uint32_t v; (void)c; memcpy(&v, A64_PTR(a), 4); return v; }
|
||||
A64_INLINE uint64_t recomp_load64(GuestContext *c, uint64_t a) { uint64_t v; (void)c; memcpy(&v, A64_PTR(a), 8); return v; }
|
||||
A64_INLINE void recomp_store8(GuestContext *c, uint64_t a, uint64_t v) { (void)c; *(uint8_t *)A64_PTR(a) = (uint8_t)v; A64_MARK(a, 1); }
|
||||
A64_INLINE void recomp_store16(GuestContext *c, uint64_t a, uint64_t v) { uint16_t t = (uint16_t)v; (void)c; memcpy(A64_PTR(a), &t, 2); A64_MARK(a, 2); }
|
||||
A64_INLINE void recomp_store32(GuestContext *c, uint64_t a, uint64_t v) { uint32_t t = (uint32_t)v; (void)c; memcpy(A64_PTR(a), &t, 4); A64_MARK(a, 4); }
|
||||
A64_INLINE void recomp_store64(GuestContext *c, uint64_t a, uint64_t v) { (void)c; memcpy(A64_PTR(a), &v, 8); A64_MARK(a, 8); }
|
||||
/* Stores through SP (stack_stores in lift/a64_lift.py): the guest stack is never memory the GPU reads, so they mark no
|
||||
page. They are a quarter of TOTK's store sites (prologue saves, nearly every STP). */
|
||||
A64_INLINE void recomp_store8_sp(GuestContext *c, uint64_t a, uint64_t v) { (void)c; *(uint8_t *)A64_PTR(a) = (uint8_t)v; }
|
||||
A64_INLINE void recomp_store16_sp(GuestContext *c, uint64_t a, uint64_t v) { uint16_t t = (uint16_t)v; (void)c; memcpy(A64_PTR(a), &t, 2); }
|
||||
A64_INLINE void recomp_store32_sp(GuestContext *c, uint64_t a, uint64_t v) { uint32_t t = (uint32_t)v; (void)c; memcpy(A64_PTR(a), &t, 4); }
|
||||
A64_INLINE void recomp_store64_sp(GuestContext *c, uint64_t a, uint64_t v) { (void)c; memcpy(A64_PTR(a), &v, 8); }
|
||||
A64_INLINE void recomp_stp32_sp(GuestContext *c, uint64_t a, uint64_t v0, uint64_t v1) { recomp_store32_sp(c, a, v0); recomp_store32_sp(c, a + 4, v1); }
|
||||
A64_INLINE void recomp_stp64_sp(GuestContext *c, uint64_t a, uint64_t v0, uint64_t v1) { recomp_store64_sp(c, a, v0); recomp_store64_sp(c, a + 8, v1); }
|
||||
A64_INLINE void recomp_ldp32(GuestContext *c, uint64_t a, uint64_t *lo, uint64_t *hi) { *lo = recomp_load32(c, a); *hi = recomp_load32(c, a + 4); }
|
||||
A64_INLINE void recomp_ldp64(GuestContext *c, uint64_t a, uint64_t *lo, uint64_t *hi) { *lo = recomp_load64(c, a); *hi = recomp_load64(c, a + 8); }
|
||||
/* (a pair marks once, for its whole span) */
|
||||
A64_INLINE void recomp_stp32(GuestContext *c, uint64_t a, uint64_t v0, uint64_t v1) { recomp_store32_sp(c, a, v0); recomp_store32_sp(c, a + 4, v1); A64_MARK(a, 8); }
|
||||
A64_INLINE void recomp_stp64(GuestContext *c, uint64_t a, uint64_t v0, uint64_t v1) { recomp_store64_sp(c, a, v0); recomp_store64_sp(c, a + 8, v1); A64_MARK(a, 16); }
|
||||
|
||||
/* Everything is lifted at its final guest address, so the translator's module base is a constant 0. */
|
||||
#define g_module_base ((uint64_t)0)
|
||||
|
||||
/* ---- control flow, as ppc-kit spells it -------------------------------------------------------------- */
|
||||
#if defined(__wasm__) && !defined(__wasm_tail_call__)
|
||||
#define TAILCALL(f) do { f(c); return; } while (0)
|
||||
#else
|
||||
#define TAILCALL(f) __attribute__((musttail)) return f(c)
|
||||
#endif
|
||||
|
||||
/* ---- the lifter's runtime (rt/a64_rt.c) ------------------------------------------------------------- */
|
||||
typedef void (*A64Fn)(Ctx *A64_NOESC);
|
||||
A64Fn a64_lookup(uint64_t addr); /* function at a guest address, or NULL */
|
||||
void a64_icall(Ctx *A64_NOESC c, uint64_t target); /* BLR: indirect call */
|
||||
/* (hookshot) setjmp in lifted code: a call of a function that calls __setjmp is wrapped in a host __builtin_setjmp
|
||||
on a slot (pushed per thread with that function's setjmp return addresses, popped after the call); the guest
|
||||
longjmp's final BR to one of them comes back to the newest such slot through the host's a64_call_miss
|
||||
(host/hos_setjmp.c), and the caller runs the function's code from that address (recomp_sj_target) */
|
||||
void **recomp_sj_push(Ctx *A64_NOESC c, uint64_t r0, uint64_t r1, uint64_t r2, uint64_t r3);
|
||||
uint64_t recomp_sj_target(void **slot);
|
||||
void recomp_sj_pop(void **slot);
|
||||
void a64_ijump(Ctx *A64_NOESC c, uint64_t target); /* BR that is not a jump table: indirect tail call */
|
||||
/* The lifted BLR / BR: a cache per call site, filled by its first target (the guest address in the top 32 bits, the
|
||||
function as an offset from a64_lookup in the low 32). A site calling that target again reads no shared table and
|
||||
has an indirect branch of its own to predict; other targets go through a64_icall_site / a64_ijump_site, which try
|
||||
the site's second slot (its second target) and else look them up as a64_icall does. A64_NO_SITE_CACHE: the plain
|
||||
calls. */
|
||||
void a64_icall_site(Ctx *A64_NOESC c, uint64_t target, uint64_t *site);
|
||||
void a64_ijump_site(Ctx *A64_NOESC c, uint64_t target, uint64_t *site);
|
||||
#ifdef A64_NO_SITE_CACHE
|
||||
#define A64_ICALL(c, t) a64_icall(c, t)
|
||||
#define A64_IJUMP(c, t) a64_ijump(c, t)
|
||||
#else
|
||||
#define A64_SITE_CALL(c, t, slow) do { \
|
||||
static uint64_t a64_site_[2]; \
|
||||
const uint64_t a64_site_t_ = (t), a64_site_e_ = __atomic_load_n(&a64_site_[0], __ATOMIC_RELAXED); \
|
||||
if (__builtin_expect(a64_site_e_ >> 32 == a64_site_t_ && a64_site_e_ != 0, 1)) \
|
||||
((A64Fn)((intptr_t)&a64_lookup + (int32_t)(uint32_t)a64_site_e_))(c); \
|
||||
else slow(c, a64_site_t_, a64_site_); \
|
||||
} while (0) /* (its names must not be ones the lifted code passes in, such as _t) */
|
||||
#define A64_ICALL(c, t) A64_SITE_CALL(c, t, a64_icall_site)
|
||||
#define A64_IJUMP(c, t) A64_SITE_CALL(c, t, a64_ijump_site)
|
||||
#endif
|
||||
void a64_svc(Ctx *A64_NOESC c, uint32_t imm); /* SVC #imm (sdk only) */
|
||||
void a64_trap(Ctx *A64_NOESC c, uint64_t pc, uint32_t insn); /* BRK, HLT, UDF and words that are not code */
|
||||
void a64_ret_mismatch(Ctx *A64_NOESC c, uint64_t pc); /* RET to something other than the caller (debug builds) */
|
||||
|
||||
#include "a64_suyu.h"
|
||||
#include "a64_fpx.h"
|
||||
|
||||
/* ---- A64_INLINE_RT (experiment, off): LDXR/STXR, CLREX, DMB and the high multiplies inline in lifted code instead of
|
||||
calls to a64_rt.c / a64_suyu.c. Measured 2026-10-21 (castle, 3 interleaved rounds at -O3): 69.5 -> 68.4 fps, game
|
||||
threads +0.7 ms CPU a frame, though an LDXR/STXR pair becomes one inline lock cmpxchg: not kept. */
|
||||
#if defined(A64_INLINE_RT) && !defined(A64_RT_IMPL)
|
||||
A64_INLINE uint64_t a64_i_load_n(uint64_t a, uint32_t size) {
|
||||
switch (size) {
|
||||
case 1: return __atomic_load_n((uint8_t *)A64_PTR(a), __ATOMIC_SEQ_CST);
|
||||
case 2: return __atomic_load_n((uint16_t *)A64_PTR(a), __ATOMIC_SEQ_CST);
|
||||
case 4: return __atomic_load_n((uint32_t *)A64_PTR(a), __ATOMIC_SEQ_CST);
|
||||
default: return __atomic_load_n((uint64_t *)A64_PTR(a), __ATOMIC_SEQ_CST);
|
||||
}
|
||||
}
|
||||
A64_INLINE int a64_i_cas_n(uint64_t a, uint32_t size, uint64_t expect, uint64_t value) {
|
||||
A64_MARK(a, size);
|
||||
switch (size) {
|
||||
case 1: { uint8_t e = (uint8_t)expect; return __atomic_compare_exchange_n((uint8_t *)A64_PTR(a), &e, (uint8_t)value, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); }
|
||||
case 2: { uint16_t e = (uint16_t)expect; return __atomic_compare_exchange_n((uint16_t *)A64_PTR(a), &e, (uint16_t)value, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); }
|
||||
case 4: { uint32_t e = (uint32_t)expect; return __atomic_compare_exchange_n((uint32_t *)A64_PTR(a), &e, (uint32_t)value, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); }
|
||||
default: { uint64_t e = expect; return __atomic_compare_exchange_n((uint64_t *)A64_PTR(a), &e, value, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); }
|
||||
}
|
||||
}
|
||||
A64_INLINE uint64_t a64_i_ldxr(GuestContext *c, uint64_t addr, uint32_t size) {
|
||||
const uint64_t v = a64_i_load_n(addr, size);
|
||||
c->excl_addr = addr; c->excl_size = size; c->excl_lo = v; c->excl_valid = 1;
|
||||
return v;
|
||||
}
|
||||
A64_INLINE uint32_t a64_i_stxr(GuestContext *c, uint64_t addr, uint32_t size, uint64_t value) {
|
||||
const int ok = c->excl_valid && c->excl_addr == addr && c->excl_size == size && a64_i_cas_n(addr, size, c->excl_lo, value);
|
||||
c->excl_valid = 0;
|
||||
return ok ? 0 : 1;
|
||||
}
|
||||
A64_INLINE void a64_i_clrex(GuestContext *c) { c->excl_valid = 0; }
|
||||
A64_INLINE void a64_i_barrier(void) { __atomic_thread_fence(__ATOMIC_SEQ_CST); }
|
||||
#if defined(__SIZEOF_INT128__)
|
||||
A64_INLINE uint64_t a64_i_umulh(uint64_t a, uint64_t b) { return (uint64_t)((unsigned __int128)a * b >> 64); }
|
||||
A64_INLINE uint64_t a64_i_smulh(uint64_t a, uint64_t b) { return (uint64_t)((__int128)(int64_t)a * (int64_t)b >> 64); }
|
||||
#define recomp_umulh a64_i_umulh
|
||||
#define recomp_smulh a64_i_smulh
|
||||
#endif
|
||||
#define recomp_ldxr a64_i_ldxr
|
||||
#define recomp_stxr a64_i_stxr
|
||||
#define recomp_clrex a64_i_clrex
|
||||
#define recomp_barrier a64_i_barrier
|
||||
#endif
|
||||
#endif
|
||||
333
third_party/a64-kit/rt/a64_rt.c
vendored
Normal file
333
third_party/a64-kit/rt/a64_rt.c
vendored
Normal file
@@ -0,0 +1,333 @@
|
||||
/* a64-kit runtime for lifted code: dispatch, indirect branches, traps, the exclusive monitor, the counter.
|
||||
The OS layer (hos/) overrides the weak hooks: a64_svc, a64_on_trap, a64_cntpct_host. */
|
||||
#define A64_RT_IMPL /* (the out-of-line exclusive monitor and barrier: rt/a64.h inlines them in lifted code) */
|
||||
#include "a64.h"
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#if defined(_WIN32)
|
||||
#define WIN32_LEAN_AND_MEAN
|
||||
#include <windows.h>
|
||||
#else
|
||||
#include <time.h>
|
||||
#endif
|
||||
|
||||
#if defined(_MSC_VER) && !defined(__clang__)
|
||||
#define A64_WEAK
|
||||
#else
|
||||
#define A64_WEAK __attribute__((weak))
|
||||
#endif
|
||||
|
||||
extern const uint32_t a64_fn_count;
|
||||
extern const uint32_t a64_fn_addr[];
|
||||
extern const A64Fn a64_fn_ptr[];
|
||||
|
||||
/* Indirect calls (C++ virtual calls above all) look their target up here. A direct-mapped cache in front of the
|
||||
binary search over every function: each slot is one 64-bit word (address << 32 | the function's offset from a64_lookup), written and read
|
||||
whole, so threads racing on a slot see an old entry or a new one, never a mix. */
|
||||
static uint64_t lookup_cache[1 << 18];
|
||||
#define A64_NO_FN 0x7FFFFFFF /* a cached slot's offset when the address has no lifted function */
|
||||
|
||||
/* Addresses the OS layer made at run time that are plain host functions (host calls without user data: every NVN proc)
|
||||
are answered like lifted functions, so a64_lookup's cache and each call site's slots hold them and a call goes
|
||||
straight to the handler. Before, every NVN call (tens of thousands a frame, on the game's recording threads) went
|
||||
site_call -> a64_lookup -> a64_call_miss -> its handler. */
|
||||
A64_WEAK A64Fn a64_lookup_host(uint64_t addr) { (void)addr; return NULL; }
|
||||
|
||||
A64Fn a64_lookup(uint64_t addr) {
|
||||
if (addr >> 32 || !addr) return NULL;
|
||||
uint32_t a = (uint32_t)addr, lo = 0, hi = a64_fn_count;
|
||||
uint64_t *slot = &lookup_cache[(a >> 2 ^ a >> 20) & 0x3FFFF];
|
||||
const uint64_t e = __atomic_load_n(slot, __ATOMIC_RELAXED);
|
||||
const intptr_t anchor = (intptr_t)&a64_lookup;
|
||||
if ((uint32_t)(e >> 32) == a) return (int32_t)(uint32_t)e == A64_NO_FN ? NULL : (A64Fn)(anchor + (int32_t)(uint32_t)e);
|
||||
while (lo < hi) {
|
||||
uint32_t mid = (lo + hi) >> 1;
|
||||
if (a64_fn_addr[mid] < a) lo = mid + 1;
|
||||
else hi = mid;
|
||||
}
|
||||
if (lo < a64_fn_count && a64_fn_addr[lo] == a) {
|
||||
const intptr_t delta = (intptr_t)a64_fn_ptr[lo] - anchor;
|
||||
if (delta == (int32_t)delta) __atomic_store_n(slot, (uint64_t)a << 32 | (uint32_t)(int32_t)delta, __ATOMIC_RELAXED);
|
||||
return a64_fn_ptr[lo];
|
||||
}
|
||||
A64Fn h = a64_lookup_host(addr);
|
||||
if (h) {
|
||||
const intptr_t delta = (intptr_t)h - anchor;
|
||||
if (delta == (int32_t)delta && (int32_t)delta != A64_NO_FN)
|
||||
__atomic_store_n(slot, (uint64_t)a << 32 | (uint32_t)(int32_t)delta, __ATOMIC_RELAXED);
|
||||
return h;
|
||||
}
|
||||
/* (remembered too: calls into the OS layer's other host calls searched the whole table each time) */
|
||||
__atomic_store_n(slot, (uint64_t)a << 32 | (uint32_t)A64_NO_FN, __ATOMIC_RELAXED);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* Called when a thread cannot continue (trap, missing function). The OS layer decides what that means. */
|
||||
A64_WEAK void a64_on_trap(Ctx *c, uint64_t pc, uint32_t insn, const char *why) {
|
||||
fprintf(stderr, "[a64] %s at %#llx (insn %08x), lr %#llx sp %#llx\n", why, (unsigned long long)pc, insn,
|
||||
(unsigned long long)c->x[30], (unsigned long long)c->x[31]);
|
||||
fflush(stderr);
|
||||
abort();
|
||||
}
|
||||
|
||||
void a64_trap(Ctx *c, uint64_t pc, uint32_t insn) {
|
||||
c->pc = pc;
|
||||
c->halted = 1;
|
||||
a64_on_trap(c, pc, insn, insn ? "trap" : "branch to a non-function");
|
||||
}
|
||||
|
||||
/* Addresses the OS layer made at run time (host calls: e.g. NVN procs handed out by nvnDeviceGetProcAddress) are
|
||||
not in the lifted function table; the OS layer claims them here. Returns 1 if it ran the call. */
|
||||
A64_WEAK int a64_call_miss(Ctx *c, uint64_t target) { (void)c; (void)target; return 0; }
|
||||
/* (hookshot) every BR that is not a jump table and misses its site's cache asks this first: a longjmp's BR to a
|
||||
setjmp return address (host/hos_setjmp.c), which is an entry as well, must unwind the host stack rather than
|
||||
call that entry (it does not return when it is one). Targets it takes are never cached at the site. */
|
||||
A64_WEAK void a64_ijump_hook(Ctx *c, uint64_t target) { (void)c; (void)target; }
|
||||
|
||||
void a64_icall(Ctx *c, uint64_t target) {
|
||||
A64Fn f = a64_lookup(target);
|
||||
if (!f) {
|
||||
if (a64_call_miss(c, target)) return;
|
||||
c->pc = target;
|
||||
c->halted = 1;
|
||||
a64_on_trap(c, target, 0, "indirect call to an address with no function");
|
||||
return;
|
||||
}
|
||||
f(c);
|
||||
}
|
||||
|
||||
void a64_ijump(Ctx *c, uint64_t target) {
|
||||
a64_ijump_hook(c, target);
|
||||
A64Fn f = a64_lookup(target);
|
||||
if (!f) {
|
||||
if (a64_call_miss(c, target)) return;
|
||||
c->pc = target;
|
||||
c->halted = 1;
|
||||
a64_on_trap(c, target, 0, "indirect jump to an address with no function");
|
||||
return;
|
||||
}
|
||||
f(c);
|
||||
}
|
||||
|
||||
/* A64_SITE_PROFILE=<file> (profiling runs, for the lifter's A64_DEVIRT): call sites remember no targets, so every BLR
|
||||
comes through here and counts as (call site, target), the site being the instruction before the return address the
|
||||
lifted code put in x30. Every ~5 s the file is rewritten, "site target calls" a line (hex, hex, decimal);
|
||||
scripts/devirt.py picks each site's dominant target from it. */
|
||||
#define SITE_PROF_SLOTS (1u << 20)
|
||||
static struct { uint64_t key, n; } *site_prof; /* key = site << 32 | target, open addressing */
|
||||
static int site_prof_on = -1;
|
||||
static volatile long site_prof_writing;
|
||||
static uint64_t site_prof_last;
|
||||
|
||||
static uint64_t prof_ms(void) {
|
||||
#if defined(_WIN32)
|
||||
return GetTickCount64();
|
||||
#else
|
||||
struct timespec ts;
|
||||
clock_gettime(CLOCK_MONOTONIC, &ts);
|
||||
return (uint64_t)ts.tv_sec * 1000 + (uint64_t)ts.tv_nsec / 1000000;
|
||||
#endif
|
||||
}
|
||||
|
||||
static void site_prof_write(void) {
|
||||
if (__atomic_exchange_n(&site_prof_writing, 1, __ATOMIC_ACQUIRE)) return;
|
||||
const uint64_t now = prof_ms();
|
||||
if (now - site_prof_last >= 5000) {
|
||||
site_prof_last = now;
|
||||
FILE *f = fopen(getenv("A64_SITE_PROFILE"), "w");
|
||||
if (f) {
|
||||
for (uint32_t i = 0; i < SITE_PROF_SLOTS; i++) {
|
||||
const uint64_t k = __atomic_load_n(&site_prof[i].key, __ATOMIC_RELAXED);
|
||||
if (k) fprintf(f, "%x %x %llu\n", (uint32_t)(k >> 32), (uint32_t)k,
|
||||
(unsigned long long)__atomic_load_n(&site_prof[i].n, __ATOMIC_RELAXED));
|
||||
}
|
||||
fclose(f);
|
||||
}
|
||||
}
|
||||
__atomic_store_n(&site_prof_writing, 0, __ATOMIC_RELEASE);
|
||||
}
|
||||
|
||||
static void site_prof_count(uint64_t site, uint64_t target) {
|
||||
if (site >> 32 || target >> 32) return;
|
||||
const uint64_t key = site << 32 | target;
|
||||
uint32_t i = (uint32_t)((key * 0x9E3779B97F4A7C15ULL) >> 44);
|
||||
for (uint32_t probe = 0; probe < 64; probe++, i = (i + 1) & (SITE_PROF_SLOTS - 1)) {
|
||||
uint64_t k = __atomic_load_n(&site_prof[i].key, __ATOMIC_RELAXED);
|
||||
if (!k && __atomic_compare_exchange_n(&site_prof[i].key, &k, key, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED)) k = key;
|
||||
if (k == key) {
|
||||
if ((__atomic_add_fetch(&site_prof[i].n, 1, __ATOMIC_RELAXED) & 0xFFFF) == 0) site_prof_write();
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static int site_profiling(void) {
|
||||
if (site_prof_on < 0) {
|
||||
int on = getenv("A64_SITE_PROFILE") != NULL;
|
||||
if (on) {
|
||||
site_prof = calloc(SITE_PROF_SLOTS, sizeof *site_prof);
|
||||
on = site_prof != NULL;
|
||||
site_prof_last = prof_ms();
|
||||
}
|
||||
site_prof_on = on;
|
||||
}
|
||||
return site_prof_on;
|
||||
}
|
||||
|
||||
/* A64_ICALL / A64_IJUMP (a64.h) when the site's first slot does not hold the target: its second slot, else looked
|
||||
up and remembered in a free slot (only the first two targets: a site calling many would otherwise keep rewriting a
|
||||
line other threads read) */
|
||||
static void site_call(Ctx *c, uint64_t target, uint64_t *site, int call, const char *why) {
|
||||
if (site_prof_on && site_profiling()) { /* (A64_SITE_PROFILE: nothing remembered, every call counted) */
|
||||
if (call) { /* (a BR's x30 is not its site) */
|
||||
site_prof_count(c->x[30] - 4, target);
|
||||
a64_icall(c, target);
|
||||
} else a64_ijump(c, target);
|
||||
return;
|
||||
}
|
||||
const uint64_t e = __atomic_load_n(&site[1], __ATOMIC_RELAXED);
|
||||
if (e >> 32 == target && e) { /* (the site's second target) */
|
||||
((A64Fn)((intptr_t)&a64_lookup + (int32_t)(uint32_t)e))(c);
|
||||
return;
|
||||
}
|
||||
A64Fn f = a64_lookup(target);
|
||||
if (!f) {
|
||||
if (a64_call_miss(c, target)) return;
|
||||
c->pc = target;
|
||||
c->halted = 1;
|
||||
a64_on_trap(c, target, 0, why);
|
||||
return;
|
||||
}
|
||||
const intptr_t delta = (intptr_t)f - (intptr_t)&a64_lookup;
|
||||
if (delta == (int32_t)delta) { /* (the first free slot of the two; a full site keeps its first two targets) */
|
||||
const uint64_t entry = target << 32 | (uint32_t)(int32_t)delta;
|
||||
if (!__atomic_load_n(&site[0], __ATOMIC_RELAXED)) __atomic_store_n(&site[0], entry, __ATOMIC_RELAXED);
|
||||
else if (!e) __atomic_store_n(&site[1], entry, __ATOMIC_RELAXED);
|
||||
}
|
||||
f(c);
|
||||
}
|
||||
void a64_icall_site(Ctx *c, uint64_t target, uint64_t *site) {
|
||||
site_call(c, target, site, 1, "indirect call to an address with no function");
|
||||
}
|
||||
void a64_ijump_site(Ctx *c, uint64_t target, uint64_t *site) {
|
||||
a64_ijump_hook(c, target);
|
||||
site_call(c, target, site, 0, "indirect jump to an address with no function");
|
||||
}
|
||||
|
||||
A64_WEAK void a64_svc(Ctx *c, uint32_t imm) {
|
||||
char why[32];
|
||||
snprintf(why, sizeof why, "svc #%#x", imm);
|
||||
a64_on_trap(c, c->pc, 0xD4000001u | (imm << 5), why);
|
||||
}
|
||||
|
||||
void a64_ret_mismatch(Ctx *c, uint64_t pc) { a64_on_trap(c, pc, 0, "return to an address that is not the caller"); }
|
||||
|
||||
/* Generated import stubs (gen/a64_stubs.c) land here: log the first call, return 0. */
|
||||
A64_WEAK void a64_unimplemented(Ctx *c, const char *sym, int *seen) {
|
||||
if (!*seen) {
|
||||
*seen = 1;
|
||||
fprintf(stderr, "[a64] unimplemented import %s (lr %#llx)\n", sym, (unsigned long long)c->x[30]);
|
||||
}
|
||||
c->x[0] = 0;
|
||||
}
|
||||
|
||||
void recomp_unhandled(GuestContext *c, uint32_t insn, uint64_t pc) { a64_trap(c, pc, insn); }
|
||||
void recomp_ic_ivau(GuestContext *c, uint64_t address, int v) { (void)c; (void)address; (void)v; }
|
||||
void recomp_barrier(void) { __atomic_thread_fence(__ATOMIC_SEQ_CST); }
|
||||
|
||||
/* ---- exclusive monitor --------------------------------------------------------------------------------
|
||||
LDXR remembers (address, size, value). STXR succeeds only if the address and size match and memory still
|
||||
holds that value, by compare-and-swap, so a lock taken by one host thread is respected by every other. */
|
||||
static uint64_t load_n(uint64_t a, uint32_t size) {
|
||||
switch (size) {
|
||||
case 1: return __atomic_load_n((uint8_t *)A64_PTR(a), __ATOMIC_SEQ_CST);
|
||||
case 2: return __atomic_load_n((uint16_t *)A64_PTR(a), __ATOMIC_SEQ_CST);
|
||||
case 4: return __atomic_load_n((uint32_t *)A64_PTR(a), __ATOMIC_SEQ_CST);
|
||||
default: return __atomic_load_n((uint64_t *)A64_PTR(a), __ATOMIC_SEQ_CST);
|
||||
}
|
||||
}
|
||||
|
||||
#if defined(A64_WRITE_TRACK)
|
||||
uint8_t a64_dirty[1u << 20]; /* (rt/a64.h A64_MARK) */
|
||||
#endif
|
||||
|
||||
static int cas_n(uint64_t a, uint32_t size, uint64_t expect, uint64_t value) {
|
||||
A64_MARK(a, size); /* (marked whether or not it stores: harmless) */
|
||||
switch (size) {
|
||||
case 1: { uint8_t e = (uint8_t)expect; return __atomic_compare_exchange_n((uint8_t *)A64_PTR(a), &e, (uint8_t)value, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); }
|
||||
case 2: { uint16_t e = (uint16_t)expect; return __atomic_compare_exchange_n((uint16_t *)A64_PTR(a), &e, (uint16_t)value, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); }
|
||||
case 4: { uint32_t e = (uint32_t)expect; return __atomic_compare_exchange_n((uint32_t *)A64_PTR(a), &e, (uint32_t)value, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); }
|
||||
default: { uint64_t e = expect; return __atomic_compare_exchange_n((uint64_t *)A64_PTR(a), &e, value, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); }
|
||||
}
|
||||
}
|
||||
|
||||
uint64_t recomp_ldxr(GuestContext *c, uint64_t addr, uint32_t size) {
|
||||
uint64_t v = load_n(addr, size);
|
||||
c->excl_addr = addr; c->excl_size = size; c->excl_lo = v; c->excl_valid = 1;
|
||||
return v;
|
||||
}
|
||||
|
||||
uint32_t recomp_stxr(GuestContext *c, uint64_t addr, uint32_t size, uint64_t value) {
|
||||
int ok = c->excl_valid && c->excl_addr == addr && c->excl_size == size && cas_n(addr, size, c->excl_lo, value);
|
||||
c->excl_valid = 0;
|
||||
return ok ? 0 : 1;
|
||||
}
|
||||
|
||||
void recomp_ldxp(GuestContext *c, uint64_t addr, uint32_t size, uint64_t *lo, uint64_t *hi) {
|
||||
if (size == 4) {
|
||||
uint64_t v = load_n(addr, 8);
|
||||
*lo = (uint32_t)v; *hi = v >> 32;
|
||||
} else {
|
||||
/* 128-bit: two loads. Single-copy atomicity of the pair is not guaranteed here; the STXP below is. */
|
||||
*lo = load_n(addr, 8); *hi = load_n(addr + 8, 8);
|
||||
}
|
||||
c->excl_addr = addr; c->excl_size = 2 * size; c->excl_lo = *lo; c->excl_hi = *hi; c->excl_valid = 1;
|
||||
}
|
||||
|
||||
uint32_t recomp_stxp(GuestContext *c, uint64_t addr, uint32_t size, uint64_t lo, uint64_t hi) {
|
||||
int ok = c->excl_valid && c->excl_addr == addr && c->excl_size == 2 * size;
|
||||
if (ok && size == 4) {
|
||||
ok = cas_n(addr, 8, (c->excl_lo & 0xFFFFFFFFu) | (c->excl_hi << 32), (lo & 0xFFFFFFFFu) | (hi << 32));
|
||||
} else if (ok) {
|
||||
#if defined(__SIZEOF_INT128__) && !defined(__wasm__)
|
||||
unsigned __int128 e = ((unsigned __int128)c->excl_hi << 64) | c->excl_lo;
|
||||
unsigned __int128 v = ((unsigned __int128)hi << 64) | lo;
|
||||
ok = __atomic_compare_exchange_n((unsigned __int128 *)A64_PTR(addr), &e, v, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
|
||||
#else
|
||||
ok = load_n(addr, 8) == c->excl_lo && load_n(addr + 8, 8) == c->excl_hi;
|
||||
if (ok) { recomp_store64(c, addr, lo); recomp_store64(c, addr + 8, hi); }
|
||||
#endif
|
||||
}
|
||||
c->excl_valid = 0;
|
||||
return ok ? 0 : 1;
|
||||
}
|
||||
|
||||
void recomp_clrex(GuestContext *c) { c->excl_valid = 0; }
|
||||
|
||||
/* ---- CNTPCT_EL0: the 19.2 MHz counter ---------------------------------------------------------------- */
|
||||
A64_WEAK uint64_t a64_cntpct_host(void) {
|
||||
#if defined(_WIN32)
|
||||
static LARGE_INTEGER base;
|
||||
static uint64_t mult;
|
||||
LARGE_INTEGER now;
|
||||
if (!__atomic_load_n(&mult, __ATOMIC_ACQUIRE)) {
|
||||
LARGE_INTEGER f, b;
|
||||
QueryPerformanceFrequency(&f);
|
||||
QueryPerformanceCounter(&b);
|
||||
base = b;
|
||||
__atomic_store_n(&mult, (19200000ULL << 32) / (uint64_t)f.QuadPart, __ATOMIC_RELEASE);
|
||||
}
|
||||
QueryPerformanceCounter(&now);
|
||||
const uint64_t d = (uint64_t)(now.QuadPart - base.QuadPart);
|
||||
return (uint64_t)(((unsigned __int128)d * mult) >> 32);
|
||||
#else
|
||||
static uint64_t base;
|
||||
struct timespec ts;
|
||||
clock_gettime(CLOCK_MONOTONIC, &ts);
|
||||
uint64_t ns = (uint64_t)ts.tv_sec * 1000000000ULL + ts.tv_nsec;
|
||||
if (!base) base = ns;
|
||||
return (ns - base) * 192ULL / 10000ULL;
|
||||
#endif
|
||||
}
|
||||
|
||||
uint64_t recomp_cntpct(GuestContext *c) { (void)c; return a64_cntpct_host(); }
|
||||
Reference in New Issue
Block a user