mirror of
https://git.triforceproject.com/triforce/source
synced 2026-11-20 16:49:57 -04:00
8095 lines
427 KiB
C++
8095 lines
427 KiB
C++
// SPDX-FileCopyrightText: Copyright 2026 suyu Emulator Project
|
|
// SPDX-License-Identifier: GPL-2.0-or-later
|
|
//
|
|
// Header-only AArch64 -> portable C static recompiler engine, shared by the standalone
|
|
// tools/static_recompiler CLI and the in-app game export feature.
|
|
//
|
|
// It decodes a subset of user-mode AArch64 and emits C against a GuestContext (N64Recomp-style).
|
|
// Every instruction either translates to native C or emits a runtime fallback, so the generated
|
|
// project ALWAYS builds into a native binary (Windows .exe / Linux+BSD ELF / macOS Mach-O) or can
|
|
// be emitted as plain C source. A full game additionally needs suyu's HLE/GPU runtime, wired in via
|
|
// the generated runtime's recomp_svc()/MMIO hooks.
|
|
|
|
#pragma once
|
|
|
|
#include <algorithm>
|
|
#include <iomanip>
|
|
#include <utility>
|
|
#include <array>
|
|
#include <cmath>
|
|
#include <cstdint>
|
|
#include <cstdio>
|
|
#include <cstdlib>
|
|
#include <cstring>
|
|
#include <filesystem>
|
|
#include <fstream>
|
|
#include <sstream>
|
|
#include <unordered_set>
|
|
#include <map>
|
|
#include <string>
|
|
#include <string_view>
|
|
#include <vector>
|
|
#ifdef _WIN32
|
|
#include <direct.h>
|
|
#else
|
|
#include <sys/stat.h>
|
|
#endif
|
|
|
|
namespace suyu::recomp {
|
|
|
|
using u8 = uint8_t;
|
|
using u32 = uint32_t;
|
|
using u64 = uint64_t;
|
|
using s32 = int32_t;
|
|
using s64 = int64_t;
|
|
|
|
// Output paths reach this engine as UTF-8 (the frontend hands over
|
|
// QString::toStdString()), but the narrow char overloads of _mkdir/mkdir and
|
|
// std::ofstream interpret their argument in the process code page - CP-1252 on
|
|
// a stock Windows install. A title whose name carries a non-ASCII character
|
|
// ("Pokemon Sword" reads as "Pokémon Sword" once the name comes from the
|
|
// ROM's own NACP) therefore had every directory creation and every file open
|
|
// silently fail, leaving empty module directories and a build that could not
|
|
// configure. Route both through std::filesystem::path, which is constructed
|
|
// from the UTF-8 bytes explicitly and uses the wide OS entry points.
|
|
inline std::filesystem::path Utf8Path(const std::string& s) {
|
|
return std::filesystem::path(
|
|
std::u8string(reinterpret_cast<const char8_t*>(s.data()), s.size()));
|
|
}
|
|
|
|
struct Block {
|
|
u64 vaddr;
|
|
u32 size;
|
|
u32 count;
|
|
bool is_entry;
|
|
};
|
|
|
|
inline bool IsTerminator(u32 i) {
|
|
if ((i & 0xFC000000) == 0x14000000) return true; // B
|
|
if ((i & 0xFC000000) == 0x94000000) return true; // BL
|
|
if ((i & 0xFFFFFC1F) == 0xD61F0000) return true; // BR
|
|
if ((i & 0xFFFFFC1F) == 0xD63F0000) return true; // BLR
|
|
if ((i & 0xFFFFFC1F) == 0xD65F0000) return true; // RET
|
|
if ((i & 0x7F000000) == 0x34000000) return true; // CBZ
|
|
if ((i & 0x7F000000) == 0x35000000) return true; // CBNZ
|
|
if ((i & 0x7F000000) == 0x36000000) return true; // TBZ
|
|
if ((i & 0x7F000000) == 0x37000000) return true; // TBNZ
|
|
if ((i & 0xFF000010) == 0x54000000) return true; // B.cond
|
|
if ((i & 0xFFE0001F) == 0xD4000001) return true; // SVC
|
|
if ((i & 0xFFE0001F) == 0xD4200000) return true; // BRK
|
|
if ((i & 0xFFFFF0FF) == 0xD50330DF) return true; // ISB: new fetch boundary
|
|
if ((i & 0xFFFFFFE0) == 0xD50B7520) return true; // IC IVAU host event
|
|
return false;
|
|
}
|
|
|
|
// Statically known target of a direct branch, if the instruction has one.
|
|
//
|
|
// This has to cover every form whose target the translator bakes in, not just
|
|
// the unconditional imm26 pair: a conditional branch's target is equally
|
|
// static, and it is overwhelmingly the common case for loop back-edges. Leaving
|
|
// B.cond/CBZ/CBNZ/TBZ/TBNZ out here meant their targets never began a block, so
|
|
// the generated code would set pc to an address that recomp_lookup - which
|
|
// matches block starts exactly - could not resolve, and execution died with
|
|
// "No recompiled block at PC" at the top of the first loop it entered.
|
|
inline bool DirectBranchTarget(u32 i, u64 pc, u64& out) {
|
|
if ((i & 0xFC000000) == 0x14000000 || (i & 0xFC000000) == 0x94000000) { // B / BL
|
|
s32 imm26 = (s32)(i << 6) >> 6;
|
|
out = pc + (s64)imm26 * 4;
|
|
return true;
|
|
}
|
|
if ((i & 0xFF000010) == 0x54000000 || // B.cond
|
|
(i & 0x7E000000) == 0x34000000) { // CBZ / CBNZ
|
|
s64 imm19 = ((s32)(((i >> 5) & 0x7FFFF) << 13) >> 13);
|
|
out = pc + imm19 * 4;
|
|
return true;
|
|
}
|
|
if ((i & 0x7E000000) == 0x36000000) { // TBZ / TBNZ
|
|
s64 imm14 = ((s32)(((i >> 5) & 0x3FFF) << 18) >> 18);
|
|
out = pc + imm14 * 4;
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
// Function addresses the code builds for itself with ADRP+ADD.
|
|
//
|
|
// A callback handed to the OS - a thread entry above all - is never branched to
|
|
// from inside .text and never appears in a relocation either: the compiler just
|
|
// materialises its address into a register (adrp xN, page; add xN, xN, #lo12)
|
|
// and passes it to svcCreateThread. Without seeding those, every thread the
|
|
// game starts begins at an address no block covers and drops straight to the
|
|
// interpreter for its whole life, which is most of the "No recompiled block at
|
|
// PC" traffic in a real run.
|
|
//
|
|
// Deliberately loose: any ADRP+ADD landing in .text becomes a root. A pair that
|
|
// was really computing a data address only costs one extra block boundary.
|
|
inline void CollectAdrpAddTargets(const u8* text, size_t n_bytes, u64 base,
|
|
std::vector<u64>& out) {
|
|
const u32 n = static_cast<u32>(n_bytes / 4);
|
|
const u32* p = reinterpret_cast<const u32*>(text);
|
|
// Last ADRP seen per destination register, as a page address; ~0 means the
|
|
// register no longer holds one.
|
|
std::array<u64, 32> page{};
|
|
page.fill(~0ULL);
|
|
for (u32 i = 0; i < n; ++i) {
|
|
const u32 insn = p[i];
|
|
if ((insn & 0x9F000000) == 0x90000000) { // ADRP
|
|
const u32 rd = insn & 31;
|
|
s64 immhi = static_cast<s32>((((insn >> 5) & 0x7FFFF) << 13)) >> 13;
|
|
const u32 immlo = (insn >> 29) & 3;
|
|
page[rd] = ((base + static_cast<u64>(i) * 4) & ~0xFFFULL) +
|
|
static_cast<u64>(((immhi << 2) | immlo) << 12);
|
|
} else if ((insn & 0xFFC00000) == 0x91000000) { // ADD (immediate, 64-bit, LSL #0)
|
|
const u32 rd = insn & 31, rn = (insn >> 5) & 31;
|
|
const u32 imm12 = (insn >> 10) & 0xFFF;
|
|
if (page[rn] != ~0ULL) {
|
|
const u64 target = page[rn] + imm12;
|
|
if (target >= base && (target - base) / 4 < n && (target & 3) == 0) {
|
|
out.push_back(target);
|
|
}
|
|
}
|
|
if (rd != rn) {
|
|
page[rd] = ~0ULL;
|
|
}
|
|
} else {
|
|
// Any other write to a register invalidates the page it held. Only
|
|
// the common destination encodings are decoded here; missing one
|
|
// can add a stale root, never remove a real one.
|
|
const u32 rd = insn & 31;
|
|
page[rd] = ~0ULL;
|
|
}
|
|
}
|
|
}
|
|
|
|
inline std::vector<Block> DiscoverBlocks(const u8* text, size_t n_bytes, u64 base,
|
|
u64 entry_pc = 0,
|
|
const std::vector<u64>* extra_roots = nullptr) {
|
|
const u32 n = (u32)(n_bytes / 4);
|
|
if (n == 0) return {};
|
|
std::vector<bool> start(n, false);
|
|
start[0] = true;
|
|
// The module entry has to begin a block in its own right. Nothing branches
|
|
// to it from inside the image, and for a real NSO it isn't offset 0
|
|
// either - .text opens with a MOD0 header - so without this the entry
|
|
// address falls in the middle of some other block and lookup fails.
|
|
if (entry_pc > base && (entry_pc - base) / 4 < n) {
|
|
start[(u32)((entry_pc - base) / 4)] = true;
|
|
}
|
|
// Exported dynsym addresses (functions like nn::init::Start, only ever
|
|
// reached indirectly via another module's resolved GOT/PLT entry, never
|
|
// by a direct branch inside this module's own .text). Without seeding
|
|
// these as roots too, a function preceded by alignment padding rather
|
|
// than a terminator falls mid-block and the runtime dispatcher can never
|
|
// resolve a call landing exactly on its real entry address.
|
|
if (extra_roots) {
|
|
for (u64 addr : *extra_roots) {
|
|
if (addr >= base && (addr - base) / 4 < n) {
|
|
start[(u32)((addr - base) / 4)] = true;
|
|
}
|
|
}
|
|
}
|
|
{
|
|
std::vector<u64> computed;
|
|
CollectAdrpAddTargets(text, n_bytes, base, computed);
|
|
for (u64 addr : computed) {
|
|
start[(u32)((addr - base) / 4)] = true;
|
|
}
|
|
}
|
|
// Preserve roots independently of the mechanical boundary after a terminator.
|
|
// Only the latter can be omitted when an unconditional transfer precedes zeros.
|
|
std::vector<bool> required = start;
|
|
const u32* p = reinterpret_cast<const u32*>(text);
|
|
for (u32 i = 0; i < n; ++i) {
|
|
const u32 insn = p[i];
|
|
const u64 pc = base + (u64)i * 4;
|
|
if (IsTerminator(insn)) {
|
|
if (i + 1 < n) start[i + 1] = true;
|
|
u64 t = 0;
|
|
if (DirectBranchTarget(insn, pc, t) && t >= base && (t - base) / 4 < n) {
|
|
const u32 target = (u32)((t - base) / 4);
|
|
start[target] = required[target] = true;
|
|
}
|
|
}
|
|
}
|
|
std::vector<Block> blocks;
|
|
for (u32 s = 0; s < n;) {
|
|
u32 end = s + 1;
|
|
if (p[s] != 0) {
|
|
while (end < n && !start[end] && p[end] != 0) ++end;
|
|
}
|
|
// A zero immediately after B/BR/RET has no architectural fallthrough.
|
|
// Omit only that mechanical boundary, retaining every independently
|
|
// rooted address. Calls, conditional branches and SVC can return/fall
|
|
// through, so their first zero remains an undefined-instruction trap.
|
|
const u32 previous = s > 0 ? p[s - 1] : 0;
|
|
const bool no_fallthrough = (previous & 0xFC000000) == 0x14000000 || // B, not BL
|
|
(previous & 0xFFFFFC1F) == 0xD61F0000 || // BR
|
|
(previous & 0xFFFFFC1F) == 0xD65F0000; // RET
|
|
if (p[s] != 0 || required[s] || !no_fallthrough) {
|
|
blocks.push_back(Block{base + (u64)s * 4, (end - s) * 4, end - s, s == 0});
|
|
}
|
|
// Further unrooted zeros cannot execute past the retained trap (or
|
|
// preceding transfer). Resume conservatively at the next nonzero word.
|
|
if (p[s] == 0) {
|
|
while (end < n && p[end] == 0 && !start[end]) ++end;
|
|
}
|
|
s = end;
|
|
}
|
|
return blocks;
|
|
}
|
|
|
|
// ARM's logical-immediate encoding: N:immr:imms describe a repeating bit
|
|
// pattern rather than a literal value. This is the standard DecodeBitMasks
|
|
// from the architecture reference, restricted to the immediate (non-tested)
|
|
// result. Returns false for the reserved encodings.
|
|
inline bool DecodeBitMasks(u32 N, u32 imms, u32 immr, bool is64, u64& out) {
|
|
if (!is64 && N) return false;
|
|
// len = index of the highest set bit of (N:~imms), computed portably.
|
|
const u32 bits = (N << 6) | ((~imms) & 0x3F);
|
|
if (bits == 0) return false;
|
|
u32 len = 0;
|
|
for (u32 t = bits; t > 1; t >>= 1) ++len;
|
|
if (len < 1) return false;
|
|
const u32 esize = 1u << len;
|
|
if (esize > (is64 ? 64u : 32u)) return false;
|
|
const u32 levels = esize - 1;
|
|
const u32 s = imms & levels;
|
|
const u32 r = immr & levels;
|
|
if (s == levels) return false; // reserved
|
|
u64 welem = (s + 1 >= 64) ? ~0ULL : ((1ULL << (s + 1)) - 1);
|
|
// Rotate right within the element, then replicate to the register width.
|
|
// The rotate applies at every element size, 64 included: skipping it there
|
|
// (to dodge the undefined `welem << 64` when r is 0) silently turned every
|
|
// 64-bit rotated mask into its unrotated form - so "and x8, x8, #~0xF",
|
|
// the standard align-down, decoded as `& 0x0FFFFFFFFFFFFFFF` and left the
|
|
// pointer unaligned instead. Guard r == 0 explicitly instead.
|
|
u64 elem;
|
|
if (r == 0) {
|
|
elem = welem;
|
|
} else if (esize >= 64) {
|
|
elem = (welem >> r) | (welem << (64 - r));
|
|
} else {
|
|
elem = ((welem >> r) | (welem << (esize - r))) & ((1ULL << esize) - 1);
|
|
}
|
|
u64 result = 0;
|
|
for (u32 i = 0; i < (is64 ? 64u : 32u); i += esize) result |= elem << i;
|
|
if (!is64) result &= 0xFFFFFFFFULL;
|
|
out = result;
|
|
return true;
|
|
}
|
|
|
|
inline std::string FuncName(const std::string& mod, u64 v) {
|
|
char b[64]; snprintf(b, sizeof b, "blk_%s_%016llx", mod.c_str(), (unsigned long long)v); return b;
|
|
}
|
|
|
|
// Same name, written into a caller-owned buffer. The emit loops call this once
|
|
// per block for the body and twice more per block for the dispatch table, so on
|
|
// a multi-million-block title the returned-std::string form above is millions of
|
|
// heap allocations for a name that is always well under 64 bytes.
|
|
inline size_t FuncNameTo(char (&b)[64], const char* mod, u64 v) {
|
|
return (size_t)snprintf(b, sizeof b, "blk_%s_%016llx", mod, (unsigned long long)v);
|
|
}
|
|
|
|
// Set by the emit loop so a direct branch whose target is a known block start
|
|
// can call that block instead of returning to the dispatcher. Null while the
|
|
// decode tests run, which is what keeps their expected output stable.
|
|
// Emits "carry on at t": a direct call when t is a block we emitted, otherwise
|
|
// the original park-the-PC-and-return. Loop back edges are conditional, so the
|
|
// conditional forms need this as much as the unconditional ones do.
|
|
inline std::string ChainTo(u64 t);
|
|
|
|
// With no JIT behind it there is no cheaper alternative to translating an
|
|
// instruction, so the forms that lose to the JIT are turned on here.
|
|
inline bool g_translate_all = false;
|
|
|
|
// The per-module standalone runner: main.c plus the module's text, rodata and
|
|
// data segments bundled beside it, so the translated code can run on its own
|
|
// with no emulator, no game file and no keys. The recompiler's own tests use it
|
|
// with synthetic code. The game exporter turns it off: a game's translated code
|
|
// may only run inside suyu, loaded against the user's own game file and keys.
|
|
inline bool g_emit_standalone_runner = true;
|
|
|
|
// ABI 6 "FM1": memory helpers that read the host page table directly and fall
|
|
// back to the unchanged ABI 5 helpers for anything else. Off by default; while
|
|
// it is off the emitted text is byte-identical to ABI 5, which
|
|
// tests/recompiler_smoke checks against a golden hash.
|
|
inline bool g_emit_fastmem = false;
|
|
|
|
// ABI 6 feature GG1: blocks re-run the code guard only when their module's
|
|
// generation word has moved since they last passed it, instead of on every
|
|
// entry. An extension of ABI 6, so it takes effect only together with
|
|
// g_emit_fastmem. Off by default; while it is off the ABI 5 and ABI 6 texts are
|
|
// byte-identical to what they were before it existed (golden hashes in
|
|
// tests/recompiler_smoke).
|
|
inline bool g_emit_guard_gen = false;
|
|
inline bool EmitGuardGen() {
|
|
return g_emit_fastmem && g_emit_guard_gen;
|
|
}
|
|
|
|
// ABI 6 "FPX1": exact native floating point. Each FP op the option covers gets a
|
|
// native fast path whose result is kept only when it provably equals the
|
|
// architectural result and adds no FPSR bit the guest has not already set;
|
|
// anything else runs the unchanged exact body. Off by default, and while it is
|
|
// off the emitted text is byte-identical to what it would be without it.
|
|
inline bool g_emit_fpx = false;
|
|
// Instrumentation only, never timed: with g_emit_fpx, the exact body runs
|
|
// after every fast path too, and recomp_fpx_shadow compares the two.
|
|
// It compares the inline form only: the outlined wrappers' rescues past the
|
|
// fast path (a64_lift.py's a64_fpx.h) never run in it.
|
|
inline bool g_emit_fpx_shadow = false;
|
|
|
|
// Hybrid exports translate the gated forms too. g_translate_all gates only SHL
|
|
// by immediate (vector and scalar D), whose translation is exact
|
|
// (tests/recompiler_smoke shl_unit). Hybrid left it to the JIT on the strength
|
|
// of one measurement from before ABI 6, chaining and the clang-cl build
|
|
// (2.13 ms/frame against 1.95 for one SIMD-heavy block). Measured 2026-10-06,
|
|
// TOTK 1.4.3 Hybrid, in-world, alternating, quiet machine: the fallback made
|
|
// ~290,000 static->JIT transitions per run, all SHL vec imm, at 22.5/23.1 fps;
|
|
// translating it made 0, at 29.5/28.9 (JIT and strict static are ~30). MK8D
|
|
// Hybrid race: 53.6 fps mean over 3 runs against 54.2 over 5, within noise,
|
|
// with 0 transitions against ~55,000. Set this false to restore the old split.
|
|
inline constexpr bool kHybridTranslatesGatedForms = true;
|
|
|
|
// Strict static exports have no fallback to carry deliberately gated forms.
|
|
inline bool TranslateAllForExport(bool strict_static, bool explicitly_requested) {
|
|
return strict_static || explicitly_requested || kHybridTranslatesGatedForms;
|
|
}
|
|
|
|
inline const std::unordered_set<u64>* g_chain_blocks = nullptr;
|
|
inline const char* g_chain_mod = nullptr;
|
|
|
|
// How many blocks may run before control goes back to the dispatcher. The
|
|
// dispatcher checks for interrupts and services SVCs between blocks, so an
|
|
// unbounded chain would let a guest loop run uninterruptibly; it also derives
|
|
// the executed-block count from how much of this budget was spent.
|
|
inline constexpr int kChainBudget = 256;
|
|
|
|
// Conditional branches deliberately do not chain. Measured: chaining them too
|
|
// takes 2.00 ms/frame against 1.80, and turns a tight distribution into one
|
|
// spanning 1.70-2.54 across six reps while the dynarmic arm stays at 2.905 in
|
|
// the same runs. It also pushed JIT transitions from 69,008 to 88,983 and more
|
|
// than doubled lookup misses, which is unexplained and worth understanding
|
|
// before trying this again.
|
|
inline std::string ChainTo(u64 t) {
|
|
char b[256];
|
|
if (g_chain_blocks && g_chain_mod && g_chain_blocks->count(t)) {
|
|
char nm[64];
|
|
FuncNameTo(nm, g_chain_mod, t);
|
|
// The declaration is at block scope so a unit needs no list of the
|
|
// blocks it reaches. `return f(c)` is written as a tail call, though
|
|
// nothing may come of that below -O2 - which is why the budget has to
|
|
// bound the depth rather than assume it stays at one frame.
|
|
snprintf(b, sizeof b,
|
|
"{ void %s(GuestContext*); c->pc=g_module_base+0x%llxULL; "
|
|
"if (--c->chain_budget <= 0) return; %s(c); return; }",
|
|
nm, (unsigned long long)t, nm);
|
|
} else {
|
|
snprintf(b, sizeof b, "{ c->pc=g_module_base+0x%llxULL; return; }",
|
|
(unsigned long long)t);
|
|
}
|
|
return b;
|
|
}
|
|
|
|
// The same for a computed target. A chain of blocks otherwise ends at the first
|
|
// BR, BLR or RET, and every one of those is a round trip out to the host
|
|
// dispatcher: RunThread plus the dispatch lambda are 24.7% of all cycles in a
|
|
// JIT-free image against 17.7% in a hybrid one. recomp_lookup is this module's
|
|
// own flat block index - one bounds check and one load - so resolving the
|
|
// target here keeps execution inside the image whenever it stays in the module.
|
|
// A target in another module misses the index and falls back to the dispatcher,
|
|
// which is where it had to go anyway.
|
|
inline std::string ChainIndirect(const std::string& target) {
|
|
return "{ c->pc=" + target +
|
|
"; { BlockFn _f=recomp_lookup(c->pc-g_module_base); "
|
|
"if (_f && --c->chain_budget > 0) { _f(c); return; } } return; }";
|
|
}
|
|
|
|
// The condition is a literal at every site, so the switch inside recomp_cond
|
|
// is resolvable here rather than at run time. It was 2.1% of all cycles as a
|
|
// call - and the expression that replaces it is smaller than the call was,
|
|
// which matters: the memory helpers were measured as *worse* inlined because
|
|
// the instruction cache cost more than the call did.
|
|
// NZCV, with the operation width and add/subtract resolved here. Same argument
|
|
// as Cond: is_sub and is64 are literals at every site, and a call with six
|
|
// arguments costs more to set up than the arithmetic it hides.
|
|
inline std::string SetFlags(bool is_sub, const std::string& a, const std::string& b,
|
|
const std::string& r, bool is64) {
|
|
const std::string ty = is64 ? "uint64_t" : "uint32_t";
|
|
const std::string cast = is64 ? "" : "(uint32_t)";
|
|
const char* top = is64 ? "63" : "31";
|
|
std::string s = "{ " + ty + " _fa=" + cast + "(" + a + "),_fb=" + cast + "(" + b +
|
|
"),_fr=" + cast + "(" + r + "); ";
|
|
s += "c->z=(_fr==0); c->n=(int)((_fr>>" + std::string(top) + ")&1); ";
|
|
if (is_sub) {
|
|
s += "c->c=(_fa>=_fb); ";
|
|
s += "c->v=(int)((((_fa^_fb)&(_fa^_fr))>>" + std::string(top) + ")&1); }";
|
|
} else {
|
|
s += "c->c=(_fr<_fa); ";
|
|
s += "c->v=(int)((((" + ty + ")~(_fa^_fb)&(_fa^_fr))>>" + std::string(top) + ")&1); }";
|
|
}
|
|
return s;
|
|
}
|
|
|
|
inline std::string Cond(u32 cond) {
|
|
switch (cond) {
|
|
case 0: return "(c->z)";
|
|
case 1: return "(!c->z)";
|
|
case 2: return "(c->c)";
|
|
case 3: return "(!c->c)";
|
|
case 4: return "(c->n)";
|
|
case 5: return "(!c->n)";
|
|
case 6: return "(c->v)";
|
|
case 7: return "(!c->v)";
|
|
case 8: return "(c->c && !c->z)";
|
|
case 9: return "(!(c->c && !c->z))";
|
|
case 10: return "(c->n == c->v)";
|
|
case 11: return "(c->n != c->v)";
|
|
case 12: return "((c->n == c->v) && !c->z)";
|
|
case 13: return "(!((c->n == c->v) && !c->z))";
|
|
default: return "(1)"; // AL and the NV encoding, which also always passes
|
|
}
|
|
}
|
|
|
|
inline std::string Xz(u32 r) {
|
|
return r == 31 ? std::string("(uint64_t)0") : ("c->x[" + std::to_string(r) + "]");
|
|
}
|
|
inline std::string Wz(u32 r) {
|
|
return r == 31 ? std::string("(uint32_t)0") : ("(uint32_t)c->x[" + std::to_string(r) + "]");
|
|
}
|
|
|
|
/// Register 31 as the stack pointer rather than the zero register.
|
|
///
|
|
/// AArch64 spells both with the same encoding and the meaning depends on the
|
|
/// instruction: ADD/SUB with an immediate or an extended register read it as
|
|
/// SP, while the shifted-register and logical forms read it as XZR. Using the
|
|
/// zero-register spelling everywhere makes every function prologue -
|
|
/// "sub sp, sp, #N", "add x29, sp, #N" - compute from zero, so the frame lands
|
|
/// at a tiny address and every local access writes into unmapped memory near
|
|
/// null.
|
|
inline std::string Xsp(u32 r) {
|
|
return "c->x[" + std::to_string(r) + "]";
|
|
}
|
|
|
|
// Append C for one instruction. Returns false if the instruction terminates the block.
|
|
/// Translate one AArch64 instruction to C.
|
|
///
|
|
/// Returns true if the block stays open. `unhandled`, when supplied, reports
|
|
/// whether this instruction fell through the decoder to recomp_unhandled -
|
|
/// which the return value cannot express, because an unhandled instruction is
|
|
/// stepped over and leaves the block open exactly like a translated one. Without
|
|
/// this signal the emitter cannot tell a full translation from a stream of
|
|
/// fallbacks, which is how a whole module of misdecoded ARM32 once passed for a
|
|
/// successful export.
|
|
// Emit one baseline S/D addition/subtraction from raw uint64_t _a/_b to _v.
|
|
// Three guard bits plus sticky alignment preserve exact rounding, including
|
|
// cancellation. No host FP arithmetic or host FP state is involved.
|
|
// The integer value emitter is the reference on every host. With FPX1 each
|
|
// covered op first tries recomp_fpx_<op> (recomp_runtime.h), which computes
|
|
// natively and answers 0 unless the result is provably exact (see the FPX1
|
|
// block of the runtime header); the exact body is the fallback. FPX1 sites call
|
|
// it out of line (EmitFPOutlined). Inline, as the shadow build keeps it, the
|
|
// prefix opens a do{ ... }while(0) whose break skips the exact body, and
|
|
// EmitFPNativeSuffix closes it. `op` names the helper: add, sub, mul, div,
|
|
// fma, recps, rsqrts (raw inputs _aa/_bb) or sqrt (input _a). Every helper
|
|
// also takes the guest's FPCR.FZ bit (RECOMP_FPX_FZ), which the gate leaves
|
|
// open: under FZ it keeps a result only when flushing cannot change it.
|
|
inline std::string FpxArgs(const char* op) {
|
|
if (!std::strcmp(op, "fma")) return "_a,_b,_z";
|
|
if (!std::strcmp(op, "recps") || !std::strcmp(op, "rsqrts")) return "_aa,_bb";
|
|
if (!std::strcmp(op, "sqrt")) return "_a";
|
|
return "_a,_b";
|
|
}
|
|
inline std::string EmitFPNativeValue(bool dbl, const char* op) {
|
|
if (!g_emit_fpx) return {};
|
|
const std::string call =
|
|
std::string("recomp_fpx_") + op + (dbl ? "64(" : "32(") + FpxArgs(op) + ",RECOMP_FPX_FZ(c)";
|
|
if (g_emit_fpx_shadow) {
|
|
return "do{uint64_t _fxv=0,_fxs=c->fpsr;int _fxg=RECOMP_FPX_OPEN(c),_fxk=_fxg&&" + call +
|
|
",&_fxv);";
|
|
}
|
|
return "do{if(RECOMP_FPX_OPEN(c)&&RECOMP_LIKELY(" + call +
|
|
",&_v))){RECOMP_FPX_PROBE(1);break;}RECOMP_FPX_PROBE(2);";
|
|
}
|
|
// FPX1 sites call the exact body out of line: recomp_fpe_<op>{32,64}, defined
|
|
// once per module in the runtime C (FpxExactC). Inlining it per site made the
|
|
// exact bodies about 55% of a large title's code (TotK main: 672k sites at
|
|
// ~1.4 KB each, past the 2 GB image limit), and that code only runs when the
|
|
// gate is closed or the native result is not provably exact. The fast path
|
|
// stays inline and unchanged. The shadow build keeps the bodies inline.
|
|
inline bool FpxOutline() {
|
|
return g_emit_fpx && !g_emit_fpx_shadow;
|
|
}
|
|
// The outlined site: `_v` and the operands are declared by the caller. FMUL's
|
|
// exact body is the multiply-add with the caller's sign-matched zero addend.
|
|
inline std::string EmitFPOutlined(bool dbl, const char* op) {
|
|
const char* exact = std::strcmp(op, "mul") ? op : "fma";
|
|
const char* width = dbl ? "64(" : "32(";
|
|
return std::string("{if(RECOMP_FPX_OPEN(c)&&RECOMP_LIKELY(recomp_fpx_") + op + width +
|
|
FpxArgs(op) + ",RECOMP_FPX_FZ(c),&_v))){RECOMP_FPX_PROBE(1);}else{RECOMP_FPX_PROBE(2);_v=recomp_fpe_" + exact +
|
|
width + "c," + FpxArgs(exact) + ");}}";
|
|
}
|
|
inline std::string EmitFPNativeSuffix(bool dbl, const char* op) {
|
|
if (!g_emit_fpx) return {};
|
|
if (g_emit_fpx_shadow) {
|
|
static const char* const kinds[] = {"add", "sub", "mul", "div", "fma", "recps", "rsqrts", "sqrt"};
|
|
unsigned kind = 0;
|
|
while (kind < 8 && std::strcmp(kinds[kind], op)) ++kind;
|
|
const std::string args = FpxArgs(op);
|
|
const std::string first = args.substr(0, args.find(','));
|
|
const std::string second = args.find(',') == std::string::npos
|
|
? std::string("0") : args.substr(args.find(',') + 1, args.find(',', args.find(',') + 1) -
|
|
args.find(',') - 1);
|
|
return "recomp_fpx_shadow(" + std::to_string(kind * 2 + (dbl ? 1 : 0)) +
|
|
"u,_fxg,_fxk,_fxv,_v,_fxs,c->fpsr," + first + "," + second + ");}while(0);";
|
|
}
|
|
return "}while(0);";
|
|
}
|
|
|
|
// `exact_only`: the bare exact body, for the out-of-line helpers.
|
|
inline std::string EmitFPAddSubValue(bool dbl, bool subtract, bool exact_only = false) {
|
|
if (!exact_only && FpxOutline()) return EmitFPOutlined(dbl, subtract ? "sub" : "add");
|
|
return (exact_only ? std::string() : EmitFPNativeValue(dbl, subtract ? "sub" : "add")) +
|
|
"{ const unsigned _f=" + std::string(dbl?"52":"23") +
|
|
";const uint64_t _hidden=1ULL<<_f,_frac=_hidden-1,_quiet=_hidden>>1,_exp="+
|
|
(dbl?"0x7ff0000000000000ULL":"0x7f800000ULL")+",_sign="+
|
|
(dbl?"0x8000000000000000ULL":"0x80000000ULL")+";"
|
|
"unsigned _mode=(unsigned)(c->fpcr>>22)&3;"
|
|
"if(c->fpcr&(1ULL<<24)){if(!(_a&_exp)&&(_a&_frac)){_a&=_sign;c->fpsr|=128;}"
|
|
"if(!(_b&_exp)&&(_b&_frac)){_b&=_sign;c->fpsr|=128;}}"
|
|
"int _an=(_a&_exp)==_exp&&(_a&_frac),_bn=(_b&_exp)==_exp&&(_b&_frac);"
|
|
"unsigned _sa=!!(_a&_sign),_sb="+std::string(subtract?"!(_b&_sign)":"!!(_b&_sign)")+";"
|
|
"if(_an||_bn){int _as=_an&&!(_a&_quiet),_bs=_bn&&!(_b&_quiet);"
|
|
"_v=(_as?_a:_bs?_b:_an?_a:_b)|_quiet;if(_as||_bs){c->fpsr|=1;}"
|
|
"if(c->fpcr&(1ULL<<25)){_v=_exp|_quiet;}}"
|
|
"else if((_a&_exp)==_exp||(_b&_exp)==_exp){"
|
|
"if((_a&_exp)==_exp&&(_b&_exp)==_exp&&_sa!=_sb){_v=_exp|_quiet;c->fpsr|=1;}"
|
|
"else _v=_exp|(((_a&_exp)==_exp?_sa:_sb)?_sign:0);}"
|
|
"else {unsigned _ea=(unsigned)((_a&_exp)>>_f),_eb=(unsigned)((_b&_exp)>>_f);"
|
|
"uint64_t _ma=(_a&_frac)|(_ea?_hidden:0),_mb=(_b&_frac)|(_eb?_hidden:0);"
|
|
"if(!_ea){_ea=1;}if(!_eb){_eb=1;}"
|
|
"if(_ea<_eb||(_ea==_eb&&_ma<_mb)){unsigned _t=_ea;_ea=_eb;_eb=_t;_t=_sa;_sa=_sb;_sb=_t;"
|
|
"uint64_t _u=_ma;_ma=_mb;_mb=_u;}"
|
|
"_ma<<=3;_mb<<=3;unsigned _distance=_ea-_eb;"
|
|
"if(_distance>=64)_mb=!!_mb;else if(_distance)_mb=(_mb>>_distance)|!!(_mb&((1ULL<<_distance)-1));"
|
|
"uint64_t _magnitude=_sa==_sb?_ma+_mb:_ma-_mb;"
|
|
"if(!_magnitude)_v=(_sa==_sb?_sa:_mode==2)?_sign:0;"
|
|
"else {if(_magnitude>=(_hidden<<4)){_magnitude=(_magnitude>>1)|(_magnitude&1);++_ea;}"
|
|
"while(_magnitude<(_hidden<<3)&&_ea>1){_magnitude<<=1;--_ea;}"
|
|
"int _tiny=_ea==1&&_magnitude<(_hidden<<3);"
|
|
"if(_tiny&&(c->fpcr&(1ULL<<24))){c->fpsr|=8;_v=0;}"
|
|
"else {unsigned _lost=(unsigned)(_magnitude&7);uint64_t _rounded=_magnitude>>3;"
|
|
"if(_lost){c->fpsr|=16;if(_tiny)c->fpsr|=8;"
|
|
"if((_mode==0&&(_lost>4||(_lost==4&&(_rounded&1))))||(_mode==1&&!_sa)||(_mode==2&&_sa))++_rounded;}"
|
|
"if(_rounded>=(_hidden<<1)){_rounded>>=1;++_ea;}"
|
|
"if(_ea>=(unsigned)(_exp>>_f)){c->fpsr|=20;_v=(_mode==0||(_mode==1&&!_sa)||(_mode==2&&_sa))?_exp:_exp-1;}"
|
|
"else _v=((_rounded>=_hidden?(uint64_t)_ea:0)<<_f)|(_rounded&_frac);}"
|
|
"if(_sa)_v|=_sign;}}}" + (exact_only ? std::string() : EmitFPNativeSuffix(dbl, subtract ? "sub" : "add"));
|
|
}
|
|
|
|
// Generated exact product/addend lattice. Inputs _a,_b,_z and output _v are
|
|
// IEEE bit patterns. A multiply supplies a same-sign zero addend so its zero
|
|
// result has the product sign in every rounding mode.
|
|
inline std::string EmitFPDivideValue(bool dbl, bool exact_only = false) {
|
|
if (!exact_only && FpxOutline()) return EmitFPOutlined(dbl, "div");
|
|
return (exact_only ? std::string() : EmitFPNativeValue(dbl, "div")) +
|
|
"{ const unsigned _f="+std::string(dbl?"52":"23")+",_bias="+(dbl?"1023":"127")+R"C(;
|
|
const uint64_t _hidden=1ULL<<_f,_frac=_hidden-1,_quiet=_hidden>>1;
|
|
const uint64_t _exp=((uint64_t)(2*_bias+1))<<_f,_sign=1ULL<<(_f+(_f==52?11:8));
|
|
const unsigned _mode=(unsigned)(c->fpcr>>22)&3;unsigned _negative=!!((_a^_b)&_sign);
|
|
if(c->fpcr&(1ULL<<24)) {
|
|
if(!(_a&_exp)&&(_a&_frac)){_a&=_sign;c->fpsr|=128;}
|
|
if(!(_b&_exp)&&(_b&_frac)){_b&=_sign;c->fpsr|=128;}
|
|
}
|
|
int _an=(_a&_exp)==_exp&&(_a&_frac),_bn=(_b&_exp)==_exp&&(_b&_frac);
|
|
int _as=_an&&!(_a&_quiet),_bs=_bn&&!(_b&_quiet);
|
|
int _ai=(_a&~_sign)==_exp,_bi=(_b&~_sign)==_exp,_az=!(_a&~_sign),_bz=!(_b&~_sign);
|
|
if(_an||_bn){_v=(_as?_a:_bs?_b:_an?_a:_b)|_quiet;if(_as||_bs)c->fpsr|=1;
|
|
if(c->fpcr&(1ULL<<25))_v=_exp|_quiet;
|
|
}else if((_ai&&_bi)||(_az&&_bz)){_v=_exp|_quiet;c->fpsr|=1;}
|
|
else if(_ai||_bz){_v=_exp|(_negative?_sign:0);if(!_ai)c->fpsr|=2;}
|
|
else if(_az||_bi)_v=_negative?_sign:0;
|
|
else {
|
|
unsigned _ea=(unsigned)((_a&_exp)>>_f),_eb=(unsigned)((_b&_exp)>>_f);
|
|
uint64_t _ma=(_a&_frac)|(_ea?_hidden:0),_mb=(_b&_frac)|(_eb?_hidden:0);
|
|
int _ae=_ea?(int)_ea-(int)_bias:1-(int)_bias,_be=_eb?(int)_eb-(int)_bias:1-(int)_bias;
|
|
while(_ma<_hidden){_ma<<=1;--_ae;}while(_mb<_hidden){_mb<<=1;--_be;}
|
|
int _e=_ae-_be,_emin=1-(int)_bias;if(_ma<_mb){_ma<<=1;--_e;}
|
|
if(_e<_emin&&(c->fpcr&(1ULL<<24))){_v=_negative?_sign:0;c->fpsr|=8;}
|
|
else {
|
|
int _bits=(int)_f+(_e<_emin?_e-_emin:0);uint64_t _mant=0,_rem=_ma-_mb;
|
|
unsigned _roundbit=0,_sticky=0;
|
|
if(_bits>=0){_mant=1;for(int _j=0;_j<_bits;++_j){_mant<<=1;_rem<<=1;if(_rem>=_mb){++_mant;_rem-=_mb;}}
|
|
_rem<<=1;_roundbit=_rem>=_mb;if(_roundbit)_rem-=_mb;_sticky=_rem!=0;
|
|
}else{_roundbit=_bits==-1;_sticky=_bits<-1||_rem!=0;}
|
|
unsigned _lost=_roundbit|_sticky;if(_lost){c->fpsr|=16;if(_e<_emin)c->fpsr|=8;
|
|
if((_mode==0&&_roundbit&&(_sticky||(_mant&1)))||(_mode==1&&!_negative)||(_mode==2&&_negative))++_mant;}
|
|
if(_e<_emin){_e=_emin;}if(_mant>=(_hidden<<1)){_mant>>=1;++_e;}
|
|
if(_e>(int)_bias){c->fpsr|=20;_v=(_mode==0||(_mode==1&&!_negative)||(_mode==2&&_negative))?_exp:_exp-1;}
|
|
else _v=(_mant>=_hidden?(uint64_t)(_e+(int)_bias)<<_f:0)|(_mant&_frac);
|
|
if(_negative)_v|=_sign;
|
|
}
|
|
}
|
|
}
|
|
)C" + (exact_only ? std::string() : EmitFPNativeSuffix(dbl, "div"));
|
|
}
|
|
|
|
// Architectural reciprocal estimate. The estimate is deliberately computed
|
|
// with the integer table algorithm from the ARM pseudocode.
|
|
inline std::string EmitFPRecipEstimateValue(bool dbl) {
|
|
std::string s = "{ const unsigned _f=" + std::string(dbl ? "52" : "23") +
|
|
",_bias=" + (dbl ? "1023" : "127") + ";";
|
|
s += R"C(
|
|
const int _emin=1-(int)_bias;const uint64_t _hidden=1ULL<<_f,_frac=_hidden-1,_quiet=_hidden>>1;
|
|
const uint64_t _exp=((uint64_t)(2*_bias+1))<<_f,_sign=1ULL<<(_f+(_f==52?11:8));
|
|
unsigned _eraw=(unsigned)((_x&_exp)>>_f);uint64_t _fb=_x&_frac;unsigned _negative=!!(_x&_sign);
|
|
if(_eraw==2*_bias+1){
|
|
if(_fb){if(!(_x&_quiet))c->fpsr|=1;_v=_x|_quiet;if(c->fpcr&(1ULL<<25))_v=_exp|_quiet;}
|
|
else _v=_negative?_sign:0;
|
|
}else if(!_eraw&&!_fb){c->fpsr|=2;_v=(_x&_sign)|_exp;}
|
|
else if(!_eraw&&_fb&&(c->fpcr&(1ULL<<24))){c->fpsr|=130;_v=(_x&_sign)|_exp;}
|
|
else {
|
|
uint64_t _mant;int _e;
|
|
if(_eraw){_mant=_hidden|_fb;_e=(int)_eraw-(int)_bias;}
|
|
else{_mant=_fb;_e=_emin;while(!(_mant&_hidden)){_mant<<=1;--_e;}}
|
|
if(_e<_emin-2){
|
|
unsigned _mode=(unsigned)(c->fpcr>>22)&3;
|
|
unsigned _to_inf=_mode==0||(_mode==1&&!_negative)||(_mode==2&&_negative);
|
|
c->fpsr|=20;_v=(_x&_sign)|(_to_inf?_exp:_exp-1);
|
|
}else if((c->fpcr&(1ULL<<24))&&_e>=-_emin){c->fpsr|=8;_v=_x&_sign;}
|
|
else {
|
|
unsigned _index=(unsigned)(_mant>>(_f-8));uint64_t _q=_index*2+1;
|
|
uint64_t _estimate=(uint8_t)((((1ULL<<19)/_q)+1)/2);_estimate<<=_f-8;
|
|
int _re=-(_e+1);
|
|
if(_re<_emin){if(_re==_emin-1){_estimate|=_hidden;_estimate>>=1;}
|
|
else{_estimate|=_hidden;_estimate>>=2;++_re;}}
|
|
_v=(_x&_sign)|((uint64_t)(_re+(int)_bias)<<_f)|(_estimate&_frac);
|
|
}
|
|
}
|
|
}
|
|
)C";
|
|
return s;
|
|
}
|
|
|
|
// Architectural reciprocal-square-root estimate. The estimate is deliberately
|
|
// computed with the integer table algorithm from the ARM pseudocode: host
|
|
// rsqrt instructions do not guarantee the same eight result bits.
|
|
inline std::string EmitFPRSqrtEstimateValue(bool dbl) {
|
|
std::string s = "{ const unsigned _f=" + std::string(dbl ? "52" : "23") +
|
|
",_bias=" + (dbl ? "1023" : "127") + ";";
|
|
s += R"C(
|
|
const uint64_t _hidden=1ULL<<_f,_frac=_hidden-1,_quiet=_hidden>>1;
|
|
const uint64_t _exp=((uint64_t)(2*_bias+1))<<_f,_sign=1ULL<<(_f+(_f==52?11:8));
|
|
unsigned _eraw=(unsigned)((_x&_exp)>>_f);uint64_t _fb=_x&_frac;
|
|
if(!_eraw&&!_fb){c->fpsr|=2;_v=(_x&_sign)|_exp;}
|
|
else if(!_eraw&&_fb&&(c->fpcr&(1ULL<<24))){c->fpsr|=130;_v=(_x&_sign)|_exp;}
|
|
else if(_eraw==2*_bias+1){
|
|
if(_fb){if(!(_x&_quiet))c->fpsr|=1;_v=_x|_quiet;if(c->fpcr&(1ULL<<25))_v=_exp|_quiet;}
|
|
else if(_x&_sign){c->fpsr|=1;_v=_exp|_quiet;}else _v=0;
|
|
}else if(_x&_sign){c->fpsr|=1;_v=_exp|_quiet;}
|
|
else {
|
|
uint64_t _mant;int _e;
|
|
if(_eraw){_mant=_hidden|_fb;_e=(int)_eraw-(int)_bias;}
|
|
else{_mant=_fb;_e=1-(int)_bias;while(!(_mant&_hidden)){_mant<<=1;--_e;}}
|
|
unsigned _index=(unsigned)(_mant>>(_f-((_e%2)==0?7:8)));
|
|
uint64_t _q=_index<256?_index*2+1:(_index|1)*2,_b=512;
|
|
while(_q*(_b+1)*(_b+1)<(1ULL<<28))++_b;
|
|
uint64_t _estimate=(uint8_t)((_b+1)/2);int _re=(-(_e+1))>>1;
|
|
_v=((uint64_t)(_re+(int)_bias)<<_f)|((_estimate<<(_f-8))&_frac);
|
|
}
|
|
}
|
|
)C";
|
|
return s;
|
|
}
|
|
|
|
// `mul` marks a multiply (FMUL/FNMUL), whose addend is the sign-matched zero
|
|
// the caller supplies: the fast path then needs no fused multiply-add.
|
|
inline std::string EmitFPMulAddValue(bool dbl, bool mul = false, bool exact_only = false) {
|
|
if (!exact_only && FpxOutline()) return EmitFPOutlined(dbl, mul ? "mul" : "fma");
|
|
std::string s=(exact_only ? std::string() : EmitFPNativeValue(dbl, mul ? "mul" : "fma"))+
|
|
"{ const unsigned _f="+std::string(dbl?"52":"23")+
|
|
",_bias="+(dbl?"1023":"127")+",_count="+(dbl?"67":"9")+
|
|
"; const int _base="+(dbl?"-2148":"-298")+"; uint64_t _p["+
|
|
(dbl?"67":"9")+"]={0},_c["+(dbl?"67":"9")+"]={0};";
|
|
s+=R"C(
|
|
const uint64_t _hidden=1ULL<<_f,_frac=_hidden-1,_quiet=_hidden>>1;
|
|
const uint64_t _exp=((uint64_t)(2*_bias+1))<<_f,_sign=1ULL<<(_f+(_f==52?11:8));
|
|
unsigned _mode=(unsigned)(c->fpcr>>22)&3;
|
|
if(c->fpcr&(1ULL<<24)) {
|
|
if(!(_a&_exp)&&(_a&_frac)){_a&=_sign;c->fpsr|=128;}
|
|
if(!(_b&_exp)&&(_b&_frac)){_b&=_sign;c->fpsr|=128;}
|
|
if(!(_z&_exp)&&(_z&_frac)){_z&=_sign;c->fpsr|=128;}
|
|
}
|
|
int _an=(_a&_exp)==_exp&&(_a&_frac),_bn=(_b&_exp)==_exp&&(_b&_frac),_zn=(_z&_exp)==_exp&&(_z&_frac);
|
|
int _as=_an&&!(_a&_quiet),_bs=_bn&&!(_b&_quiet),_zs=_zn&&!(_z&_quiet);
|
|
int _ai=(_a&~_sign)==_exp,_bi=(_b&~_sign)==_exp,_zi=(_z&~_sign)==_exp;
|
|
int _az=!(_a&~_sign),_bz=!(_b&~_sign),_zz=!(_z&~_sign),_badproduct=(_ai&&_bz)||(_bi&&_az);
|
|
unsigned _ps=!!((_a^_b)&_sign),_cs=!!(_z&_sign);
|
|
if(_an||_bn||_zn) {
|
|
_v=(_zs?_z:_as?_a:_bs?_b:_zn?_z:_an?_a:_b)|_quiet;
|
|
if(_as||_bs||_zs)c->fpsr|=1;
|
|
if((c->fpcr&(1ULL<<25))||(_zn&&!_zs&&_badproduct))_v=_exp|_quiet;
|
|
if(_zn&&!_zs&&_badproduct)c->fpsr|=1;
|
|
}else if(_badproduct||(_zi&&(_ai||_bi)&&_ps!=_cs)){_v=_exp|_quiet;c->fpsr|=1;}
|
|
else if(_zi||_ai||_bi)_v=_exp|((_zi?_cs:_ps)?_sign:0);
|
|
else {
|
|
unsigned _ea=(unsigned)((_a&_exp)>>_f),_eb=(unsigned)((_b&_exp)>>_f),_ez=(unsigned)((_z&_exp)>>_f);
|
|
uint64_t _ma=(_a&_frac)|(_ea?_hidden:0),_mb=(_b&_frac)|(_eb?_hidden:0),_mz=(_z&_frac)|(_ez?_hidden:0);
|
|
uint64_t _a0=(uint32_t)_ma,_a1=_ma>>32,_b0=(uint32_t)_mb,_b1=_mb>>32;
|
|
uint64_t _lo=_a0*_b0,_x=_a0*_b1,_y=_a1*_b0,_hi=_a1*_b1+(_x>>32)+(_y>>32);
|
|
uint64_t _old=_lo;_lo+=_x<<32;_hi+=_lo<_old;_old=_lo;_lo+=_y<<32;_hi+=_lo<_old;
|
|
unsigned _shift=(_ea?_ea-1:0)+(_eb?_eb-1:0),_j=_shift/64,_s=_shift%64;
|
|
_p[_j]=_lo<<_s;_p[_j+1]=(_hi<<_s)|(_s?_lo>>(64-_s):0);
|
|
if(_s&&_j+2<_count)_p[_j+2]=_hi>>(64-_s);
|
|
_shift=(_ez?_ez-1:0)+_bias+_f-1;_j=_shift/64;_s=_shift%64;
|
|
_c[_j]=_mz<<_s;if(_s)_c[_j+1]=_mz>>(64-_s);
|
|
unsigned _negative=_ps;
|
|
if(_ps!=_cs) {
|
|
int _compare=0;for(int _k=(int)_count-1;_k>=0;--_k){if(_p[_k]!=_c[_k]){_compare=_p[_k]>_c[_k]?1:-1;break;}}
|
|
_negative=_compare>=0?_ps:_cs;uint64_t _borrow=0;
|
|
for(unsigned _k=0;_k<_count;++_k){uint64_t _left=_compare>=0?_p[_k]:_c[_k],_right=_compare>=0?_c[_k]:_p[_k];
|
|
uint64_t _diff=_left-_right,_next=(_left<_right)||(_diff<_borrow);_p[_k]=_diff-_borrow;_borrow=_next;}
|
|
}else {uint64_t _carry=0;for(unsigned _k=0;_k<_count;++_k){uint64_t _sum=_p[_k]+_c[_k],_next=_sum<_p[_k];
|
|
uint64_t _total=_sum+_carry;_carry=_next||_total<_sum;_p[_k]=_total;}}
|
|
int _top=recomp_fp_top_bit(_p,_count);
|
|
if(_top<0)_v=((_zz&&(_az||_bz)&&_ps==_cs)?_ps:_mode==2)?_sign:0;
|
|
else {int _e=_top+_base,_emin=1-(int)_bias;
|
|
if(_e<_emin&&(c->fpcr&(1ULL<<24))){c->fpsr|=8;_v=_negative?_sign:0;}
|
|
else {int _cut=_top-(int)_f,_mincut=_emin-(int)_f-_base;if(_cut<_mincut)_cut=_mincut;
|
|
unsigned _index=(unsigned)_cut/64,_offset=(unsigned)_cut%64;
|
|
uint64_t _mant=_p[_index]>>_offset;if(_offset&&_index+1<_count)_mant|=_p[_index+1]<<(64-_offset);
|
|
unsigned _roundbit=(unsigned)((_p[(unsigned)(_cut-1)/64]>>((_cut-1)%64))&1);
|
|
unsigned _sticky=recomp_fp_sticky(_p,_count,_cut);
|
|
unsigned _lost=_roundbit|_sticky;if(_lost){c->fpsr|=16;if(_e<_emin)c->fpsr|=8;
|
|
if((_mode==0&&_roundbit&&(_sticky||(_mant&1)))||(_mode==1&&!_negative)||(_mode==2&&_negative))++_mant;}
|
|
if(_e<_emin){_e=_emin;}if(_mant>=(_hidden<<1)){_mant>>=1;++_e;}
|
|
if(_e>(int)_bias){c->fpsr|=20;_v=(_mode==0||(_mode==1&&!_negative)||(_mode==2&&_negative))?_exp:_exp-1;}
|
|
else _v=(_mant>=_hidden?(uint64_t)(_e+(int)_bias)<<_f:0)|(_mant&_frac);
|
|
if(_negative)_v|=_sign;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
)C";
|
|
if (!exact_only) s += EmitFPNativeSuffix(dbl, mul ? "mul" : "fma");
|
|
return s;
|
|
}
|
|
|
|
// FRECPS/FRSQRTS S/D, evaluated exactly before one rounding: 2 - a*b, or
|
|
// (3 - a*b)/2 for FRSQRTS (`half`). Inputs are the raw operand bits _aa and
|
|
// _bb; the result bits are left in _v, which this text declares. A fixed
|
|
// multiword product lattice covers every finite input exponent. This
|
|
// deliberately favors correctness over speed; no host fma or intermediate
|
|
// halve can introduce double rounding. Baseline FP modes only: exception/
|
|
// access trap delivery and FEAT_AFP remain unsupported. The line breaks match
|
|
// what put() emits, so the scalar text is unchanged by sharing it.
|
|
inline std::string EmitFPStepValue(bool dbl, bool half, bool exact_only = false) {
|
|
if (!exact_only && FpxOutline()) {
|
|
return "uint64_t _v=0;" + EmitFPOutlined(dbl, half ? "rsqrts" : "recps");
|
|
}
|
|
std::string s = std::string("const unsigned _f=")+(dbl?"52":"23")+",_bias="+(dbl?"1023":"127")+
|
|
",_count="+(dbl?"67":"9")+";const int _base="+
|
|
std::to_string((dbl?-2148:-298)-(half?1:0))+";"
|
|
"const uint64_t _hidden=1ULL<<_f,_frac=_hidden-1,_quiet=_hidden>>1,_exp="+
|
|
(dbl?"0x7ff0000000000000ULL":"0x7f800000ULL")+",_sign="+
|
|
(dbl?"0x8000000000000000ULL":"0x80000000ULL")+";"
|
|
// FPNeg precedes NaN processing in the architectural pseudocode.
|
|
"uint64_t _a=(uint64_t)_aa^_sign,_b=_bb,_v=0;"+
|
|
(exact_only ? std::string() : EmitFPNativeValue(dbl,half?"rsqrts":"recps"))+
|
|
"unsigned _mode=(unsigned)(c->fpcr>>22)&3;"
|
|
"if(c->fpcr&(1ULL<<24)) {if(!(_a&_exp)&&(_a&_frac)) {_a&=_sign;c->fpsr|=128;}"
|
|
"if(!(_b&_exp)&&(_b&_frac)) {_b&=_sign;c->fpsr|=128;}}"
|
|
"int _an=(_a&_exp)==_exp&&(_a&_frac),_bn=(_b&_exp)==_exp&&(_b&_frac);"
|
|
"if(_an||_bn) {int _as=_an&&!(_a&_quiet),_bs=_bn&&!(_b&_quiet);"
|
|
"_v=(_as?_a:_bs?_b:_an?_a:_b)|_quiet;if(_as||_bs) {c->fpsr|=1;}"
|
|
"if(c->fpcr&(1ULL<<25)) {_v=_exp|_quiet;}}"
|
|
"else if(((_a&_exp)==_exp&&!(_b&~_sign))||((_b&_exp)==_exp&&!(_a&~_sign)))"
|
|
"_v="+(dbl?(half?"0x3ff8000000000000ULL":"0x4000000000000000ULL"):
|
|
(half?"0x3fc00000ULL":"0x40000000ULL"))+";"
|
|
"else if((_a&_exp)==_exp||(_b&_exp)==_exp)_v=_exp|((_a^_b)&_sign);"
|
|
"else {unsigned _ea=(unsigned)((_a&_exp)>>_f),_eb=(unsigned)((_b&_exp)>>_f);"
|
|
"uint64_t _ma=(_a&_frac)|(_ea?_hidden:0),_mb=(_b&_frac)|(_eb?_hidden:0);"
|
|
"uint64_t _p["+std::string(dbl?"67":"9")+"]={0},_c["+(dbl?"67":"9")+"]={0};"
|
|
"uint64_t _a0=(uint32_t)_ma,_a1=_ma>>32,_b0=(uint32_t)_mb,_b1=_mb>>32;"
|
|
"uint64_t _lo=_a0*_b0,_x=_a0*_b1,_y=_a1*_b0,_hi=_a1*_b1+(_x>>32)+(_y>>32);"
|
|
"uint64_t _old=_lo;_lo+=_x<<32;_hi+=_lo<_old;_old=_lo;_lo+=_y<<32;_hi+=_lo<_old;"
|
|
"unsigned _shift=(_ea?_ea-1:0)+(_eb?_eb-1:0),_j=_shift/64,_s=_shift%64;"
|
|
"_p[_j]=_lo<<_s;_p[_j+1]=(_hi<<_s)|(_s?_lo>>(64-_s):0);"
|
|
"if(_s&&_j+2<_count)_p[_j+2]=_hi>>(64-_s);"
|
|
"unsigned _bit="+std::string(dbl?"2149":"299")+";_c[_bit/64]|=1ULL<<(_bit%64);\n ";
|
|
if(half)s+="--_bit;_c[_bit/64]|=1ULL<<(_bit%64);\n ";
|
|
s+="unsigned _negative=0;"
|
|
"if((_a^_b)&_sign) {int _compare=0;for(int _k=(int)_count-1;_k>=0;--_k)"
|
|
"{if(_p[_k]!=_c[_k]) {_compare=_p[_k]>_c[_k]?1:-1;break;}}"
|
|
"_negative=_compare>0;uint64_t _borrow=0;for(unsigned _k=0;_k<_count;++_k)"
|
|
"{uint64_t _left=_negative?_p[_k]:_c[_k],_right=_negative?_c[_k]:_p[_k];"
|
|
"uint64_t _diff=_left-_right,_next=(_left<_right)||(_diff<_borrow);"
|
|
"_p[_k]=_diff-_borrow;_borrow=_next;}}"
|
|
"else {uint64_t _carry=0;for(unsigned _k=0;_k<_count;++_k)"
|
|
"{uint64_t _sum=_p[_k]+_c[_k],_next=_sum<_p[_k];"
|
|
"uint64_t _total=_sum+_carry;_carry=_next||_total<_sum;_p[_k]=_total;}}"
|
|
"int _top=recomp_fp_top_bit(_p,_count);"
|
|
"if(_top<0)_v=_mode==2?_sign:0;else {int _e=_top+_base,_emin=1-(int)_bias;"
|
|
"if(_e<_emin&&(c->fpcr&(1ULL<<24))) {c->fpsr|=8;_v=_negative?_sign:0;}"
|
|
"else {int _cut=_top-(int)_f,_mincut=_emin-(int)_f-_base;if(_cut<_mincut)_cut=_mincut;"
|
|
"unsigned _index=(unsigned)_cut/64,_offset=(unsigned)_cut%64;"
|
|
"uint64_t _mant=_p[_index]>>_offset;if(_offset&&_index+1<_count)_mant|=_p[_index+1]<<(64-_offset);"
|
|
"unsigned _roundbit=(unsigned)((_p[(unsigned)(_cut-1)/64]>>((_cut-1)%64))&1);"
|
|
"unsigned _sticky=recomp_fp_sticky(_p,_count,_cut);"
|
|
"unsigned _lost=_roundbit|_sticky;if(_lost) {c->fpsr|=16;if(_e<_emin)c->fpsr|=8;"
|
|
"if((_mode==0&&_roundbit&&(_sticky||(_mant&1)))||(_mode==1&&!_negative)||(_mode==2&&_negative))++_mant;}"
|
|
"if(_e<_emin)_e=_emin;if(_mant>=(_hidden<<1)) {_mant>>=1;++_e;}"
|
|
"if(_e>(int)_bias) {c->fpsr|=20;_v=(_mode==0||(_mode==1&&!_negative)||(_mode==2&&_negative))?_exp:_exp-1;}"
|
|
"else _v=(_mant>=_hidden?(uint64_t)(_e+(int)_bias)<<_f:0)|(_mant&_frac);"
|
|
"if(_negative)_v|=_sign;}}}"+
|
|
(exact_only ? std::string() : EmitFPNativeSuffix(dbl,half?"rsqrts":"recps"));
|
|
return s;
|
|
}
|
|
|
|
// FSQRT/FABD lane text from raw uint64_t _a/_b to _v, after the FPX prefix:
|
|
// constants, FZ input flushing and NaN propagation. FSQRT continues with
|
|
// EmitFPSqrtTail, FABD with an exact subtraction.
|
|
inline std::string EmitFPSqrtAbdHead(bool dbl, bool sqrt) {
|
|
return std::string(" const unsigned _f=") + (dbl ? "52" : "23") + "; const uint64_t _hidden=1ULL<<_f,"
|
|
" _frac=_hidden-1,_exp=" + (dbl ? "0x7ff0000000000000ULL" : "0x7f800000ULL") +
|
|
",_sign=" + (dbl ? "0x8000000000000000ULL" : "0x80000000ULL") +
|
|
",_quiet=_hidden>>1; " + std::string(sqrt ? "unsigned _mode=(unsigned)(c->fpcr>>22)&3;" : "") +
|
|
" if(c->fpcr&(1ULL<<24)) {"
|
|
" if(!(_a&_exp)&&(_a&_frac)) { _a&=_sign;c->fpsr|=128; }"
|
|
" if(!(_b&_exp)&&(_b&_frac)) { _b&=_sign;c->fpsr|=128; } }"
|
|
" int _an=(_a&_exp)==_exp&&(_a&_frac),_bn=(_b&_exp)==_exp&&(_b&_frac);"
|
|
" int _as=_an&&!(_a&_quiet),_bs=_bn&&!(_b&_quiet);"
|
|
" if(_an||_bn) { _v=(_as?_a:_bs?_b:_an?_a:_b)|_quiet;"
|
|
" if(_as||_bs) { c->fpsr|=1; } if(c->fpcr&(1ULL<<25)) { _v=_exp|_quiet; } }";
|
|
}
|
|
inline std::string EmitFPSqrtTail(bool dbl) {
|
|
return std::string("else if(!(_a&~_sign))_v=_a;"
|
|
" else if(_a&_sign) { _v=_exp|_quiet;c->fpsr|=1; }"
|
|
" else if((_a&_exp)==_exp)_v=_a;"
|
|
" else { const int _bias=") + (dbl ? "1023" : "127") +
|
|
"; uint64_t _magnitude=_a&_frac;"
|
|
" int _e=(int)((_a&_exp)>>_f)-(int)_bias;"
|
|
" if(_a&_exp)_magnitude|=_hidden; else { ++_e;"
|
|
" while(_magnitude<_hidden) { _magnitude<<=1;--_e; } }"
|
|
" if(_e%2) { _magnitude<<=1;--_e; }"
|
|
// Restoring square root of magnitude * 2^fraction_bits.
|
|
// The radicand can exceed 64 bits, but only two bits are
|
|
// consumed each step; root and remainder fit uint64_t.
|
|
" uint64_t _root=0,_rem=0;"
|
|
" for(int _k=(int)_f;_k>=0;--_k) { int _shift=2*_k-(int)_f;"
|
|
" uint64_t _pair=(_shift>=0?_magnitude>>_shift:_magnitude<<(-_shift))&3;"
|
|
" _rem=(_rem<<2)|_pair;uint64_t _trial=(_root<<2)|1;_root<<=1;"
|
|
" if(_rem>=_trial) { _rem-=_trial;_root|=1; } }"
|
|
" if(_rem) { c->fpsr|=16;"
|
|
" if(_mode==1||(_mode==0&&_rem>_root))++_root; }"
|
|
" _e=_e/2+(int)_bias;"
|
|
" if(_root>=(_hidden<<1)) { _root>>=1;++_e; }"
|
|
" _v=((uint64_t)_e<<_f)|(_root&_frac); }";
|
|
}
|
|
|
|
// The out-of-line exact bodies FPX1 sites call (EmitFPOutlined): one function
|
|
// per op and width, in the runtime C, so once per module (once per image when
|
|
// the static build shares the runtime). Each takes the raw operand bits the
|
|
// inline body read, returns the result bits and updates c->fpsr exactly as the
|
|
// inline body did; the text is the same emitter output. FMUL uses fma.
|
|
inline std::string FpxExactC() {
|
|
std::string s = R"RT(
|
|
/* FPX1: the exact bodies, out of line. The native fast path stays inline at
|
|
each site; these run only when its gate is closed or its result is not
|
|
provably exact, so they are kept out of the hot code. */
|
|
#if defined(__GNUC__) || defined(__clang__)
|
|
#define RECOMP_FPE_COLD __attribute__((noinline, cold))
|
|
#elif defined(_MSC_VER)
|
|
#define RECOMP_FPE_COLD __declspec(noinline)
|
|
#else
|
|
#define RECOMP_FPE_COLD
|
|
#endif
|
|
)RT";
|
|
for (const bool dbl : {false, true}) {
|
|
const std::string w = dbl ? "64" : "32";
|
|
const std::string two = "(GuestContext* c,uint64_t _a,uint64_t _b){uint64_t _v=0;\n";
|
|
s += "RECOMP_FPE_COLD uint64_t recomp_fpe_add" + w + two + EmitFPAddSubValue(dbl, false, true) +
|
|
"\nreturn _v;}\n";
|
|
s += "RECOMP_FPE_COLD uint64_t recomp_fpe_sub" + w + two + EmitFPAddSubValue(dbl, true, true) +
|
|
"\nreturn _v;}\n";
|
|
s += "RECOMP_FPE_COLD uint64_t recomp_fpe_div" + w + two + EmitFPDivideValue(dbl, true) +
|
|
"\nreturn _v;}\n";
|
|
s += "RECOMP_FPE_COLD uint64_t recomp_fpe_fma" + w +
|
|
"(GuestContext* c,uint64_t _a,uint64_t _b,uint64_t _z){uint64_t _v=0;\n" +
|
|
EmitFPMulAddValue(dbl, false, true) + "\nreturn _v;}\n";
|
|
for (const bool half : {false, true}) {
|
|
s += "RECOMP_FPE_COLD uint64_t recomp_fpe_" + std::string(half ? "rsqrts" : "recps") + w +
|
|
"(GuestContext* c,uint64_t _aa,uint64_t _bb){\n" + EmitFPStepValue(dbl, half, true) +
|
|
"\nreturn _v;}\n";
|
|
}
|
|
s += "RECOMP_FPE_COLD uint64_t recomp_fpe_sqrt" + w +
|
|
"(GuestContext* c,uint64_t _a){uint64_t _b=0,_v=0;\n" + EmitFPSqrtAbdHead(dbl, true) +
|
|
EmitFPSqrtTail(dbl) + "\nreturn _v;}\n";
|
|
}
|
|
return s;
|
|
}
|
|
|
|
// Declarations for the runtime header.
|
|
inline const char* FpxExactH() {
|
|
return R"RT(/* FPX1: the exact bodies the fast path falls back to, out of line
|
|
(recomp_runtime.c). Raw operand bits in, result bits out, FPSR in c. */
|
|
uint64_t recomp_fpe_add32(GuestContext*,uint64_t,uint64_t);
|
|
uint64_t recomp_fpe_sub32(GuestContext*,uint64_t,uint64_t);
|
|
uint64_t recomp_fpe_div32(GuestContext*,uint64_t,uint64_t);
|
|
uint64_t recomp_fpe_fma32(GuestContext*,uint64_t,uint64_t,uint64_t);
|
|
uint64_t recomp_fpe_recps32(GuestContext*,uint64_t,uint64_t);
|
|
uint64_t recomp_fpe_rsqrts32(GuestContext*,uint64_t,uint64_t);
|
|
uint64_t recomp_fpe_sqrt32(GuestContext*,uint64_t);
|
|
uint64_t recomp_fpe_add64(GuestContext*,uint64_t,uint64_t);
|
|
uint64_t recomp_fpe_sub64(GuestContext*,uint64_t,uint64_t);
|
|
uint64_t recomp_fpe_div64(GuestContext*,uint64_t,uint64_t);
|
|
uint64_t recomp_fpe_fma64(GuestContext*,uint64_t,uint64_t,uint64_t);
|
|
uint64_t recomp_fpe_recps64(GuestContext*,uint64_t,uint64_t);
|
|
uint64_t recomp_fpe_rsqrts64(GuestContext*,uint64_t,uint64_t);
|
|
uint64_t recomp_fpe_sqrt64(GuestContext*,uint64_t);
|
|
)RT";
|
|
}
|
|
|
|
// Arm FPMin/FPMax/FPMinNum/FPMaxNum on raw S/D bits `a` and `b`, with bitwise
|
|
// ordering so host NaN, signed-zero and denormal modes cannot intervene.
|
|
// Implements baseline FPCR.DN/FZ and cumulative FPSR.IOC/IDC; `store` consumes
|
|
// the result in _v. The line breaks match what put() emits.
|
|
inline std::string EmitFPMinMax(bool dbl, bool minimum, bool numeric, const std::string& a,
|
|
const std::string& b, const std::string& store) {
|
|
std::string s = "{ uint64_t _a=" + a + ",_b=" + b + ",_v; const uint64_t _sign=" +
|
|
(dbl ? std::string("0x8000000000000000ULL") : "0x80000000ULL") +
|
|
",_exp=" + (dbl ? "0x7ff0000000000000ULL" : "0x7f800000ULL") +
|
|
",_frac=" + (dbl ? "0xfffffffffffffULL" : "0x7fffffULL") +
|
|
",_quiet=" + (dbl ? "0x8000000000000ULL" : "0x400000ULL") + ";\n ";
|
|
s += "if(c->fpcr & (1ULL<<24)) {"
|
|
" if(!(_a&_exp)&&(_a&_frac)) { _a&=_sign; c->fpsr|=128; }"
|
|
" if(!(_b&_exp)&&(_b&_frac)) { _b&=_sign; c->fpsr|=128; } }\n ";
|
|
s += "{ int _an=(_a&_exp)==_exp&&(_a&_frac),"
|
|
" _bn=(_b&_exp)==_exp&&(_b&_frac);"
|
|
" int _as=_an&&!(_a&_quiet),_bs=_bn&&!(_b&_quiet);"
|
|
" if(_as||_bs) { c->fpsr|=1; _v=(_as?_a:_b)|_quiet; }\n ";
|
|
if (numeric) {
|
|
s += "else if(_an&&!_bn) _v=_b; else if(_bn&&!_an) _v=_a;\n ";
|
|
}
|
|
s += "else if(_an||_bn) _v=(_an?_a:_b)|_quiet;"
|
|
" else if(!((_a|_b)&~_sign)) _v=" +
|
|
std::string(minimum ? "(_a|_b)" : "(_a&_b)") + ";"
|
|
" else { int _less=((_a^_b)&_sign)?!!(_a&_sign):"
|
|
" ((_a&_sign)?_a>_b:_a<_b); _v=" +
|
|
std::string(minimum ? "(_less?_a:_b)" : "(_less?_b:_a)") + "; }\n ";
|
|
s += "if((c->fpcr&(1ULL<<25))&&((_v&_exp)==_exp)&&(_v&_frac))"
|
|
" { _v=_exp|_quiet; } " + store + " } }";
|
|
return s;
|
|
}
|
|
|
|
// FCMP-style comparison of S/D registers `a` and `b` (register arrays; an
|
|
// empty `b` compares with +0.0) into NZCV: 0110 equal, 1000 less, 0010
|
|
// greater, 0011 unordered. Integer ordering of the IEEE bits, so host compare
|
|
// semantics and modes cannot intervene. FZ flushes subnormal inputs (IDC); a
|
|
// NaN raises IOC when it is signalling, or for any NaN when `signal` is set.
|
|
inline std::string EmitFPCompareFlags(bool dbl, bool signal, const std::string& a,
|
|
const std::string& b) {
|
|
const std::string sz = dbl ? "8" : "4";
|
|
return "{ uint64_t _a=0,_b=0; memcpy(&_a," + a + "," + sz + ");" +
|
|
(b.empty() ? std::string() : " memcpy(&_b," + b + "," + sz + ");") +
|
|
" const uint64_t _sign=" + (dbl ? "0x8000000000000000ULL" : "0x80000000ULL") +
|
|
",_exp=" + (dbl ? "0x7ff0000000000000ULL" : "0x7f800000ULL") +
|
|
",_frac=" + (dbl ? "0xfffffffffffffULL" : "0x7fffffULL") +
|
|
",_quiet=" + (dbl ? "0x8000000000000ULL" : "0x400000ULL") + ";"
|
|
" if(c->fpcr&(1ULL<<24)) { if(!(_a&_exp)&&(_a&_frac)) { _a&=_sign; c->fpsr|=128; }"
|
|
" if(!(_b&_exp)&&(_b&_frac)) { _b&=_sign; c->fpsr|=128; } }"
|
|
" int _an=(_a&_exp)==_exp&&(_a&_frac),_bn=(_b&_exp)==_exp&&(_b&_frac);"
|
|
" if(_an||_bn) { if(" + (signal ? "1" : "(_an&&!(_a&_quiet))||(_bn&&!(_b&_quiet))") +
|
|
") c->fpsr|=1; c->n=0; c->z=0; c->c=1; c->v=1; }"
|
|
" else { int _eq=_a==_b||!((_a|_b)&~_sign);"
|
|
" int _lt=!_eq&&(((_a^_b)&_sign)?!!(_a&_sign):((_a&_sign)?_a>_b:_a<_b));"
|
|
" c->n=(uint8_t)_lt; c->z=(uint8_t)_eq; c->c=(uint8_t)!_lt; c->v=0; } }";
|
|
}
|
|
|
|
// FCMEQ/FCMGE/FCMGT (`kind` "eq", "ge", "gt") of raw S/D bits `a` and `b` into
|
|
// an all-ones or zero _v, which `store` consumes. Integer ordering of the IEEE
|
|
// bits; FZ flushes subnormal inputs (IDC). A NaN compares false and raises IOC
|
|
// when it is signalling, or for any NaN in the ordered GE/GT forms.
|
|
inline std::string EmitFPCompareMask(bool dbl, const std::string& kind, const std::string& a,
|
|
const std::string& b, const std::string& store) {
|
|
const bool eq = kind == "eq";
|
|
return "{ uint64_t _a=" + a + ",_b=" + b + ",_v=0; const uint64_t _sign=" +
|
|
(dbl ? "0x8000000000000000ULL" : "0x80000000ULL") +
|
|
",_exp=" + (dbl ? "0x7ff0000000000000ULL" : "0x7f800000ULL") +
|
|
",_frac=" + (dbl ? "0xfffffffffffffULL" : "0x7fffffULL") +
|
|
",_quiet=" + (dbl ? "0x8000000000000ULL" : "0x400000ULL") + ";"
|
|
" if(c->fpcr&(1ULL<<24)) { if(!(_a&_exp)&&(_a&_frac)) { _a&=_sign; c->fpsr|=128; }"
|
|
" if(!(_b&_exp)&&(_b&_frac)) { _b&=_sign; c->fpsr|=128; } }"
|
|
" int _an=(_a&_exp)==_exp&&(_a&_frac),_bn=(_b&_exp)==_exp&&(_b&_frac);"
|
|
" if(_an||_bn) { if(" + std::string(eq ? "(_an&&!(_a&_quiet))||(_bn&&!(_b&_quiet))" : "1") +
|
|
") c->fpsr|=1; }"
|
|
" else { int _eq=_a==_b||!((_a|_b)&~_sign);"
|
|
" int _gt=!_eq&&(((_a^_b)&_sign)?!(_a&_sign):((_a&_sign)?_a<_b:_a>_b)); (void)_gt; (void)_eq;"
|
|
" if(" + (eq ? "_eq" : kind == "ge" ? "_eq||_gt" : "_gt") + ") _v=~(uint64_t)0; } " + store + " }";
|
|
}
|
|
|
|
inline bool Translate(u32 i, u64 pc, std::string& out, bool* unhandled = nullptr) {
|
|
if (unhandled) {
|
|
*unhandled = false;
|
|
}
|
|
char buf[256];
|
|
// Appends in place. Taking a string_view and appending piecewise (rather
|
|
// than `out += " " + s + "\n"`) matters at scale: that expression built
|
|
// two temporary std::strings per emitted line, and a full title is tens of
|
|
// millions of lines - it was the single largest cost in AOT translation.
|
|
auto put = [&](std::string_view s) {
|
|
out.append(" ", 4);
|
|
out.append(s);
|
|
out.push_back('\n');
|
|
};
|
|
// Every "we cannot translate this" path goes through here. Three sites used
|
|
// to emit the call by hand and return early, which meant they never set the
|
|
// coverage flag - the exclusives alone were 5,036 instructions per export
|
|
// that the static coverage figure silently did not count.
|
|
auto put_unhandled = [&]() {
|
|
if (unhandled) {
|
|
*unhandled = true;
|
|
}
|
|
char b[128];
|
|
snprintf(b, sizeof b,
|
|
"recomp_unhandled(c,0x%08xU,g_module_base+0x%llxULL); if(c->halted) return;", i,
|
|
(unsigned long long)pc);
|
|
put(b);
|
|
};
|
|
// S/D FRECPE and FRSQRTE share one lane core; U selects reciprocal sqrt.
|
|
// Decode before the legacy conversion class, whose opcode is shared here.
|
|
const bool estimate_scalar = (i & 0xDFBFFC00u) == 0x5EA1D800u;
|
|
const bool estimate_vector = (i & 0x9FBFFC00u) == 0x0EA1D800u;
|
|
if (estimate_scalar || estimate_vector) {
|
|
const bool dbl = (i & (1u << 22)) != 0;
|
|
const bool q = (i & (1u << 30)) != 0;
|
|
if (estimate_vector && dbl && !q) { put_unhandled(); return true; }
|
|
const unsigned rn = (i >> 5) & 31, rd = i & 31;
|
|
const unsigned width = dbl ? 64 : 32;
|
|
const unsigned bytes = estimate_scalar ? width / 8 : q ? 16 : 8;
|
|
const unsigned lanes = bytes / (width / 8);
|
|
const std::string ty = dbl ? "uint64_t" : "uint32_t";
|
|
std::string s = "{ " + ty + " _a[" + std::to_string(lanes) + "],_r[" +
|
|
std::to_string(lanes) + "]; memcpy(_a,c->vreg[" +
|
|
std::to_string(rn) + "]," + std::to_string(bytes) + "); ";
|
|
s += "for(unsigned _i=0;_i<" + std::to_string(lanes) + ";++_i) _r[_i]=(" + ty +
|
|
")recomp_fp_estimate(c,_a[_i]," + std::to_string(width) + "," +
|
|
std::to_string((i >> 29) & 1) + "); ";
|
|
s += "memset(c->vreg[" + std::to_string(rd) + "],0,16); memcpy(c->vreg[" +
|
|
std::to_string(rd) + "],_r," + std::to_string(bytes) + "); }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
const u64 next = pc + 4;
|
|
|
|
if ((i & 0xFFFFF01F) == 0xD503201F) { put("/* nop/hint */"); return true; }
|
|
|
|
// Memory barriers: DSB, DMB, ISB and CLREX. The generated runtime executes
|
|
// one guest thread on one host thread, so there is no other observer for
|
|
// these to order against and nothing to synchronise - they are genuinely
|
|
// no-ops here. Under Core::ArmRecomp, where real threads exist, these need
|
|
// the host's own barriers instead; noted so the assumption stays visible.
|
|
// Emitting nothing at all is only right for the standalone runtime, which
|
|
// drives one guest thread on one host thread and so has no second observer
|
|
// to order against. Under Core::ArmRecomp the guest really is
|
|
// multi-threaded, and a comment does not stop the host compiler reordering
|
|
// the recomp_load/recomp_store calls around it - so a publish pattern (fill
|
|
// an object, barrier, store the pointer) can be observed pointer-first by
|
|
// another thread, surfacing much later as a live vtable pointer reading
|
|
// back as zero.
|
|
// CLREX before the barrier group: it shares their encoding shape but has to
|
|
// drop the exclusive mark, and treating it as a plain barrier left a stale
|
|
// mark that could make a later unrelated STXR succeed.
|
|
if ((i & 0xFFFFF0FF) == 0xD503305F) { put("recomp_clrex(c);"); return true; }
|
|
|
|
if ((i & 0xFFFFF0FF) == 0xD50330DF) {
|
|
snprintf(buf, sizeof buf, "recomp_barrier(); c->pc=g_module_base+0x%llxULL; return;", (unsigned long long)next);
|
|
put(buf); return false;
|
|
}
|
|
if ((i & 0xFFFFF01F) == 0xD503301F) { put("recomp_barrier();"); return true; }
|
|
|
|
if ((i & 0x1F800000) == 0x12800000) { // MOVZ/MOVN/MOVK
|
|
u32 sf = i >> 31, opc = (i >> 29) & 3, hw = (i >> 21) & 3, imm16 = (i >> 5) & 0xFFFF, rd = i & 31;
|
|
if (rd != 31) {
|
|
u64 shift = (u64)hw * 16;
|
|
if (opc == 2) {
|
|
snprintf(buf, sizeof buf, "c->x[%u] = 0x%llxULL;", rd, (unsigned long long)((u64)imm16 << shift));
|
|
put(buf);
|
|
} else if (opc == 0) {
|
|
u64 v = ~((u64)imm16 << shift); if (!sf) v &= 0xFFFFFFFF;
|
|
snprintf(buf, sizeof buf, "c->x[%u] = 0x%llxULL;", rd, (unsigned long long)v); put(buf);
|
|
} else if (opc == 3) {
|
|
snprintf(buf, sizeof buf, "c->x[%u] = (c->x[%u] & ~(0xFFFFULL<<%llu)) | (0x%xULL<<%llu);",
|
|
rd, rd, (unsigned long long)shift, imm16, (unsigned long long)shift); put(buf);
|
|
}
|
|
if (!sf) { snprintf(buf, sizeof buf, "c->x[%u] &= 0xFFFFFFFFULL;", rd); put(buf); }
|
|
}
|
|
return true;
|
|
}
|
|
|
|
if ((i & 0x1F000000) == 0x11000000) { // ADD/SUB immediate
|
|
u32 sf = i >> 31, op = (i >> 30) & 1, S = (i >> 29) & 1, sh = (i >> 22) & 1;
|
|
u32 imm12 = (i >> 10) & 0xFFF, rn = (i >> 5) & 31, rd = i & 31;
|
|
u64 imm = sh ? ((u64)imm12 << 12) : imm12;
|
|
snprintf(buf, sizeof buf, "{ uint64_t _a=c->x[%u], _b=%lluULL; uint64_t _r=%s; ", rn,
|
|
(unsigned long long)imm, op ? "_a-_b" : "_a+_b");
|
|
std::string s = buf;
|
|
if (!sf) s += "_r&=0xFFFFFFFFULL; ";
|
|
// With the flag-setting form, register 31 as the destination is the
|
|
// zero register - that encoding is CMP - so the result is dropped.
|
|
// Without it, register 31 is SP and the write is real.
|
|
if (!(rd == 31 && S)) s += "c->x[" + std::to_string(rd) + "]=_r; ";
|
|
if (S) s += SetFlags(op != 0, "_a", "_b", "_r", sf != 0) + " ";
|
|
s += "}";
|
|
// Register 31 is SP here, not the zero register, so a write to it is
|
|
// real and must not be discarded: dropping it throws away every
|
|
// "sub sp, sp, #N" that opens a stack frame.
|
|
put(s);
|
|
return true;
|
|
}
|
|
|
|
// Build a shifted-register operand. LSR/ASR/ROR all used to collapse to a
|
|
// plain ">>" on a uint64_t, which zero-fills - so ASR lost the sign (the
|
|
// "bic w0, w0, w0, asr #31" max(x,0) idiom produced ~0 instead of 0) and
|
|
// ROR dropped the wrapped-around bits entirely (breaking, among other
|
|
// things, software CRC32). The 32-bit forms also have to be narrowed
|
|
// before shifting, whatever the shift amount, because the register may
|
|
// still carry high garbage from an earlier 64-bit write.
|
|
auto shifted_operand = [](const std::string& v, u32 shift, u32 imm6, u32 sf) -> std::string {
|
|
char sb[256];
|
|
if (!sf) {
|
|
const std::string w = "((uint32_t)(" + v + "))";
|
|
switch (shift) {
|
|
case 0: snprintf(sb, sizeof sb, "((uint64_t)(uint32_t)(%s << %u))", w.c_str(), imm6); break;
|
|
case 1: snprintf(sb, sizeof sb, "((uint64_t)(%s >> %u))", w.c_str(), imm6); break;
|
|
case 2: snprintf(sb, sizeof sb, "((uint64_t)(uint32_t)((int32_t)%s >> %u))", w.c_str(), imm6); break;
|
|
default:
|
|
if (imm6 == 0) { snprintf(sb, sizeof sb, "((uint64_t)%s)", w.c_str()); }
|
|
else { snprintf(sb, sizeof sb, "((uint64_t)(uint32_t)((%s >> %u) | (%s << %u)))", w.c_str(), imm6, w.c_str(), 32 - imm6); }
|
|
break;
|
|
}
|
|
} else {
|
|
switch (shift) {
|
|
case 0: snprintf(sb, sizeof sb, "((%s) << %u)", v.c_str(), imm6); break;
|
|
case 1: snprintf(sb, sizeof sb, "((%s) >> %u)", v.c_str(), imm6); break;
|
|
case 2: snprintf(sb, sizeof sb, "((uint64_t)((int64_t)(%s) >> %u))", v.c_str(), imm6); break;
|
|
default:
|
|
if (imm6 == 0) { snprintf(sb, sizeof sb, "(%s)", v.c_str()); }
|
|
else { snprintf(sb, sizeof sb, "(((%s) >> %u) | ((%s) << %u))", v.c_str(), imm6, v.c_str(), 64 - imm6); }
|
|
break;
|
|
}
|
|
}
|
|
return sb;
|
|
};
|
|
|
|
if ((i & 0x1F000000) == 0x0A000000) { // logical shifted register
|
|
u32 sf = i >> 31, opc = (i >> 29) & 3, rm = (i >> 16) & 31, rn = (i >> 5) & 31, rd = i & 31;
|
|
u32 shift = (i >> 22) & 3, imm6 = (i >> 10) & 0x3F, N = (i >> 21) & 1;
|
|
// The 32-bit form only allocates imm6 < 32: "if sf == '0' && imm6<5>
|
|
// == '1' then UNDEFINED". Translating it anyway shifted a value that
|
|
// had already been narrowed to uint32_t by a 64-bit amount, and the
|
|
// ROR complement 32-imm6 underflowed to ~4.29e9 - undefined behaviour
|
|
// that MSVC rejects outright (C4293). It is not a real instruction, so
|
|
// the answer is to not decode it: these encodings only ever turn up
|
|
// where the ARM64 decoder speculatively walks an ARM32 region (0x0AFFFFF0,
|
|
// an ARM32 "beq", is exactly the BIC w16,wzr,wzr,ror #63 seen here).
|
|
if (!sf && imm6 >= 32) {
|
|
put_unhandled();
|
|
return true;
|
|
}
|
|
std::string rmv = shifted_operand(Xz(rm), shift, imm6, sf);
|
|
std::string a = Xz(rn);
|
|
const char* lop = opc == 0 ? "&" : opc == 1 ? "|" : opc == 2 ? "^" : "&";
|
|
std::string expr = N ? ("(" + a + " " + lop + " ~" + rmv + ")") : ("(" + a + " " + lop + " " + rmv + ")");
|
|
// opc==3 is ANDS/BICS: it sets the flags, and with rd==31 it is TST,
|
|
// whose only effect IS the flag update. Gating the whole instruction on
|
|
// rd != 31 discarded every TST, leaving the following b.cond to branch
|
|
// on whatever flags happened to be left over from an earlier compare.
|
|
std::string s = "{ uint64_t _r = " + expr + "; ";
|
|
if (!sf) s += "_r &= 0xFFFFFFFFULL; ";
|
|
if (rd != 31) s += "c->x[" + std::to_string(rd) + "] = _r; ";
|
|
if (opc == 3) s += SetFlags(false, "_r", "0", "_r", sf != 0) + " ";
|
|
s += "}";
|
|
put(s);
|
|
return true;
|
|
}
|
|
|
|
if ((i & 0x1F200000) == 0x0B000000) { // ADD/SUB shifted register
|
|
u32 sf = i >> 31, op = (i >> 30) & 1, S = (i >> 29) & 1, shift = (i >> 22) & 3, rm = (i >> 16) & 31, imm6 = (i >> 10) & 0x3F, rn = (i >> 5) & 31, rd = i & 31;
|
|
// ROR is reserved for ADD/SUB shifted register - decoding it as a shift
|
|
// would silently invent an instruction the CPU does not have.
|
|
// Same width rule as the logical form: the 32-bit encoding is only
|
|
// allocated for imm6 < 32, and shifting a uint32_t by 32..63 is
|
|
// undefined behaviour rather than a translation.
|
|
if (shift == 3 || (!sf && imm6 >= 32)) {
|
|
put_unhandled();
|
|
return true;
|
|
}
|
|
std::string rmv = shifted_operand(Xz(rm), shift, imm6, sf);
|
|
std::string a = Xz(rn);
|
|
snprintf(buf, sizeof buf, "{ uint64_t _a=%s,_b=%s,_r=%s; ", a.c_str(), rmv.c_str(), op ? "_a-_b" : "_a+_b");
|
|
std::string s = buf; if (!sf) s += "_r&=0xFFFFFFFFULL; ";
|
|
if (rd != 31) s += "c->x[" + std::to_string(rd) + "]=_r; ";
|
|
if (S) s += SetFlags(op != 0, "_a", "_b", "_r", sf != 0) + " ";
|
|
s += "}"; put(s); return true;
|
|
}
|
|
|
|
if ((i & 0x1F000000) == 0x10000000) { // ADR/ADRP
|
|
u32 op = i >> 31, rd = i & 31; s64 immhi = (s32)(((i >> 5) & 0x7FFFF) << 13) >> 13; u32 immlo = (i >> 29) & 3;
|
|
if (rd != 31) {
|
|
// These produce data pointers the guest then dereferences, so they
|
|
// have to be real addresses. Everything the static pass knows is
|
|
// module-relative, so the module's load base is added at run time -
|
|
// without it every computed pointer lands near null.
|
|
if (op) { u64 b = (pc & ~0xFFFULL); s64 imm = ((immhi << 2) | immlo) << 12; snprintf(buf, sizeof buf, "c->x[%u]=g_module_base + 0x%llxULL + (int64_t)%lld;", rd, (unsigned long long)b, (long long)imm); }
|
|
else { s64 imm = (immhi << 2) | immlo; snprintf(buf, sizeof buf, "c->x[%u]=g_module_base + 0x%llxULL + (int64_t)%lld;", rd, (unsigned long long)pc, (long long)imm); }
|
|
put(buf);
|
|
}
|
|
return true;
|
|
}
|
|
|
|
// PRFM / PRFUM - prefetch hints.
|
|
//
|
|
// A hint has no architectural effect, so emitting nothing is not an
|
|
// approximation, it is the correct translation. These were reaching the
|
|
// "size==3 with opc>=2 is not a defined load/store" path and being sent to
|
|
// the JIT, which is both wrong and expensive: 1,606 instructions in Super
|
|
// a second title, 16.8% of the static gap, for something that does nothing.
|
|
if ((i & 0xFFC00000) == 0xF9800000 || // PRFM (unsigned offset)
|
|
(i & 0xFFE00C00) == 0xF8A00800 || // PRFM (register offset)
|
|
(i & 0xFFE00C00) == 0xF8800000) { // PRFUM (unscaled)
|
|
put("/* prfm: hint, no effect */");
|
|
return true;
|
|
}
|
|
|
|
// PC-relative literal loads. GPR register 31 discards the value, but still
|
|
// performs the memory access; vector register 31 is an ordinary register.
|
|
if ((i & 0x3B000000) == 0x18000000) {
|
|
const u32 opc = i >> 30, V = (i >> 26) & 1, rt = i & 31;
|
|
const u32 field = (i >> 5) & 0x7FFFF;
|
|
const s64 offset = (field & 0x40000) ? (s64)field - 0x80000 : (s64)field;
|
|
const u64 address = pc + (u64)(offset * 4);
|
|
// Module-relative like ADR/ADRP above: the load base is only known at
|
|
// run time.
|
|
char literal[48];
|
|
snprintf(literal, sizeof literal, "(g_module_base+0x%llxULL)",
|
|
(unsigned long long)address);
|
|
const std::string addr = literal;
|
|
if (!V && opc == 3) {
|
|
put("/* prfm literal: hint, no effect */");
|
|
return true;
|
|
}
|
|
if (opc < 3) {
|
|
const std::string bits = opc == 0 || (!V && opc == 2) ? "32" : "64";
|
|
std::string s = "{ uint64_t _v=recomp_load" + bits + "(c," + addr + "); ";
|
|
if (V) {
|
|
s += "c->vreg[" + std::to_string(rt) + "][0]=_v; ";
|
|
s += "c->vreg[" + std::to_string(rt) + "][1]=" +
|
|
(opc == 2 ? "recomp_load64(c," + addr + "+8)" : "0") + "; ";
|
|
} else if (rt != 31) {
|
|
// Unsigned subtraction defines signed-word extension without
|
|
// an out-of-range signed cast in the generated C.
|
|
if (opc == 2) s += "_v=(_v^0x80000000ULL)-0x80000000ULL; ";
|
|
s += "c->x[" + std::to_string(rt) + "]=_v; ";
|
|
} else {
|
|
s += "(void)_v; ";
|
|
}
|
|
put(s + "}");
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// LDR/STR immediate unsigned offset. Bit 26 is the V bit: it selects the
|
|
// SIMD/FP register file rather than the general registers. Leaving it out
|
|
// of the mask made every "LDR s0, [x1, #8]" compile into an integer load
|
|
// of x0 - silently wrong code rather than an honest fallback. The SIMD
|
|
// form is handled separately further down.
|
|
if ((i & 0x3F000000) == 0x39000000) {
|
|
u32 size = (i >> 30) & 3, opc = (i >> 22) & 3, imm12 = (i >> 10) & 0xFFF, rn = (i >> 5) & 31, rt = i & 31;
|
|
u64 off = (u64)imm12 << size;
|
|
std::string addr = "c->x[" + std::to_string(rn) + "] + " + std::to_string(off);
|
|
const char* ty = size == 0 ? "8" : size == 1 ? "16" : size == 2 ? "32" : "64";
|
|
// opc: 00 store, 01 load (zero-extend), 10 load signed to 64-bit,
|
|
// 11 load signed to 32-bit. Testing (opc & 1) got this wrong in both
|
|
// directions - opc 11 loaded without sign extension, and opc 10 (a
|
|
// *load*) fell into the store branch and wrote to memory instead.
|
|
if (opc == 0) {
|
|
snprintf(buf, sizeof buf, "recomp_store%s(c,%s,%s);", ty, addr.c_str(), Xz(rt).c_str());
|
|
put(buf);
|
|
} else if (opc == 1) {
|
|
if (rt != 31) {
|
|
snprintf(buf, sizeof buf, "c->x[%u]=recomp_load%s(c,%s);", rt, ty, addr.c_str());
|
|
put(buf);
|
|
}
|
|
} else if (size < 3) {
|
|
// Signed load: sign-extend from the accessed width.
|
|
const char* st = size == 0 ? "int8_t" : size == 1 ? "int16_t" : "int32_t";
|
|
if (rt != 31) {
|
|
std::string s = "{ uint64_t _r = (uint64_t)(int64_t)(" + std::string(st) +
|
|
")recomp_load" + ty + "(c," + addr + "); ";
|
|
if (opc == 3) s += "_r &= 0xFFFFFFFFULL; "; // 32-bit destination
|
|
s += "c->x[" + std::to_string(rt) + "] = _r; }";
|
|
put(s);
|
|
}
|
|
} else {
|
|
// size==3 with opc>=2 is not a defined load/store here.
|
|
put_unhandled();
|
|
return true;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
u64 t = 0;
|
|
// Only the unconditional forms here: DirectBranchTarget also decodes
|
|
// B.cond/CBZ/TBZ for block discovery, but those have their own translations
|
|
// further down and must not be turned into unconditional jumps.
|
|
if (((i & 0xFC000000) == 0x14000000 || (i & 0xFC000000) == 0x94000000) &&
|
|
DirectBranchTarget(i, pc, t)) {
|
|
if ((i & 0xFC000000) == 0x94000000) {
|
|
snprintf(buf, sizeof buf, "c->x[30]=g_module_base+0x%llxULL;", (unsigned long long)next);
|
|
put(buf);
|
|
}
|
|
snprintf(buf, sizeof buf, "c->pc=g_module_base+0x%llxULL; return;",
|
|
(unsigned long long)t);
|
|
put(buf);
|
|
return false;
|
|
}
|
|
if ((i & 0xFFFFFC1F) == 0xD65F0000) { put("c->pc=c->x[30]; return; /* RET */"); return false; }
|
|
if ((i & 0xFFFFFC1F) == 0xD61F0000) { u32 rn = (i >> 5) & 31; put("c->pc=c->x[" + std::to_string(rn) + "]; return; /* BR */"); return false; }
|
|
if ((i & 0xFFFFFC1F) == 0xD63F0000) {
|
|
u32 rn = (i >> 5) & 31;
|
|
snprintf(buf, sizeof buf,
|
|
"{ uint64_t _target=c->x[%u]; c->x[30]=g_module_base+0x%llxULL; "
|
|
"c->pc=_target; return; } /* BLR */",
|
|
rn, (unsigned long long)next);
|
|
put(buf);
|
|
return false;
|
|
}
|
|
if ((i & 0xFF000010) == 0x54000000) {
|
|
const s64 off = ((s32)((i >> 5) << 13) >> 13);
|
|
const u64 tt = pc + off * 4;
|
|
const u32 cond = i & 15;
|
|
snprintf(buf, sizeof buf,
|
|
"if %s { c->pc=g_module_base+0x%llxULL; } else { "
|
|
"c->pc=g_module_base+0x%llxULL; } return;",
|
|
Cond(cond).c_str(), (unsigned long long)tt,
|
|
(unsigned long long)next);
|
|
put(buf);
|
|
return false;
|
|
}
|
|
if ((i & 0x7E000000) == 0x34000000) { u32 sf = i >> 31; bool nz = (i >> 24) & 1; u32 rt = i & 31; s64 off = ((s32)(((i >> 5) & 0x7FFFF) << 13) >> 13); u64 tt = pc + off * 4; std::string v = sf ? Xz(rt) : Wz(rt); snprintf(buf, sizeof buf, "if ((%s)%s0) { c->pc=g_module_base+0x%llxULL; } else { c->pc=g_module_base+0x%llxULL; } return;", v.c_str(), nz ? "!=" : "==", (unsigned long long)tt, (unsigned long long)next); put(buf); return false; }
|
|
if ((i & 0x7E000000) == 0x36000000) { bool nz = (i >> 24) & 1; u32 b = ((i >> 31) << 5) | ((i >> 19) & 31); u32 rt = i & 31; s64 off = ((s32)(((i >> 5) & 0x3FFF) << 18) >> 18); u64 tt = pc + off * 4; snprintf(buf, sizeof buf, "if (((%s>>%u)&1)%s0) { c->pc=g_module_base+0x%llxULL; } else { c->pc=g_module_base+0x%llxULL; } return;", Xz(rt).c_str(), b, nz ? "!=" : "==", (unsigned long long)tt, (unsigned long long)next); put(buf); return false; }
|
|
if ((i & 0xFFE0001F) == 0xD4000001) { u32 imm = (i >> 5) & 0xFFFF; snprintf(buf, sizeof buf, "c->pc=g_module_base+0x%llxULL; c->pending_svc=%uULL; recomp_svc(c,%u); return;", (unsigned long long)next, imm, imm); put(buf); return false; }
|
|
if ((i & 0xFFE0001F) == 0xD4200000) {
|
|
const u32 imm = (i >> 5) & 0xFFFF;
|
|
snprintf(buf, sizeof buf, "/* brk #0x%x */ c->pc=g_module_base+0x%llxULL; c->halted=RECOMP_HALT_BREAKPOINT; return;",
|
|
imm, (unsigned long long)pc);
|
|
put(buf);
|
|
return false;
|
|
}
|
|
|
|
// STP/LDP - load/store pair. Every non-leaf AArch64 function opens and
|
|
// closes with these, so without them a real game stops at its first
|
|
// prologue. Covers the signed-offset, pre-index and post-index forms for
|
|
// both 32- and 64-bit operands.
|
|
// Bit 26 (V) must be clear here; the SIMD/FP pair form is handled below.
|
|
if ((i & 0x3E000000) == 0x28000000) {
|
|
const u32 opc = i >> 30; // 0 = 32-bit, 2 = 64-bit
|
|
const bool is_load = (i >> 22) & 1;
|
|
const u32 mode = (i >> 23) & 3; // 1 post-index, 2 signed offset, 3 pre-index
|
|
const u32 rt2 = (i >> 10) & 31, rn = (i >> 5) & 31, rt = i & 31;
|
|
s32 imm7 = (s32)((i >> 15) & 0x7F);
|
|
if (imm7 & 0x40) imm7 |= ~0x7F; // sign-extend 7 bits
|
|
if ((opc == 0 || opc == 2) && mode >= 1 && mode <= 3) {
|
|
const u32 sz = (opc == 2) ? 8 : 4;
|
|
const s64 off = (s64)imm7 * sz;
|
|
std::string s = "{ uint64_t _b=c->x[" + std::to_string(rn) + "]; ";
|
|
// Pre-index and post-index both write the new base back; only
|
|
// pre-index applies the offset before the access.
|
|
const char* addr = (mode == 1) ? "_b" : "(_b+_o)";
|
|
s += "int64_t _o=" + std::to_string((long long)off) + "; ";
|
|
const std::string w = std::to_string(sz * 8);
|
|
if (is_load) {
|
|
// Both words come back through temporaries. Rt/Rt2 == 31 is XZR
|
|
// here, so that half is simply not written back - writing it
|
|
// would land on c->x[31], which is where SP lives. The load
|
|
// itself still happens, which is what the architecture does.
|
|
s += "{ uint64_t _p0,_p1; recomp_ldp" + w + "(c," + addr + ",&_p0,&_p1); ";
|
|
if (rt != 31) s += "c->x[" + std::to_string(rt) + "]=_p0; ";
|
|
if (rt2 != 31) s += "c->x[" + std::to_string(rt2) + "]=_p1; ";
|
|
s += "} ";
|
|
} else {
|
|
s += "recomp_stp" + w + "(c," + addr + "," +
|
|
(rt == 31 ? std::string("(uint64_t)0") : ("c->x[" + std::to_string(rt) + "]")) +
|
|
"," +
|
|
(rt2 == 31 ? std::string("(uint64_t)0") : ("c->x[" + std::to_string(rt2) + "]")) +
|
|
"); ";
|
|
}
|
|
if (mode == 1 || mode == 3) {
|
|
s += "c->x[" + std::to_string(rn) + "]=_b+_o; ";
|
|
}
|
|
s += "}";
|
|
put(s);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// Logical immediate: AND/ORR/EOR/ANDS. By far the largest single gap in
|
|
// real code - the immediate is a repeating bit pattern, not a literal.
|
|
if ((i & 0x1F800000) == 0x12000000) {
|
|
const u32 sf = i >> 31, opc = (i >> 29) & 3, N = (i >> 22) & 1;
|
|
const u32 immr = (i >> 16) & 0x3F, imms = (i >> 10) & 0x3F;
|
|
const u32 rn = (i >> 5) & 31, rd = i & 31;
|
|
u64 imm = 0;
|
|
if (DecodeBitMasks(N, imms, immr, sf != 0, imm)) {
|
|
const char* op = (opc == 0 || opc == 3) ? "&" : (opc == 1 ? "|" : "^");
|
|
std::string s = "{ uint64_t _r = " + Xz(rn) + " " + op + " 0x" ;
|
|
char hb[32]; snprintf(hb, sizeof hb, "%llxULL", (unsigned long long)imm);
|
|
s += hb; s += "; ";
|
|
if (!sf) s += "_r &= 0xFFFFFFFFULL; ";
|
|
// Rd==31 is SP for AND/ORR/EOR immediate and only reads as XZR
|
|
// for ANDS (opc==3). Treating it as XZR everywhere silently
|
|
// dropped every "and sp, xN, #imm" stack realignment.
|
|
if (!(rd == 31 && opc == 3)) s += "c->x[" + std::to_string(rd) + "] = _r; ";
|
|
if (opc == 3) s += SetFlags(false, "_r", "0", "_r", sf != 0) + " ";
|
|
s += "}";
|
|
put(s);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// Bitfield: UBFM/SBFM/BFM - the encoding behind LSL/LSR/ASR/UBFX/SBFX.
|
|
if ((i & 0x1F800000) == 0x13000000) {
|
|
const u32 sf = i >> 31, opc = (i >> 29) & 3;
|
|
const u32 immr = (i >> 16) & 0x3F, imms = (i >> 10) & 0x3F;
|
|
const u32 rn = (i >> 5) & 31, rd = i & 31;
|
|
const u32 width = sf ? 64 : 32;
|
|
|
|
// BFM (opc 1) - the encoding behind BFI and BFXIL. Unlike UBFM/SBFM it
|
|
// merges into the destination, leaving the bits outside the inserted
|
|
// field untouched, so it needs an explicit mask rather than a shift.
|
|
if (rd != 31 && opc == 1 && immr < width && imms < width) {
|
|
u32 nbits, src_shift, dst_pos;
|
|
if (imms >= immr) { // BFXIL: extract to the bottom of Rd
|
|
nbits = imms - immr + 1;
|
|
src_shift = immr;
|
|
dst_pos = 0;
|
|
} else { // BFI: insert at (width - immr)
|
|
nbits = imms + 1;
|
|
src_shift = 0;
|
|
dst_pos = width - immr;
|
|
}
|
|
if (nbits + dst_pos <= width) {
|
|
// Build the field mask at 64 bits; nbits is < 64 here because
|
|
// dst_pos + nbits <= width <= 64 and a 64-wide field would
|
|
// make the shift below undefined.
|
|
const u64 field = (nbits >= 64) ? ~0ULL : ((1ULL << nbits) - 1ULL);
|
|
const u64 mask = field << dst_pos;
|
|
char mb[48], fb[48];
|
|
snprintf(mb, sizeof mb, "0x%llxULL", (unsigned long long)mask);
|
|
snprintf(fb, sizeof fb, "0x%llxULL", (unsigned long long)field);
|
|
std::string s = "{ uint64_t _s = " + Xz(rn);
|
|
if (src_shift) s += " >> " + std::to_string(src_shift);
|
|
s += "; uint64_t _r = (c->x[" + std::to_string(rd) + "] & ~" + mb +
|
|
") | ((_s & " + fb + ") << " + std::to_string(dst_pos) + "); ";
|
|
if (!sf) s += "_r &= 0xFFFFFFFFULL; ";
|
|
s += "c->x[" + std::to_string(rd) + "] = _r; }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// UBFM (opc 2) and SBFM (opc 0).
|
|
if (rd != 31 && opc != 1 && immr < width && imms < width) {
|
|
const char* mask = sf ? "" : " & 0xFFFFFFFFULL";
|
|
std::string src = Xz(rn);
|
|
std::string s = "{ uint64_t _s = " + src + mask + "; uint64_t _r; ";
|
|
if (imms >= immr) {
|
|
// Extract (imms-immr+1) bits starting at immr.
|
|
const u32 nbits = imms - immr + 1;
|
|
s += "_r = (_s >> " + std::to_string(immr) + ")";
|
|
if (nbits < 64) s += " & ((1ULL << " + std::to_string(nbits) + ") - 1)";
|
|
s += "; ";
|
|
if (opc == 0 && nbits < 64) { // SBFM: sign-extend from nbits
|
|
s += "if (_r & (1ULL << " + std::to_string(nbits - 1) + ")) _r |= ~((1ULL << " +
|
|
std::to_string(nbits) + ") - 1); ";
|
|
}
|
|
} else {
|
|
// Insert: bits [0..imms] moved to start at (width-immr).
|
|
const u32 nbits = imms + 1;
|
|
const u32 shift = width - immr;
|
|
s += "_r = (_s";
|
|
if (nbits < 64) s += " & ((1ULL << " + std::to_string(nbits) + ") - 1)";
|
|
s += ") << " + std::to_string(shift) + "; ";
|
|
if (opc == 0 && shift + nbits < 64) {
|
|
s += "if (_r & (1ULL << " + std::to_string(shift + nbits - 1) +
|
|
")) _r |= ~((1ULL << " + std::to_string(shift + nbits) + ") - 1); ";
|
|
}
|
|
}
|
|
if (!sf) s += "_r &= 0xFFFFFFFFULL; ";
|
|
s += "c->x[" + std::to_string(rd) + "] = _r; }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// Conditional compare: CCMP/CCMN, both the register and 5-bit immediate
|
|
// forms. When the condition holds this behaves as an ordinary compare;
|
|
// when it does not, the flags are loaded straight from the nzcv field.
|
|
// Short-circuit conditionals in C compile to chains of these.
|
|
if ((i & 0x1FE00000) == 0x1A400000 && ((i >> 29) & 1) && ((i >> 10) & 1) == 0 &&
|
|
((i >> 4) & 1) == 0) {
|
|
const u32 sf = i >> 31, op = (i >> 30) & 1; // op: 0 = CCMN, 1 = CCMP
|
|
const u32 imm_or_rm = (i >> 16) & 31, cond = (i >> 12) & 15;
|
|
const bool is_imm = ((i >> 11) & 1) != 0;
|
|
const u32 rn = (i >> 5) & 31, nzcv = i & 15;
|
|
const std::string b = is_imm ? (std::to_string(imm_or_rm) + "ULL") : Xz(imm_or_rm);
|
|
std::string s = "{ if " + Cond(cond) + " { ";
|
|
s += "uint64_t _a=" + Xz(rn) + ", _b=" + b + ", _r=" +
|
|
std::string(op ? "_a-_b" : "_a+_b") + "; ";
|
|
if (!sf) s += "_r &= 0xFFFFFFFFULL; ";
|
|
s += SetFlags(op != 0, "_a", "_b", "_r", sf != 0) + " ";
|
|
s += "} else { ";
|
|
s += "c->n=" + std::to_string((nzcv >> 3) & 1) + "; ";
|
|
s += "c->z=" + std::to_string((nzcv >> 2) & 1) + "; ";
|
|
s += "c->c=" + std::to_string((nzcv >> 1) & 1) + "; ";
|
|
s += "c->v=" + std::to_string(nzcv & 1) + "; } }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
|
|
// Conditional select: CSEL/CSINC/CSINV/CSNEG.
|
|
// Bit 11 must be clear: with it set this encoding is not a conditional
|
|
// select at all, and decoding it as one silently invents an instruction.
|
|
if ((i & 0x1FE00800) == 0x1A800000) {
|
|
const u32 sf = i >> 31, op = (i >> 30) & 1, o2 = (i >> 10) & 1;
|
|
const u32 rm = (i >> 16) & 31, cond = (i >> 12) & 15, rn = (i >> 5) & 31, rd = i & 31;
|
|
if (rd != 31) {
|
|
std::string a = Xz(rn), b = Xz(rm);
|
|
std::string els = b;
|
|
if (!op && o2) els = "(" + b + " + 1)"; // CSINC
|
|
else if (op && !o2) els = "(~" + b + ")"; // CSINV
|
|
else if (op && o2) els = "((uint64_t)(0 - " + b + "))"; // CSNEG
|
|
std::string s = "{ uint64_t _r = " + Cond(cond) + " ? " +
|
|
a + " : " + els + "; ";
|
|
if (!sf) s += "_r &= 0xFFFFFFFFULL; ";
|
|
s += "c->x[" + std::to_string(rd) + "] = _r; }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// Load/store with unscaled 9-bit signed offset (LDUR/STUR) and the
|
|
// pre/post-indexed immediate forms.
|
|
// Again bit 26 (V) must be clear - the SIMD/FP form is handled below.
|
|
// Bit 21 set with mode==0 is the ARMv8.1 LSE atomic group, not LDUR/STUR;
|
|
// decoding those here would read a garbage imm9 out of the register field.
|
|
if ((i & 0x3F000000) == 0x38000000 && ((i >> 24) & 1) == 0 && ((i >> 21) & 1) == 0) {
|
|
const u32 size = i >> 30, opc = (i >> 22) & 3, mode = (i >> 10) & 3;
|
|
const u32 rn = (i >> 5) & 31, rt = i & 31;
|
|
s32 imm9 = (s32)((i >> 12) & 0x1FF);
|
|
if (imm9 & 0x100) imm9 |= ~0x1FF;
|
|
// mode 0 = LDUR/STUR, 1 = post-index, 3 = pre-index
|
|
if ((mode == 0 || mode == 1 || mode == 3) && size <= 3) {
|
|
const u32 bits = 8u << size;
|
|
std::string s = "{ uint64_t _b=c->x[" + std::to_string(rn) + "]; int64_t _o=" +
|
|
std::to_string((long long)imm9) + "; ";
|
|
const char* addr = (mode == 1) ? "_b" : "(_b+_o)";
|
|
// opc selects the operation, and testing only its low bit gets this
|
|
// wrong in both directions - the same mistake the unsigned-offset
|
|
// handler already documents, never applied here. opc==2 (LDURSW /
|
|
// LDURSB / LDURSH, sign-extending into a 64-bit destination) has
|
|
// bit 0 clear and so was emitted as a STORE, turning tens of
|
|
// thousands of ordinary signed loads into wild writes over live
|
|
// guest memory; opc==3 loaded but never sign-extended.
|
|
const char* signed_cast = size == 0 ? "(int8_t)"
|
|
: size == 1 ? "(int16_t)"
|
|
: "(int32_t)";
|
|
if (opc == 0) {
|
|
s += "recomp_store" + std::to_string(bits) + "(c," + addr + "," + Xz(rt) + "); ";
|
|
} else if (opc == 1) {
|
|
if (rt != 31) {
|
|
s += "c->x[" + std::to_string(rt) + "]=recomp_load" + std::to_string(bits) +
|
|
"(c," + addr + "); ";
|
|
}
|
|
} else if (size == 3) {
|
|
// opc>=2 with size==3 is PRFUM, a prefetch hint. It has no
|
|
// architectural effect, so emitting nothing is exact - what it
|
|
// must never do is store.
|
|
} else if (rt != 31) {
|
|
s += "c->x[" + std::to_string(rt) + "]=(uint64_t)(int64_t)" + signed_cast +
|
|
"recomp_load" + std::to_string(bits) + "(c," + addr + "); ";
|
|
// opc==3 sign-extends into a 32-bit destination, so the result
|
|
// is truncated back to W width after the extension.
|
|
if (opc == 3) s += "c->x[" + std::to_string(rt) + "]&=0xFFFFFFFFULL; ";
|
|
}
|
|
if (mode == 1 || mode == 3) s += "c->x[" + std::to_string(rn) + "]=_b+_o; ";
|
|
s += "}";
|
|
put(s);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// Load/store exclusive and acquire/release.
|
|
//
|
|
// The non-exclusive acquire/release forms (LDAR/STLR, o2 set) are ordinary
|
|
// loads and stores as far as this backend is concerned - suyu's memory
|
|
// accessors are already atomic at these widths, and there is no weaker
|
|
// ordering here to fence against - so they are translated directly.
|
|
//
|
|
// The genuinely exclusive forms (LDXR/LDAXR/STXR/STLXR, o2 clear) are not.
|
|
// They used to become a plain load/store with STXR unconditionally
|
|
// reporting success, which is exact only when nothing else can touch the
|
|
// address. Under this backend real guest threads run concurrently, so an
|
|
// always-succeeds STXR makes every compare-and-swap non-atomic: two
|
|
// threads both "win" the same lock, the data it protects is then updated
|
|
// from both, and the next thread to wait on it spins forever. Hand these
|
|
// to the fallback engine, which owns the kernel's real exclusive monitor.
|
|
if ((i & 0x3F000000) == 0x08000000) {
|
|
const u32 size = i >> 30, o2 = (i >> 23) & 1, L = (i >> 22) & 1, o1 = (i >> 21) & 1;
|
|
const u32 rt2 = (i >> 10) & 31, rn = (i >> 5) & 31, rt = i & 31;
|
|
// Pair forms: LDXP/LDAXP/STXP/STLXP. size 00/01 is unallocated here,
|
|
// so only the 32- and 64-bit register widths exist.
|
|
//
|
|
// These carry the lock traffic. A threaded engine implements its mutex
|
|
// as a 128-bit compare-and-swap over {owner, count} or {ptr, tag}, and
|
|
// on a second title the pair forms alone were 73.5% of every
|
|
// transition to the fallback engine once cntpct_el0 was implemented.
|
|
if (o1 && !o2 && (size == 2 || size == 3)) {
|
|
const u32 rbytes = (size == 3) ? 8u : 4u;
|
|
const std::string addr = "c->x[" + std::to_string(rn) + "]";
|
|
const std::string sz = std::to_string(rbytes);
|
|
if (L) {
|
|
// LDXP/LDAXP. Rt takes the low half, Rt2 the high half - the
|
|
// same order DynarmicExclusiveMonitor::ExclusiveRead128 fills,
|
|
// and for the 32-bit form the low and high words of the
|
|
// doubleword at the address.
|
|
std::string s2 = "{ uint64_t _lo,_hi; recomp_ldxp(c," + addr + "," + sz +
|
|
",&_lo,&_hi); ";
|
|
// A W-register destination is written zero-extended.
|
|
const char* cast = (rbytes == 4) ? "(uint32_t)" : "";
|
|
if (rt != 31) {
|
|
s2 += "c->x[" + std::to_string(rt) + "] = " + cast + "_lo; ";
|
|
}
|
|
if (rt2 != 31) {
|
|
s2 += "c->x[" + std::to_string(rt2) + "] = " + cast + "_hi; ";
|
|
}
|
|
s2 += "}";
|
|
put(s2);
|
|
} else {
|
|
// STXP/STLXP. Rs (bits 20:16) receives the status, 0 on success.
|
|
const u32 rs = (i >> 16) & 31;
|
|
const std::string lo =
|
|
(rbytes == 4) ? ("(uint64_t)(uint32_t)" + Xz(rt)) : Xz(rt);
|
|
const std::string hi =
|
|
(rbytes == 4) ? ("(uint64_t)(uint32_t)" + Xz(rt2)) : Xz(rt2);
|
|
std::string s2 = "{ uint32_t _st = recomp_stxp(c," + addr + "," + sz + "," +
|
|
lo + "," + hi + "); ";
|
|
if (rs != 31) {
|
|
s2 += "c->x[" + std::to_string(rs) + "] = _st; ";
|
|
}
|
|
s2 += "}";
|
|
put(s2);
|
|
}
|
|
return true;
|
|
}
|
|
if (!o1 && rt2 == 31) {
|
|
const u32 bits = 8u << size;
|
|
if (!o2) {
|
|
// LDXR/LDAXR and STXR/STLXR. Both route through the emulator's
|
|
// own exclusive monitor, so an exclusive taken here is visible
|
|
// to a thread running on the fallback JIT.
|
|
//
|
|
// Acquire/release ordering (the LDAXR/STLXR spelling, o0 set)
|
|
// is not modelled separately: suyu's memory accessors are
|
|
// already atomic at these widths and the host is x86-64, whose
|
|
// ordering is strong enough that the fence these imply is a
|
|
// no-op in practice. Noted rather than silently assumed.
|
|
const std::string addr = "c->x[" + std::to_string(rn) + "]";
|
|
if (L) {
|
|
// LDXR/LDAXR: mark and load.
|
|
if (rt != 31) {
|
|
put("c->x[" + std::to_string(rt) + "] = recomp_ldxr(c," + addr + "," +
|
|
std::to_string(bits / 8) + ");");
|
|
} else {
|
|
put("(void)recomp_ldxr(c," + addr + "," + std::to_string(bits / 8) + ");");
|
|
}
|
|
} else {
|
|
// STXR/STLXR: Rs receives the status, 0 on success. Rs is
|
|
// held in the Rt2 field's neighbour (bits 20:16).
|
|
const u32 rs = (i >> 16) & 31;
|
|
std::string s2 = "{ uint32_t _st = recomp_stxr(c," + addr + "," +
|
|
std::to_string(bits / 8) + "," + Xz(rt) + "); ";
|
|
if (rs != 31) {
|
|
s2 += "c->x[" + std::to_string(rs) + "] = _st; ";
|
|
}
|
|
s2 += "}";
|
|
put(s2);
|
|
}
|
|
return true;
|
|
}
|
|
const std::string addr = "c->x[" + std::to_string(rn) + "]";
|
|
if (L) {
|
|
// LDAR
|
|
if (rt != 31) {
|
|
put("c->x[" + std::to_string(rt) + "] = recomp_load" +
|
|
std::to_string(bits) + "(c," + addr + ");");
|
|
}
|
|
} else {
|
|
// STLR - no status register.
|
|
put("recomp_store" + std::to_string(bits) + "(c," + addr + "," + Xz(rt) + ");");
|
|
}
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// Data-processing 3-source: MADD/MSUB and the widening multiplies.
|
|
if ((i & 0x1F000000) == 0x1B000000) {
|
|
const u32 sf = i >> 31, op54 = (i >> 29) & 3, op31 = (i >> 21) & 7, o0 = (i >> 15) & 1;
|
|
const u32 rm = (i >> 16) & 31, ra = (i >> 10) & 31, rn = (i >> 5) & 31, rd = i & 31;
|
|
if (rd != 31 && op54 == 0) {
|
|
std::string expr;
|
|
if (op31 == 0) {
|
|
// MADD / MSUB
|
|
const std::string prod = "(" + Xz(rn) + " * " + Xz(rm) + ")";
|
|
expr = Xz(ra) + (o0 ? " - " : " + ") + prod;
|
|
} else if (op31 == 1 && !o0) { // SMADDL
|
|
expr = Xz(ra) + " + (uint64_t)((int64_t)(int32_t)" + Xz(rn) +
|
|
" * (int64_t)(int32_t)" + Xz(rm) + ")";
|
|
} else if (op31 == 1 && o0) { // SMSUBL
|
|
expr = Xz(ra) + " - (uint64_t)((int64_t)(int32_t)" + Xz(rn) +
|
|
" * (int64_t)(int32_t)" + Xz(rm) + ")";
|
|
} else if (op31 == 5 && !o0) { // UMADDL
|
|
expr = Xz(ra) + " + ((uint64_t)(uint32_t)" + Xz(rn) +
|
|
" * (uint64_t)(uint32_t)" + Xz(rm) + ")";
|
|
} else if (op31 == 5 && o0) { // UMSUBL
|
|
expr = Xz(ra) + " - ((uint64_t)(uint32_t)" + Xz(rn) +
|
|
" * (uint64_t)(uint32_t)" + Xz(rm) + ")";
|
|
} else if (op31 == 2) { // SMULH
|
|
expr = "recomp_smulh(" + Xz(rn) + "," + Xz(rm) + ")";
|
|
} else if (op31 == 6) { // UMULH
|
|
expr = "recomp_umulh(" + Xz(rn) + "," + Xz(rm) + ")";
|
|
}
|
|
if (!expr.empty()) {
|
|
std::string s = "{ uint64_t _r = " + expr + "; ";
|
|
if (!sf && op31 == 0) s += "_r &= 0xFFFFFFFFULL; ";
|
|
s += "c->x[" + std::to_string(rd) + "] = _r; }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
}
|
|
}
|
|
|
|
// ADD/SUB extended register - the form used for pointer arithmetic with a
|
|
// 32-bit index, so extremely common around array and struct accesses.
|
|
if ((i & 0x1F200000) == 0x0B200000) {
|
|
const u32 sf = i >> 31, op = (i >> 30) & 1, S = (i >> 29) & 1;
|
|
const u32 rm = (i >> 16) & 31, option = (i >> 13) & 7, imm3 = (i >> 10) & 7;
|
|
const u32 rn = (i >> 5) & 31, rd = i & 31;
|
|
if (imm3 <= 4) {
|
|
std::string ext;
|
|
switch (option) {
|
|
case 0: ext = "(uint64_t)(uint8_t)" + Xz(rm); break; // UXTB
|
|
case 1: ext = "(uint64_t)(uint16_t)" + Xz(rm); break; // UXTH
|
|
case 2: ext = "(uint64_t)(uint32_t)" + Xz(rm); break; // UXTW
|
|
case 3: ext = Xz(rm); break; // UXTX/LSL
|
|
case 4: ext = "(uint64_t)(int64_t)(int8_t)" + Xz(rm); break; // SXTB
|
|
case 5: ext = "(uint64_t)(int64_t)(int16_t)" + Xz(rm); break; // SXTH
|
|
case 6: ext = "(uint64_t)(int64_t)(int32_t)" + Xz(rm); break; // SXTW
|
|
case 7: ext = Xz(rm); break; // SXTX
|
|
default: break;
|
|
}
|
|
if (!ext.empty()) {
|
|
if (imm3) ext = "((" + ext + ") << " + std::to_string(imm3) + ")";
|
|
// Rn is SP here, not the zero register, and so is Rd unless the
|
|
// flag-setting form is used - where it really is the zero
|
|
// register, because that is CMP.
|
|
std::string s = "{ uint64_t _a=" + Xsp(rn) + ", _b=" + ext + "; uint64_t _r=" +
|
|
std::string(op ? "_a-_b" : "_a+_b") + "; ";
|
|
if (!sf) s += "_r &= 0xFFFFFFFFULL; ";
|
|
if (rd != 31 || !S) s += "c->x[" + std::to_string(rd) + "]=_r; ";
|
|
if (S) s += SetFlags(op != 0, "_a", "_b", "_r", sf != 0) + " ";
|
|
s += "}";
|
|
put(s);
|
|
return true;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Load/store with a register offset (the [base, Xm{, extend}] form).
|
|
// Bit 26 (V) clear: general registers only, SIMD/FP form handled below.
|
|
if ((i & 0x3F200C00) == 0x38200800) {
|
|
const u32 size = i >> 30, opc = (i >> 22) & 3;
|
|
const u32 rm = (i >> 16) & 31, option = (i >> 13) & 7, S = (i >> 12) & 1;
|
|
const u32 rn = (i >> 5) & 31, rt = i & 31;
|
|
// opc 00 stores, 01 loads zero-extended, 10 loads sign-extended to 64
|
|
// bits and 11 sign-extended to 32. The signed forms are common - an
|
|
// indexed read of an int32 array compiles to LDRSW with a register
|
|
// offset - so leaving them out put thousands of real loads on the
|
|
// fallback.
|
|
if (size <= 3 && !(opc >= 2 && size == 3)) {
|
|
const u32 bits = 8u << size;
|
|
std::string idx;
|
|
switch (option) {
|
|
case 2: idx = "(uint64_t)(uint32_t)" + Xz(rm); break; // UXTW
|
|
case 3: idx = Xz(rm); break; // LSL
|
|
case 6: idx = "(uint64_t)(int64_t)(int32_t)" + Xz(rm); break; // SXTW
|
|
case 7: idx = Xz(rm); break; // SXTX
|
|
default: break;
|
|
}
|
|
if (!idx.empty()) {
|
|
if (S && size) idx = "((" + idx + ") << " + std::to_string(size) + ")";
|
|
const std::string addr = "(c->x[" + std::to_string(rn) + "] + " + idx + ")";
|
|
if (opc == 0) {
|
|
put("recomp_store" + std::to_string(bits) + "(c," + addr + "," + Xz(rt) + ");");
|
|
} else if (opc == 1) {
|
|
if (rt != 31) {
|
|
put("c->x[" + std::to_string(rt) + "] = recomp_load" +
|
|
std::to_string(bits) + "(c," + addr + ");");
|
|
}
|
|
} else if (rt != 31) {
|
|
const char* st = size == 0 ? "int8_t" : size == 1 ? "int16_t" : "int32_t";
|
|
std::string s = "{ uint64_t _r = (uint64_t)(int64_t)(" + std::string(st) +
|
|
")recomp_load" + std::to_string(bits) + "(c," + addr + "); ";
|
|
if (opc == 3) s += "_r &= 0xFFFFFFFFULL; "; // 32-bit destination
|
|
s += "c->x[" + std::to_string(rt) + "] = _r; }";
|
|
put(s);
|
|
}
|
|
return true;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Non-saturating immediate shifts. Unsigned intermediates implement sign
|
|
// extension and rounding without signed overflow or a shift by 64.
|
|
if ((i & 0x8F800400) == 0x0F000400) {
|
|
const u32 scalar = (i >> 28) & 1, Q = (i >> 30) & 1, U = (i >> 29) & 1;
|
|
const u32 opcode = (i >> 11) & 31, imm = (i >> 16) & 127;
|
|
const u32 rn = (i >> 5) & 31, rd = i & 31;
|
|
const bool narrow = opcode == 16 || opcode == 17;
|
|
const bool left = opcode == 10;
|
|
const bool insert = opcode == 8 || (left && U);
|
|
const bool round = opcode == 4 || opcode == 6 || opcode == 17;
|
|
const bool acc = opcode == 2 || opcode == 6;
|
|
const bool known = opcode == 0 || opcode == 2 || opcode == 4 || opcode == 6 ||
|
|
(opcode == 8 && U) || left || (narrow && !U);
|
|
u32 bits = 8;
|
|
while (bits < 64 && imm >= bits * 2) bits *= 2;
|
|
// SHL (left, !U) is the one form g_translate_all gates; every export
|
|
// sets it (kHybridTranslatesGatedForms). The bare emitter keeps it off.
|
|
if (known && (!left || U || g_translate_all) && imm >= 8 &&
|
|
(!scalar || (Q && !narrow && bits == 64)) &&
|
|
(scalar || narrow || bits != 64 || Q) && (!narrow || bits < 64)) {
|
|
const u32 sbits = narrow ? bits * 2 : bits;
|
|
const u32 shift = left ? imm - bits : bits * 2 - imm;
|
|
const u32 bytes = scalar ? 8 : Q ? 16 : 8;
|
|
const u32 lanes = narrow ? 64 / bits : bytes * 8 / bits;
|
|
const std::string st = "uint" + std::to_string(sbits) + "_t";
|
|
const std::string dt = "uint" + std::to_string(bits) + "_t";
|
|
std::string s = "{ " + st + " _a[" + std::to_string(lanes) + "]; " + dt +
|
|
" _d[" + std::to_string(lanes) + "],_r[" + std::to_string(lanes) + "]; ";
|
|
s += "memcpy(_a,c->vreg[" + std::to_string(rn) + "]," +
|
|
std::to_string(narrow ? 16 : bytes) + "); ";
|
|
s += "memcpy(_d,c->vreg[" + std::to_string(rd) + "]," +
|
|
std::to_string(narrow ? 8 : bytes) + "); ";
|
|
s += "for(int _i=0;_i<" + std::to_string(lanes) + ";++_i) { uint64_t _v=_a[_i],_z; ";
|
|
if (left) {
|
|
s += "_z=_v<<" + std::to_string(shift) + "; ";
|
|
if (insert) s += "_z|=(uint64_t)_d[_i]&" +
|
|
std::to_string(shift ? (uint64_t{1} << shift) - 1 : 0) + "ULL; ";
|
|
} else {
|
|
s += shift == 64 ? "_z=0; " : "_z=_v>>" + std::to_string(shift) + "; ";
|
|
if (!U && !narrow) {
|
|
const uint64_t mask = shift == 64 ? ~uint64_t{0} :
|
|
(~uint64_t{0} << (bits - shift));
|
|
s += "if(_v&(1ULL<<" + std::to_string(bits - 1) + ")) _z|=" +
|
|
std::to_string(mask) + "ULL; ";
|
|
}
|
|
if (round) s += "_z+=(_v>>" + std::to_string(shift - 1) + ")&1; ";
|
|
if (acc) s += "_z+=(uint64_t)_d[_i]; ";
|
|
if (insert) {
|
|
const uint64_t mask = shift == bits ? ~uint64_t{0} :
|
|
(~uint64_t{0} << (bits - shift));
|
|
s += "_z|=(uint64_t)_d[_i]&" + std::to_string(mask) + "ULL; ";
|
|
}
|
|
}
|
|
s += "_r[_i]=(" + dt + ")_z; } ";
|
|
if (!narrow || !Q) s += "c->vreg[" + std::to_string(rd) + "][0]=0; c->vreg[" +
|
|
std::to_string(rd) + "][1]=0; ";
|
|
s += "memcpy((uint8_t*)c->vreg[" + std::to_string(rd) + "]+" +
|
|
std::to_string(narrow && Q ? 8 : 0) + ",_r," +
|
|
std::to_string(narrow ? 8 : bytes) + "); }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// SHLL widens and shifts by exactly the source width (not SSHLL's range).
|
|
if ((i & 0xBF3FFC00) == 0x2E213800 && ((i >> 22) & 3) != 3) {
|
|
const u32 Q = (i >> 30) & 1, bits = 8u << ((i >> 22) & 3);
|
|
const u32 rn = (i >> 5) & 31, rd = i & 31, lanes = 64 / bits;
|
|
const std::string st = "uint" + std::to_string(bits) + "_t";
|
|
const std::string dt = "uint" + std::to_string(bits * 2) + "_t";
|
|
put("{ " + st + " _a[" + std::to_string(lanes) + "]; " + dt + " _r[" +
|
|
std::to_string(lanes) + "]; memcpy(_a,(uint8_t*)c->vreg[" + std::to_string(rn) +
|
|
"]+" + std::to_string(Q ? 8 : 0) + ",8); for(int _i=0;_i<" + std::to_string(lanes) +
|
|
";++_i) _r[_i]=(" + dt + ")((uint64_t)_a[_i]<<" + std::to_string(bits) +
|
|
"); memcpy(c->vreg[" + std::to_string(rd) + "],_r,16); }");
|
|
return true;
|
|
}
|
|
|
|
// SSHR / USHR / SHRN: shift right by an immediate. SSHR and USHR keep the
|
|
// element width; SHRN halves it and writes one half of the destination.
|
|
if ((i & 0x9F80FC00) == 0x0F000400 || (i & 0x9F80FC00) == 0x0F008400) {
|
|
const bool narrow = ((i >> 10) & 0x3F) == 0x21;
|
|
const u32 Q = (i >> 30) & 1, U = (i >> 29) & 1;
|
|
const u32 immh = (i >> 19) & 15, immb = (i >> 16) & 7;
|
|
const u32 rn = (i >> 5) & 31, rd = i & 31;
|
|
u32 size = 0;
|
|
bool shaped = true;
|
|
if (immh & 8) size = 3;
|
|
else if (immh & 4) size = 2;
|
|
else if (immh & 2) size = 1;
|
|
else if (immh & 1) size = 0;
|
|
else shaped = false;
|
|
if (narrow && (size == 3 || U)) shaped = false; // no 128-bit source element
|
|
if (shaped) {
|
|
// The shift is relative to the *destination* element in both
|
|
// cases; for SHRN the source is twice that, which is the whole
|
|
// point of the instruction.
|
|
const int dbits = 8 << size;
|
|
const int sbits = narrow ? (dbits * 2) : dbits;
|
|
const u32 shift = (u32)(dbits * 2) - ((immh << 3) | immb);
|
|
if (shift >= 1 && shift <= (u32)dbits) {
|
|
const int ssz = sbits / 8;
|
|
if (narrow) {
|
|
const int lanes = 8 / (ssz / 2);
|
|
const std::string sty = "uint" + std::to_string(sbits) + "_t";
|
|
const std::string dty = "uint" + std::to_string(sbits / 2) + "_t";
|
|
std::string s = "{ " + sty + " _a[" + std::to_string(lanes) + "]; " + dty +
|
|
" _r[" + std::to_string(lanes) + "]; ";
|
|
s += "memcpy(_a,c->vreg[" + std::to_string(rn) + "],16); ";
|
|
s += "for(int _i=0;_i<" + std::to_string(lanes) + ";_i++) _r[_i]=(" + dty +
|
|
")(_a[_i]>>" + std::to_string(shift) + "); ";
|
|
if (!Q) {
|
|
s += "c->vreg[" + std::to_string(rd) + "][0]=0; c->vreg[" +
|
|
std::to_string(rd) + "][1]=0; ";
|
|
}
|
|
s += "memcpy((uint8_t*)c->vreg[" + std::to_string(rd) + "]+" +
|
|
std::to_string(Q ? 8 : 0) + ",_r,8); }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
const int bytes = Q ? 16 : 8;
|
|
if (!(size == 3 && !Q)) {
|
|
const int lanes = bytes / ssz;
|
|
const std::string uty = "uint" + std::to_string(sbits) + "_t";
|
|
const std::string ity = "int" + std::to_string(sbits) + "_t";
|
|
// A shift of the full width is defined here and undefined in
|
|
// C, so it is folded to the all-sign / all-zero result.
|
|
const bool full = shift == (u32)sbits;
|
|
std::string s = "{ " + uty + " _a[" + std::to_string(lanes) + "],_r[" +
|
|
std::to_string(lanes) + "]; ";
|
|
s += "memcpy(_a,c->vreg[" + std::to_string(rn) + "]," + std::to_string(bytes) +
|
|
"); ";
|
|
s += "for(int _i=0;_i<" + std::to_string(lanes) + ";_i++) _r[_i]=";
|
|
if (U) {
|
|
s += full ? ("(" + uty + ")0")
|
|
: ("(" + uty + ")(_a[_i]>>" + std::to_string(shift) + ")");
|
|
} else {
|
|
s += "(" + uty + ")((" + ity + ")_a[_i]>>" +
|
|
std::to_string(full ? (u32)(sbits - 1) : shift) + ")";
|
|
}
|
|
s += "; ";
|
|
s += "c->vreg[" + std::to_string(rd) + "][0]=0; c->vreg[" +
|
|
std::to_string(rd) + "][1]=0; ";
|
|
s += "memcpy(c->vreg[" + std::to_string(rd) + "],_r," + std::to_string(bytes) +
|
|
"); }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// SMULL / UMULL and the accumulating forms. The sources are half-width, so
|
|
// Q picks which half of the source registers feeds the full-width result.
|
|
if ((i & 0x9F20FC00) == 0x0E208000 || (i & 0x9F20FC00) == 0x0E20A000 ||
|
|
(i & 0x9F20FC00) == 0x0E20C000) {
|
|
const u32 Q = (i >> 30) & 1, U = (i >> 29) & 1;
|
|
const u32 size = (i >> 22) & 3, opcode = (i >> 12) & 15;
|
|
const u32 rm = (i >> 16) & 31, rn = (i >> 5) & 31, rd = i & 31;
|
|
if (size != 3) {
|
|
const int sbits = 8 << size, dbits = sbits * 2;
|
|
const int lanes = 64 / sbits;
|
|
const std::string sty =
|
|
std::string(U ? "uint" : "int") + std::to_string(sbits) + "_t";
|
|
const std::string dty =
|
|
std::string(U ? "uint" : "int") + std::to_string(dbits) + "_t";
|
|
// The accumulator is held unsigned: the wrap is defined there, and
|
|
// signed overflow in the generated C would not be.
|
|
const std::string aty = "uint" + std::to_string(dbits) + "_t";
|
|
const int off = Q ? 8 : 0;
|
|
const char* acc = (opcode == 8) ? "+=" : (opcode == 0xA) ? "-=" : "=";
|
|
std::string s = "{ " + sty + " _a[" + std::to_string(lanes) + "],_b[" +
|
|
std::to_string(lanes) + "]; " + aty + " _r[" +
|
|
std::to_string(lanes) + "]; ";
|
|
s += "memcpy(_a,(const uint8_t*)c->vreg[" + std::to_string(rn) + "]+" +
|
|
std::to_string(off) + ",8); ";
|
|
s += "memcpy(_b,(const uint8_t*)c->vreg[" + std::to_string(rm) + "]+" +
|
|
std::to_string(off) + ",8); ";
|
|
if (opcode != 0xC) {
|
|
s += "memcpy(_r,c->vreg[" + std::to_string(rd) + "],16); ";
|
|
}
|
|
s += "for(int _i=0;_i<" + std::to_string(lanes) + ";_i++) _r[_i]" +
|
|
std::string(acc) + "(" + aty + ")((" + dty + ")_a[_i]*(" + dty + ")_b[_i]); ";
|
|
s += "memcpy(c->vreg[" + std::to_string(rd) + "],_r,16); }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// SQSHRN / SQRSHRN and the unsigned forms: shift right, round for the R
|
|
// variants, then saturate into the half-width destination.
|
|
if ((i & 0x8F80F400) == 0x0F009400 || (i & 0xAF80F400) == 0x2F008400) {
|
|
const bool round = ((i >> 11) & 1) != 0;
|
|
const bool scalar = ((i >> 28) & 1) != 0;
|
|
const bool to_unsigned = (i & 0x1000) == 0;
|
|
const u32 Q = (i >> 30) & 1, U = (i >> 29) & 1;
|
|
const u32 immh = (i >> 19) & 15, immb = (i >> 16) & 7;
|
|
const u32 rn = (i >> 5) & 31, rd = i & 31;
|
|
u32 size = 0;
|
|
bool shaped = true;
|
|
if (scalar && !Q) shaped = false;
|
|
if (immh & 8) shaped = false; // no 128-bit source element
|
|
else if (immh & 4) size = 2;
|
|
else if (immh & 2) size = 1;
|
|
else if (immh & 1) size = 0;
|
|
else shaped = false;
|
|
if (shaped) {
|
|
const int dbits = 8 << size, sbits = dbits * 2;
|
|
const u32 shift = (u32)dbits * 2 - ((immh << 3) | immb);
|
|
if (shift >= 1 && shift <= (u32)dbits) {
|
|
const int lanes = scalar ? 1 : 64 / dbits;
|
|
const int result_bytes = scalar ? dbits / 8 : 8;
|
|
const std::string sty =
|
|
std::string(U && !to_unsigned ? "uint" : "int") + std::to_string(sbits) + "_t";
|
|
const std::string dty = "uint" + std::to_string(dbits) + "_t";
|
|
const std::string wide = U && !to_unsigned ? "uint64_t" : "int64_t";
|
|
std::string s = "{ " + sty + " _a[" + std::to_string(lanes) + "]; " + dty +
|
|
" _r[" + std::to_string(lanes) + "]; ";
|
|
s += "memcpy(_a,c->vreg[" + std::to_string(rn) + "]," +
|
|
std::to_string(scalar ? sbits / 8 : 16) + "); ";
|
|
s += "for(int _i=0;_i<" + std::to_string(lanes) + ";_i++){ " + wide +
|
|
" _v=(" + wide + ")_a[_i]; ";
|
|
// Rounding as add-then-shift can overflow the source type; taking
|
|
// the dropped bit out of the shifted value cannot.
|
|
s += wide + " _x=(_v>>" + std::to_string(shift) + ")";
|
|
if (round) {
|
|
s += "+((_v>>" + std::to_string(shift - 1) + ")&1)";
|
|
}
|
|
s += "; ";
|
|
if (U) {
|
|
if (to_unsigned) s += "if(_x<0){_x=0;c->fpsr|=1ULL<<27;} ";
|
|
s += "if(_x>(" + wide + ")UINT" + std::to_string(dbits) + "_MAX){_x=(" + wide + ")UINT" +
|
|
std::to_string(dbits) + "_MAX;c->fpsr|=1ULL<<27;} ";
|
|
} else {
|
|
s += "if(_x>(int64_t)INT" + std::to_string(dbits) + "_MAX){_x=(int64_t)INT" +
|
|
std::to_string(dbits) + "_MAX;c->fpsr|=1ULL<<27;} ";
|
|
s += "else if(_x<(int64_t)INT" + std::to_string(dbits) +
|
|
"_MIN){_x=(int64_t)INT" + std::to_string(dbits) + "_MIN;c->fpsr|=1ULL<<27;} ";
|
|
}
|
|
s += "_r[_i]=(" + dty + ")_x; } ";
|
|
if (scalar || !Q) {
|
|
s += "c->vreg[" + std::to_string(rd) + "][0]=0; c->vreg[" +
|
|
std::to_string(rd) + "][1]=0; ";
|
|
}
|
|
s += "memcpy((uint8_t*)c->vreg[" + std::to_string(rd) + "]+" +
|
|
std::to_string(!scalar && Q ? 8 : 0) + ",_r," +
|
|
std::to_string(result_bytes) + "); }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
}
|
|
}
|
|
|
|
// CMGT / CMGE / CMHI / CMHS, register forms. Scalar forms only allow D.
|
|
if ((i & 0x9F20F400) == 0x0E203400 || (i & 0xDF20F400) == 0x5E203400) {
|
|
const bool scalar = (i & 0x10000000) != 0;
|
|
const bool Q = (i & 0x40000000) != 0, U = (i & 0x20000000) != 0;
|
|
const u32 size = (i >> 22) & 3;
|
|
if ((scalar && size == 3) || (!scalar && (size != 3 || Q))) {
|
|
const int bits = 8 << size, bytes = scalar ? 8 : (Q ? 16 : 8);
|
|
const int lanes = bytes * 8 / bits;
|
|
const u32 rm = (i >> 16) & 31, rn = (i >> 5) & 31, rd = i & 31;
|
|
const std::string ty = std::string(U ? "uint" : "int") + std::to_string(bits) + "_t";
|
|
const std::string uty = "uint" + std::to_string(bits) + "_t";
|
|
const std::string op = (i & 0x800) ? ">=" : ">";
|
|
std::string s = "{ " + ty + " _a[" + std::to_string(lanes) + "],_b[" +
|
|
std::to_string(lanes) + "]; " + uty + " _r[" + std::to_string(lanes) + "]; ";
|
|
s += "memcpy(_a,c->vreg[" + std::to_string(rn) + "]," + std::to_string(bytes) + "); ";
|
|
s += "memcpy(_b,c->vreg[" + std::to_string(rm) + "]," + std::to_string(bytes) + "); ";
|
|
s += "for(int _i=0;_i<" + std::to_string(lanes) + ";_i++) _r[_i]=(_a[_i]" +
|
|
op + "_b[_i])?(" + uty + ")~(" + uty + ")0:0; ";
|
|
s += "memset(c->vreg[" + std::to_string(rd) + "],0,16); memcpy(c->vreg[" +
|
|
std::to_string(rd) + "],_r," + std::to_string(bytes) + "); }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// Saturating variable shifts use only the signed low byte of each shift
|
|
// lane. Right shifts never saturate; rounding belongs to that branch only.
|
|
if ((i & 0x8F20EC00) == 0x0E204C00) {
|
|
const bool scalar = (i & 0x10000000) != 0, Q = (i & 0x40000000) != 0;
|
|
const bool U = (i & 0x20000000) != 0, round = (i & 0x1000) != 0;
|
|
const u32 size = (i >> 22) & 3, rn = (i >> 5) & 31, rd = i & 31, rm = (i >> 16) & 31;
|
|
if ((!scalar || Q) && (scalar || Q || size != 3)) {
|
|
const int bits = 8 << size, bytes = scalar ? bits / 8 : Q ? 16 : 8;
|
|
const int lanes = bytes * 8 / bits;
|
|
const std::string ty = "uint" + std::to_string(bits) + "_t";
|
|
const std::string n = std::to_string(bits);
|
|
std::string s = "{ " + ty + " _a[" + std::to_string(lanes) + "],_b[" +
|
|
std::to_string(lanes) + "],_r[" + std::to_string(lanes) + "]; ";
|
|
s += "memcpy(_a,c->vreg[" + std::to_string(rn) + "]," + std::to_string(bytes) +
|
|
");memcpy(_b,c->vreg[" + std::to_string(rm) + "]," + std::to_string(bytes) + "); ";
|
|
s += "for(int _i=0;_i<" + std::to_string(lanes) + ";++_i){uint64_t _x=_a[_i],_z;int _sh=(int)(_b[_i]&255);if(_sh>=128)_sh-=256; ";
|
|
s += "if(_sh>=0){ ";
|
|
if (U) {
|
|
s += "if(_x&&(_sh>=" + n + "||_x>((uint64_t)UINT" + n + "_MAX>>_sh))){_z=UINT" + n + "_MAX;c->fpsr|=UINT64_C(1)<<27;}else _z=_sh>=" + n + "?0:_x<<_sh; ";
|
|
} else {
|
|
s += "uint64_t _sign=UINT64_C(1)<<" + std::to_string(bits - 1) + ";int _neg=(_x&_sign)!=0;uint64_t _mag=_neg?(" + ty + ")((uint64_t)0-_x):_x,_lim=_neg?_sign:_sign-1; ";
|
|
s += "if(_mag&&(_sh>=" + n + "||_mag>(_lim>>_sh))){_z=_neg?_sign:_sign-1;c->fpsr|=UINT64_C(1)<<27;}else _z=_sh>=" + n + "?0:_x<<_sh; ";
|
|
}
|
|
s += "}else{unsigned _rsh=(unsigned)-_sh; ";
|
|
if (U) {
|
|
s += "_z=_rsh>=" + n + "?0:_x>>_rsh; ";
|
|
if (round) s += "if(_rsh<=" + n + ")_z+=(_x>>(_rsh-1))&1; ";
|
|
} else {
|
|
s += "int _neg=(_x&(UINT64_C(1)<<" + std::to_string(bits - 1) + "))!=0; ";
|
|
s += "if(_rsh>=" + n + ")_z=" + std::string(round ? "0" : "_neg?UINT" + n + "_MAX:0") + ";else{_z=_x>>_rsh;if(_neg)_z|=UINT" + n + "_MAX^(UINT" + n + "_MAX>>_rsh); ";
|
|
if (round) s += "_z+=(_x>>(_rsh-1))&1; ";
|
|
s += "} ";
|
|
}
|
|
s += "}_r[_i]=(" + ty + ")_z;}memset(c->vreg[" + std::to_string(rd) + "],0,16);memcpy(c->vreg[" + std::to_string(rd) + "],_r," + std::to_string(bytes) + ");}";
|
|
put(s);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// Saturating integer arithmetic. Carry/sign tests use unsigned values so
|
|
// even the 64-bit forms have no signed-overflow dependency. QC is sticky.
|
|
if ((i & 0x8F20DC00) == 0x0E200C00 ||
|
|
(i & 0x8F3FFC00) == 0x0E207800 ||
|
|
(i & 0x8F3FFC00) == 0x0E214800 ||
|
|
(i & 0xAF3FFC00) == 0x2E212800 ||
|
|
(i & 0x8F20FC00) == 0x0E20B400 ||
|
|
(i & 0xAF00E400) == 0x0F00C000) {
|
|
const bool scalar = (i & 0x10000000) != 0, Q = (i & 0x40000000) != 0;
|
|
const bool U = (i & 0x20000000) != 0;
|
|
const u32 size = (i >> 22) & 3, rn = (i >> 5) & 31, rd = i & 31;
|
|
const u32 rm = (i >> 16) & 31;
|
|
const bool binary = (i & 0x8F20DC00) == 0x0E200C00;
|
|
const bool unary = (i & 0x8F3FFC00) == 0x0E207800;
|
|
const bool indexed_mul = (i & 0xAF00E400) == 0x0F00C000;
|
|
const bool mul = (i & 0x8F20FC00) == 0x0E20B400 || indexed_mul;
|
|
const bool narrow = !binary && !unary && !mul;
|
|
const bool to_unsigned = narrow && (i & 0x2000) != 0;
|
|
if ((!scalar || Q) && (narrow ? size != 3 : mul ? (size == 1 || size == 2) :
|
|
(scalar || Q || size != 3))) {
|
|
const int bits = 8 << size, sbits = narrow ? bits * 2 : bits;
|
|
const int bytes = scalar ? bits / 8 : narrow ? 8 : Q ? 16 : 8;
|
|
const int lanes = bytes * 8 / bits;
|
|
const std::string ty = "uint" + std::to_string(bits) + "_t";
|
|
const std::string sty = "uint" + std::to_string(sbits) + "_t";
|
|
std::string s = "{ " + sty + " _a[" + std::to_string(lanes) + "]; " + ty +
|
|
" _r[" + std::to_string(lanes) + "]; ";
|
|
if (binary || mul) s += ty + " _b[" + std::to_string(lanes) + "]; ";
|
|
s += "memcpy(_a,c->vreg[" + std::to_string(rn) + "]," +
|
|
std::to_string(lanes * sbits / 8) + "); ";
|
|
if (indexed_mul) {
|
|
const u32 index = (((i >> 11) & 1) << (size == 1 ? 2 : 1)) |
|
|
(((i >> 21) & 1) << (size == 1 ? 1 : 0)) |
|
|
(size == 1 ? ((i >> 20) & 1) : 0);
|
|
s += "for(int _j=0;_j<" + std::to_string(lanes) + ";++_j)memcpy(&_b[_j],(const uint8_t*)c->vreg[" +
|
|
std::to_string(size == 1 ? rm & 15 : rm) + "]+" +
|
|
std::to_string(index * bits / 8) + ",sizeof(_b[_j])); ";
|
|
} else if (binary || mul) s += "memcpy(_b,c->vreg[" + std::to_string(rm) + "]," + std::to_string(bytes) + "); ";
|
|
s += "for(int _i=0;_i<" + std::to_string(lanes) + ";++_i){ uint64_t _z; ";
|
|
if (binary || unary || (narrow && U && !to_unsigned)) s += "uint64_t _x=_a[_i]; ";
|
|
const std::string sign = "(UINT64_C(1)<<" + std::to_string(bits - 1) + ")";
|
|
const std::string max = "UINT" + std::to_string(bits) + "_MAX";
|
|
if (binary) {
|
|
const bool sub = (i & 0x2000) != 0;
|
|
s += "uint64_t _y=_b[_i]; _z=(" + ty + ")(_x" + (sub ? "-" : "+") + "_y); ";
|
|
if (U) s += "if(" + std::string(sub ? "_x<_y" : "_z<_x") + "){_z=" + (sub ? "0" : max) + ";c->fpsr|=UINT64_C(1)<<27;} ";
|
|
else s += "if(((" + std::string(sub ? "_x^_y" : "~(_x^_y)") + ")&(_x^_z)&" + sign + ")!=0){_z=(_x&" + sign + ")?" + sign + ":" + sign + "-1;c->fpsr|=UINT64_C(1)<<27;} ";
|
|
} else if (unary) {
|
|
s += "_z=_x; if(_x==" + sign + "){_z=" + sign + "-1;c->fpsr|=UINT64_C(1)<<27;} else ";
|
|
s += U ? "_z=(uint64_t)0-_x; " : "if(_x&" + sign + ")_z=(uint64_t)0-_x; ";
|
|
} else if (narrow) {
|
|
if (U && !to_unsigned) s += "_z=_x;if(_z>" + max + "){_z=" + max + ";c->fpsr|=UINT64_C(1)<<27;} ";
|
|
else {
|
|
s += "int" + std::to_string(sbits) + "_t _v;memcpy(&_v,&_a[_i],sizeof(_v)); ";
|
|
const std::string hi = to_unsigned ? max : "INT" + std::to_string(bits) + "_MAX";
|
|
const std::string lo = to_unsigned ? "0" : "INT" + std::to_string(bits) + "_MIN";
|
|
s += "if(_v>" + hi + "){_z=" + hi + ";c->fpsr|=UINT64_C(1)<<27;}else if(_v<" + lo + "){_z=(uint64_t)(int64_t)(" + lo + ");c->fpsr|=UINT64_C(1)<<27;}else _z=(uint64_t)_v; ";
|
|
}
|
|
} else {
|
|
// The undoubled product fits int64_t, including S x S. Divide
|
|
// by 2^(bits-1) with explicit floor instead of shifting negatives.
|
|
s += "int" + std::to_string(bits) + "_t _sx,_sy;memcpy(&_sx,&_a[_i],sizeof(_sx));memcpy(&_sy,&_b[_i],sizeof(_sy));int64_t _p=(int64_t)_sx*_sy; ";
|
|
if (indexed_mul ? (i & 0x1000) != 0 : U) s += "_p+=INT64_C(1)<<" + std::to_string(bits - 2) + "; ";
|
|
s += "int64_t _d=INT64_C(1)<<" + std::to_string(bits - 1) + ",_v=_p/_d;if(_p<0&&_p%_d)--_v;if(_v>INT" + std::to_string(bits) + "_MAX){_v=INT" + std::to_string(bits) + "_MAX;c->fpsr|=UINT64_C(1)<<27;}_z=(uint64_t)_v; ";
|
|
}
|
|
s += "_r[_i]=(" + ty + ")_z;} ";
|
|
if (!(narrow && Q && !scalar)) s += "memset(c->vreg[" + std::to_string(rd) + "],0,16); ";
|
|
s += "memcpy((uint8_t*)c->vreg[" + std::to_string(rd) + "]+" +
|
|
std::to_string(narrow && Q && !scalar ? 8 : 0) + ",_r," + std::to_string(bytes) + "); }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// Scalar D ADD/SUB and CMEQ/CMTST. Snapshot both sources for aliases.
|
|
if ((i & 0xDFE0F400) == 0x5EE08400) {
|
|
const u32 rm = (i >> 16) & 31, rn = (i >> 5) & 31, rd = i & 31;
|
|
const bool U = (i & 0x20000000) != 0, compare = (i & 0x800) != 0;
|
|
const std::string expr = compare ? (U ? "(_a==_b)?~0ULL:0" : "(_a&_b)?~0ULL:0")
|
|
: (U ? "_a-_b" : "_a+_b");
|
|
put("{ uint64_t _a=c->vreg[" + std::to_string(rn) + "][0],_b=c->vreg[" +
|
|
std::to_string(rm) + "][0]; c->vreg[" + std::to_string(rd) + "][0]=" + expr +
|
|
"; c->vreg[" + std::to_string(rd) + "][1]=0; }");
|
|
return true;
|
|
}
|
|
|
|
// Saturating adds, min/max, equality and bit-test comparisons.
|
|
if ((i & 0x9F20FC00) == 0x0E200C00 || (i & 0x9F20FC00) == 0x0E208C00 ||
|
|
(i & 0x9F20FC00) == 0x0E206400 || (i & 0x9F20FC00) == 0x0E206C00) {
|
|
const u32 Q = (i >> 30) & 1, U = (i >> 29) & 1;
|
|
const u32 size = (i >> 22) & 3, opcode = (i >> 11) & 0x1F;
|
|
const u32 rm = (i >> 16) & 31, rn = (i >> 5) & 31, rd = i & 31;
|
|
const int esz = 1 << size;
|
|
const int bytes = Q ? 16 : 8;
|
|
if (!(size == 3 && !Q)) {
|
|
const int lanes = bytes / esz;
|
|
const int bits = esz * 8;
|
|
const std::string uty = "uint" + std::to_string(bits) + "_t";
|
|
const std::string ity = "int" + std::to_string(bits) + "_t";
|
|
std::string body;
|
|
if (opcode == 0x01) {
|
|
if (U) {
|
|
// Unsigned saturating add: the sum wrapping below an input
|
|
// is exactly the overflow condition.
|
|
body = "{ " + uty + " _s=(" + uty + ")(_a[_i]+_b[_i]); _r[_i]=(_s<_a[_i])?(" +
|
|
uty + ")~(" + uty + ")0:_s; }";
|
|
} else {
|
|
body = "{ " + ity + " _x=(" + ity + ")_a[_i], _y=(" + ity + ")_b[_i]; " + ity +
|
|
" _s=(" + ity + ")((" + uty + ")_x+(" + uty + ")_y); " +
|
|
"if(((_x^_s)&(_y^_s))<0) _r[_i]=(" + uty + ")(_x<0?(" + ity + ")((" +
|
|
uty + ")1<<" + std::to_string(bits - 1) + "):(" + ity + ")(((" + uty +
|
|
")1<<" + std::to_string(bits - 1) + ")-1)); else _r[_i]=(" + uty +
|
|
")_s; }";
|
|
}
|
|
} else if (opcode == 0x0C || opcode == 0x0D) {
|
|
// SMAX / SMIN and their unsigned twins.
|
|
const char* op = (opcode == 0x0C) ? ">" : "<";
|
|
if (U) {
|
|
body = "_r[_i]=(_a[_i]" + std::string(op) + "_b[_i])?_a[_i]:_b[_i];";
|
|
} else {
|
|
body = "{ " + ity + " _x=(" + ity + ")_a[_i],_y=(" + ity + ")_b[_i]; _r[_i]=(" +
|
|
uty + ")(_x" + std::string(op) + "_y?_x:_y); }";
|
|
}
|
|
} else if (opcode == 0x11) {
|
|
// CMEQ when U is set, CMTST when it is not.
|
|
body = U ? ("_r[_i]=(_a[_i]==_b[_i])?(" + uty + ")~(" + uty + ")0:(" + uty + ")0;")
|
|
: ("_r[_i]=((_a[_i]&_b[_i])!=0)?(" + uty + ")~(" + uty + ")0:(" + uty +
|
|
")0;");
|
|
}
|
|
if (!body.empty()) {
|
|
std::string s = "{ " + uty + " _a[" + std::to_string(lanes) + "],_b[" +
|
|
std::to_string(lanes) + "],_r[" + std::to_string(lanes) + "]; ";
|
|
s += "memcpy(_a,c->vreg[" + std::to_string(rn) + "]," + std::to_string(bytes) +
|
|
"); ";
|
|
s += "memcpy(_b,c->vreg[" + std::to_string(rm) + "]," + std::to_string(bytes) +
|
|
"); ";
|
|
s += "for(int _i=0;_i<" + std::to_string(lanes) + ";_i++) " + body + " ";
|
|
s += "c->vreg[" + std::to_string(rd) + "][0]=0; c->vreg[" + std::to_string(rd) +
|
|
"][1]=0; ";
|
|
s += "memcpy(c->vreg[" + std::to_string(rd) + "],_r," + std::to_string(bytes) +
|
|
"); }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Scalar/vector FADD/FSUB and pairwise FADDP share one exact FP-state core.
|
|
{
|
|
const bool scalar = (i & 0xFF20EC00) == 0x1E202800 && ((i >> 22) & 3) <= 1;
|
|
const bool vector = (i & 0xBF20FC00) == 0x0E20D400;
|
|
const bool scalar_pair = (i & 0xFFBFFC00) == 0x7E30D800;
|
|
const bool vector_pair = (i & 0xBFA0FC00) == 0x2E20D400;
|
|
const bool dbl=(i&0x00400000)!=0,q=(i&0x40000000)!=0;
|
|
if (scalar || scalar_pair || ((vector || vector_pair) && (!dbl || q))) {
|
|
const unsigned rd=i&31,rn=(i>>5)&31,rm=(i>>16)&31;
|
|
const unsigned lanes=scalar||scalar_pair?1:(q?16:8)/(dbl?8:4);
|
|
const bool sub=scalar?(i&0x1000)!=0:vector&&(i&0x800000)!=0;
|
|
const std::string ct=dbl?"uint64_t":"uint32_t",count=dbl?"2":"4";
|
|
put("{ "+ct+" _n["+count+"],_m["+count+"],_r["+count+"]={0};"
|
|
"memcpy(_n,c->vreg["+std::to_string(rn)+"],16);memcpy(_m,c->vreg["+std::to_string(rm)+"],16);");
|
|
for(unsigned e=0;e<lanes;++e) {
|
|
const std::string src=vector_pair&&e>=lanes/2?"_m":"_n";
|
|
const unsigned a=scalar_pair?0:vector_pair?2*(e%(lanes/2)):e;
|
|
const unsigned b=scalar_pair?1:vector_pair?a+1:e;
|
|
put("{uint64_t _a="+src+"["+std::to_string(a)+"],_b="+
|
|
(scalar_pair||vector_pair?src:"_m")+"["+std::to_string(b)+"],_v=0;"+
|
|
EmitFPAddSubValue(dbl,sub)+"_r["+std::to_string(e)+"]=("+ct+")_v;}");
|
|
}
|
|
put("memcpy(c->vreg["+std::to_string(rd)+"],_r,16);}");
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// Integer pairwise add/min/max and min/max reductions. Snapshot both
|
|
// sources before writing so all register aliases remain valid.
|
|
{
|
|
const u32 pair = i & 0xBF20FC00;
|
|
const bool add = pair == 0x0E20BC00;
|
|
const bool extrema = (i & 0x9F20F400) == 0x0E20A400;
|
|
const bool reduction = (i & 0x9F3EFC00) == 0x0E30A800;
|
|
const bool scalar_add = (i & 0xFFFFFC00) == 0x5EF1B800;
|
|
const u32 Q = (i >> 30) & 1, size = (i >> 22) & 3;
|
|
if (scalar_add || ((add || extrema || reduction) &&
|
|
(size < 3 || (add && Q)) && !(reduction && size == 2 && !Q))) {
|
|
const u32 rn = (i >> 5) & 31, rm = (i >> 16) & 31, rd = i & 31;
|
|
const int esz = scalar_add ? 8 : 1 << size;
|
|
const int bytes = scalar_add ? 16 : Q ? 16 : 8;
|
|
const int lanes = bytes / esz;
|
|
const bool minimum = ((i >> (reduction ? 16 : 11)) & 1) != 0;
|
|
const bool uns = ((i >> 29) & 1) != 0;
|
|
const std::string ty = std::string((add || scalar_add || uns) ? "uint" : "int") +
|
|
std::to_string(esz * 8) + "_t";
|
|
std::string s = "{ " + ty + " _a[" + std::to_string(lanes) + "],_r[" +
|
|
std::to_string(lanes) + "]={0}; ";
|
|
if (!reduction && !scalar_add) s += ty + " _b[" + std::to_string(lanes) + "]; ";
|
|
s += "memcpy(_a,c->vreg[" + std::to_string(rn) + "]," + std::to_string(bytes) + "); ";
|
|
if (reduction) {
|
|
s += "_r[0]=_a[0]; for(int _j=1;_j<" + std::to_string(lanes) +
|
|
";++_j) if(_a[_j]" + (minimum ? "<" : ">") + "_r[0]) _r[0]=_a[_j]; ";
|
|
} else if (scalar_add) {
|
|
s += "_r[0]=_a[0]+_a[1]; ";
|
|
} else {
|
|
s += "memcpy(_b,c->vreg[" + std::to_string(rm) + "]," + std::to_string(bytes) + "); ";
|
|
s += "for(int _j=0;_j<" + std::to_string(lanes / 2) + ";++_j) { ";
|
|
for (const char* src : {"_a", "_b"}) {
|
|
const std::string a = std::string(src) + "[2*_j]", b = std::string(src) + "[2*_j+1]";
|
|
s += "_r[_j" + std::string(src[1] == 'b' ? "+" + std::to_string(lanes / 2) : "") + "]=";
|
|
s += add ? "(" + ty + ")(" + a + "+" + b + "); " :
|
|
a + (minimum ? "<" : ">") + b + "?" + a + ":" + b + "; ";
|
|
}
|
|
s += "} ";
|
|
}
|
|
s += "memset(c->vreg[" + std::to_string(rd) + "],0,16); memcpy(c->vreg[" +
|
|
std::to_string(rd) + "],_r," + std::to_string(reduction || scalar_add ? esz : bytes) + "); }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// ADDV: sum every lane into the scalar destination. This is the
|
|
// across-lanes class - bits 21..17 are 11000, not the 10000 of the
|
|
// two-register-misc ops it otherwise resembles.
|
|
if ((i & 0x9F3FFC00) == 0x0E31B800) {
|
|
const u32 Q = (i >> 30) & 1;
|
|
const u32 size = (i >> 22) & 3;
|
|
const u32 rn = (i >> 5) & 31, rd = i & 31;
|
|
const int esz = 1 << size;
|
|
const int bytes = Q ? 16 : 8;
|
|
// 64-bit elements have no ADDV form, and 32-bit needs the full register.
|
|
if (size != 3 && !(size == 2 && !Q)) {
|
|
const int lanes = bytes / esz;
|
|
const std::string ty = "uint" + std::to_string(esz * 8) + "_t";
|
|
std::string s = "{ " + ty + " _a[" + std::to_string(lanes) + "],_s=0; ";
|
|
s += "memcpy(_a,c->vreg[" + std::to_string(rn) + "]," + std::to_string(bytes) + "); ";
|
|
s += "for(int _i=0;_i<" + std::to_string(lanes) + ";_i++) _s=(" + ty + ")(_s+_a[_i]); ";
|
|
s += "c->vreg[" + std::to_string(rd) + "][0]=0; c->vreg[" + std::to_string(rd) +
|
|
"][1]=0; ";
|
|
s += "memcpy(c->vreg[" + std::to_string(rd) + "],&_s," + std::to_string(esz) + "); }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// FRECPS: the Newton-Raphson step for reciprocal estimation, 2 - n*m.
|
|
// Bit 23 selects FRSQRTS, whose step is (3 - n*m)/2. Both are fused: one
|
|
// rounding of the exact value, per lane through the scalar forms' exact
|
|
// body. Host arithmetic would round the product first unless the compiler
|
|
// happened to contract it, and gives NaN for inf*0 where the step is 2 or
|
|
// 1.5; it also never raised FPSR flags.
|
|
if ((i & 0xBF20FC00) == 0x0E20FC00) {
|
|
const u32 Q = (i >> 30) & 1;
|
|
const bool rsqrt = ((i >> 23) & 1) != 0;
|
|
const bool dbl = ((i >> 22) & 1) != 0;
|
|
const u32 rm = (i >> 16) & 31, rn = (i >> 5) & 31, rd = i & 31;
|
|
if (!(dbl && !Q)) {
|
|
const std::string ct = dbl ? "uint64_t" : "uint32_t", count = dbl ? "2" : "4";
|
|
const unsigned lanes = (Q ? 16 : 8) / (dbl ? 8 : 4);
|
|
put("{ " + ct + " _n[" + count + "],_m[" + count + "],_r[" + count +
|
|
"]={0};memcpy(_n,c->vreg[" + std::to_string(rn) + "],16);memcpy(_m,c->vreg[" +
|
|
std::to_string(rm) + "],16);for(unsigned _l=0;_l<" + std::to_string(lanes) +
|
|
";++_l){uint64_t _aa=_n[_l],_bb=_m[_l];" + EmitFPStepValue(dbl, rsqrt) + "_r[_l]=(" +
|
|
ct + ")_v;}memcpy(c->vreg[" + std::to_string(rd) + "],_r,16);}");
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// SHA256SU0: the sigma0 half of the message schedule update.
|
|
// W[t] = W[t-16] + s0(W[t-15]) + W[t-7] + s1(W[t-2])
|
|
// This instruction contributes the first two terms; SHA256SU1 adds the
|
|
// rest. Verified against the scalar recurrence, not transcribed.
|
|
if ((i & 0xFFFE0C00) == 0x5E280800) {
|
|
const u32 opcode = (i >> 12) & 0x1F;
|
|
const u32 rn = (i >> 5) & 31, rd = i & 31;
|
|
if (opcode == 0) {
|
|
// SHA1H: rotate left by 30.
|
|
std::string s = "{ uint32_t _x; memcpy(&_x,c->vreg[" + std::to_string(rn) +
|
|
"],4); _x=(_x<<30)|(_x>>2); ";
|
|
s += "c->vreg[" + std::to_string(rd) + "][0]=0; c->vreg[" + std::to_string(rd) +
|
|
"][1]=0; memcpy(c->vreg[" + std::to_string(rd) + "],&_x,4); }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
if (opcode == 1) {
|
|
// SHA1SU1: finish the schedule group. The fourth word folds in the
|
|
// first of this very group, so it cannot be one loop.
|
|
std::string s = "{ uint32_t _d[4],_n[4],_t[4],_r[4]; ";
|
|
s += "memcpy(_d,c->vreg[" + std::to_string(rd) + "],16); ";
|
|
s += "memcpy(_n,c->vreg[" + std::to_string(rn) + "],16); ";
|
|
s += "_t[0]=_d[0]^_n[1]; _t[1]=_d[1]^_n[2]; _t[2]=_d[2]^_n[3]; _t[3]=_d[3]; ";
|
|
s += "for(int _e=0;_e<3;_e++) _r[_e]=(_t[_e]<<1)|(_t[_e]>>31); ";
|
|
s += "{ uint32_t _v=_t[3]^_r[0]; _r[3]=(_v<<1)|(_v>>31); } ";
|
|
s += "memcpy(c->vreg[" + std::to_string(rd) + "],_r,16); }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
if (opcode == 2) {
|
|
std::string s = "{ uint32_t _d[4],_n[4],_t[4],_r[4]; ";
|
|
s += "memcpy(_d,c->vreg[" + std::to_string(rd) + "],16); ";
|
|
s += "memcpy(_n,c->vreg[" + std::to_string(rn) + "],16); ";
|
|
// T is the window one word along: Vn<31:0> : Vd<127:32>.
|
|
s += "_t[0]=_d[1]; _t[1]=_d[2]; _t[2]=_d[3]; _t[3]=_n[0]; ";
|
|
s += "for(int _e=0;_e<4;_e++){ uint32_t _x=_t[_e]; ";
|
|
s += "uint32_t _s0=((_x>>7)|(_x<<25))^((_x>>18)|(_x<<14))^(_x>>3); ";
|
|
s += "_r[_e]=_s0+_d[_e]; } ";
|
|
s += "memcpy(c->vreg[" + std::to_string(rd) + "],_r,16); }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// SHA1C / SHA1P / SHA1M and SHA1SU0. Same class as SHA256SU1, picked
|
|
// apart by opcode: 000 choose, 001 parity, 010 majority, 011 schedule.
|
|
if ((i & 0xFFE08C00) == 0x5E000000) {
|
|
const u32 opcode = (i >> 12) & 7;
|
|
const u32 rm = (i >> 16) & 31, rn = (i >> 5) & 31, rd = i & 31;
|
|
if (opcode <= 2) {
|
|
const char* f = (opcode == 0) ? "((_b&_cc)|(~_b&_dd))"
|
|
: (opcode == 1) ? "(_b^_cc^_dd)"
|
|
: "((_b&_cc)|(_b&_dd)|(_cc&_dd))";
|
|
std::string s = "{ uint32_t _v[4],_w[4],_x,_y; ";
|
|
s += "memcpy(_v,c->vreg[" + std::to_string(rd) + "],16); ";
|
|
s += "memcpy(_w,c->vreg[" + std::to_string(rm) + "],16); ";
|
|
s += "memcpy(&_y,c->vreg[" + std::to_string(rn) + "],4); ";
|
|
s += "_x=_v[0]; ";
|
|
s += "for(int _e=0;_e<4;_e++){ uint32_t _b=_v[1],_cc=_v[2],_dd=_v[3]; ";
|
|
s += "uint32_t _t=((_x<<5)|(_x>>27))+" + std::string(f) + "+_y+_w[_e]; ";
|
|
s += "_y=_dd; _v[3]=_cc; _v[2]=(_b<<30)|(_b>>2); _v[1]=_x; _x=_t; _v[0]=_t; } ";
|
|
// Only Vd is written; the running e stays inside the instruction.
|
|
s += "memcpy(c->vreg[" + std::to_string(rd) + "],_v,16); }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
if (opcode == 4 || opcode == 5) {
|
|
// SHA256H / SHA256H2. Four rounds over a 256-bit state split across
|
|
// two registers; the pair differ only in which half is X and which
|
|
// half is returned.
|
|
const bool part1 = opcode == 4;
|
|
const u32 xs = part1 ? rd : rn, ys = part1 ? rn : rd;
|
|
std::string s = "{ uint32_t _X[4],_Y[4],_W[4],_nx[4],_ny[4]; ";
|
|
s += "memcpy(_X,c->vreg[" + std::to_string(xs) + "],16); ";
|
|
s += "memcpy(_Y,c->vreg[" + std::to_string(ys) + "],16); ";
|
|
s += "memcpy(_W,c->vreg[" + std::to_string(rm) + "],16); ";
|
|
s += "for(int _e=0;_e<4;_e++){ uint32_t _y0=_Y[0],_x0=_X[0]; ";
|
|
s += "uint32_t _chs=(_y0&_Y[1])^(~_y0&_Y[2]); ";
|
|
s += "uint32_t _maj=(_x0&_X[1])^(_x0&_X[2])^(_X[1]&_X[2]); ";
|
|
s += "uint32_t _s1=((_y0>>6)|(_y0<<26))^((_y0>>11)|(_y0<<21))^((_y0>>25)|(_y0<<7)); ";
|
|
s += "uint32_t _s0=((_x0>>2)|(_x0<<30))^((_x0>>13)|(_x0<<19))^((_x0>>22)|(_x0<<10)); ";
|
|
s += "uint32_t _t=_Y[3]+_s1+_chs+_W[_e]; ";
|
|
// The 256-bit state rotates left by one word each round, so the two
|
|
// freshly written words land at the bottom of the other half.
|
|
s += "_nx[0]=_t+_s0+_maj; _nx[1]=_X[0]; _nx[2]=_X[1]; _nx[3]=_X[2]; ";
|
|
s += "_ny[0]=_t+_X[3]; _ny[1]=_Y[0]; _ny[2]=_Y[1]; _ny[3]=_Y[2]; ";
|
|
s += "memcpy(_X,_nx,16); memcpy(_Y,_ny,16); } ";
|
|
s += "memcpy(c->vreg[" + std::to_string(rd) + "]," +
|
|
std::string(part1 ? "_X" : "_Y") + ",16); }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
if (opcode == 3) {
|
|
// SHA1SU0: the window two words along, folded with Vd and Vm.
|
|
std::string s = "{ uint32_t _d[4],_n[4],_m[4],_r[4]; ";
|
|
s += "memcpy(_d,c->vreg[" + std::to_string(rd) + "],16); ";
|
|
s += "memcpy(_n,c->vreg[" + std::to_string(rn) + "],16); ";
|
|
s += "memcpy(_m,c->vreg[" + std::to_string(rm) + "],16); ";
|
|
s += "_r[0]=_d[2]; _r[1]=_d[3]; _r[2]=_n[0]; _r[3]=_n[1]; ";
|
|
s += "for(int _e=0;_e<4;_e++) _r[_e]^=_d[_e]^_m[_e]; ";
|
|
s += "memcpy(c->vreg[" + std::to_string(rd) + "],_r,16); }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// SHA256SU1: the sigma1 half, plus the two carried terms. The upper two
|
|
// words need sigma1 of the two just computed, because W[t+2] depends on
|
|
// W[t] - which is why this cannot be written as one loop.
|
|
if ((i & 0xFFE0FC00) == 0x5E006000) {
|
|
const u32 rm = (i >> 16) & 31, rn = (i >> 5) & 31, rd = i & 31;
|
|
std::string s = "{ uint32_t _d[4],_n[4],_m[4],_t0[4],_t1[2],_r[4]; ";
|
|
s += "memcpy(_d,c->vreg[" + std::to_string(rd) + "],16); ";
|
|
s += "memcpy(_n,c->vreg[" + std::to_string(rn) + "],16); ";
|
|
s += "memcpy(_m,c->vreg[" + std::to_string(rm) + "],16); ";
|
|
s += "_t0[0]=_n[1]; _t0[1]=_n[2]; _t0[2]=_n[3]; _t0[3]=_m[0]; ";
|
|
s += "_t1[0]=_m[2]; _t1[1]=_m[3]; ";
|
|
s += "for(int _e=0;_e<2;_e++){ uint32_t _x=_t1[_e]; ";
|
|
s += "uint32_t _s1=((_x>>17)|(_x<<15))^((_x>>19)|(_x<<13))^(_x>>10); ";
|
|
s += "_r[_e]=_s1+_d[_e]+_t0[_e]; } ";
|
|
s += "for(int _e=2;_e<4;_e++){ uint32_t _x=_r[_e-2]; ";
|
|
s += "uint32_t _s1=((_x>>17)|(_x<<15))^((_x>>19)|(_x<<13))^(_x>>10); ";
|
|
s += "_r[_e]=_s1+_d[_e]+_t0[_e]; } ";
|
|
s += "memcpy(c->vreg[" + std::to_string(rd) + "],_r,16); }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
|
|
// LD1-LD4 / ST1-ST4, whole registers. opcode says how many registers and
|
|
// whether they are interleaved: LD2/3/4 spread consecutive elements across
|
|
// the registers, while the LD1 forms are plain consecutive blocks.
|
|
if ((i & 0xBFBF0000) == 0x0C000000 || (i & 0xBFA00000) == 0x0C800000) {
|
|
const u32 Q = (i >> 30) & 1;
|
|
const bool post = ((i >> 23) & 1) != 0;
|
|
const bool load = ((i >> 22) & 1) != 0;
|
|
const u32 rm = (i >> 16) & 31;
|
|
const u32 opcode = (i >> 12) & 15, size = (i >> 10) & 3;
|
|
const u32 rn = (i >> 5) & 31, rt = i & 31;
|
|
int regs = 0, step = 0;
|
|
switch (opcode) {
|
|
case 0x0: regs = 4; step = 4; break; // LD4/ST4, interleaved
|
|
case 0x2: regs = 4; step = 1; break; // LD1/ST1, four registers
|
|
case 0x4: regs = 3; step = 3; break; // LD3/ST3, interleaved
|
|
case 0x6: regs = 3; step = 1; break;
|
|
case 0x7: regs = 1; step = 1; break;
|
|
case 0x8: regs = 2; step = 2; break; // LD2/ST2, interleaved
|
|
case 0xA: regs = 2; step = 1; break;
|
|
default: break;
|
|
}
|
|
const int esz = 1 << size;
|
|
const int bytes = Q ? 16 : 8;
|
|
// The .1D arrangement exists only for the non-interleaved LD1/ST1 forms.
|
|
const bool shaped = regs > 0 && !(size == 3 && !Q && step != 1);
|
|
if (shaped) {
|
|
const int lanes = bytes / esz;
|
|
const int bits = esz * 8;
|
|
std::string s = "{ uint64_t _a=" + Xsp(rn) + "; uint64_t _v; ";
|
|
// Emit in ascending memory order, including non-interleaved LD1/ST1.
|
|
for (int k = 0; k < lanes * regs; ++k) {
|
|
const int e = step == 1 ? k % lanes : k / regs;
|
|
const int r = step == 1 ? k / lanes : k % regs;
|
|
const std::string off = std::to_string(k * esz);
|
|
const std::string vr = std::to_string((rt + (u32)r) & 31);
|
|
const std::string lane = std::to_string(e * esz);
|
|
if (load) {
|
|
s += "_v=recomp_load" + std::to_string(bits) + "(c,_a+" +
|
|
off + "); memcpy((uint8_t*)c->vreg[" + vr + "]+" +
|
|
lane + ",&_v," + std::to_string(esz) + "); ";
|
|
} else {
|
|
s += "_v=0; memcpy(&_v,(const uint8_t*)c->vreg[" + vr + "]+" + lane + "," +
|
|
std::to_string(esz) + "); recomp_store" + std::to_string(bits) +
|
|
"(c,_a+" + off + ",_v); ";
|
|
}
|
|
}
|
|
if (load && !Q) {
|
|
// The 64-bit forms clear the top half of every register written.
|
|
for (int r = 0; r < regs; ++r) {
|
|
s += "c->vreg[" + std::to_string((rt + (u32)r) & 31) + "][1]=0; ";
|
|
}
|
|
}
|
|
if (post) {
|
|
const std::string adv = (rm == 31)
|
|
? (std::to_string(regs * bytes) + "ULL")
|
|
: ("c->x[" + std::to_string(rm) + "]");
|
|
s += "c->x[" + std::to_string(rn) + "]=_a+" + adv + "; ";
|
|
}
|
|
s += "}";
|
|
put(s);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// AESMC / AESIMC: the MixColumns step and its inverse. The matrix and the
|
|
// xtime-based GF(2^8) multiply below are dynarmic's
|
|
// (common/crypto/aes.cpp), not written from the specification.
|
|
if ((i & 0xFFFE0C00) == 0x4E280800) {
|
|
const u32 opcode = (i >> 12) & 0x1F;
|
|
const u32 rn = (i >> 5) & 31, rd = i & 31;
|
|
if (opcode == 4 || opcode == 5) {
|
|
// AESE: Vd = SubBytes(ShiftRows(Vd EOR Vn)).
|
|
// AESD is the same with the inverse of each step. ShiftRows rotates
|
|
// row r left by r, which over the column-major state is a shift of
|
|
// 4*(i&3) - the permutation dynarmic spells out byte by byte.
|
|
const bool dec = (opcode == 5);
|
|
std::string s = "{ uint8_t _s[16],_k[16],_t[16]; const uint8_t* _sb=recomp_aes_sbox(" +
|
|
std::string(dec ? "1" : "0") + "); ";
|
|
s += "memcpy(_s,c->vreg[" + std::to_string(rd) + "],16); ";
|
|
s += "memcpy(_k,c->vreg[" + std::to_string(rn) + "],16); ";
|
|
s += "for(int _i=0;_i<16;_i++) _s[_i]=(uint8_t)(_s[_i]^_k[_i]); ";
|
|
if (dec) {
|
|
s += "for(int _i=0;_i<16;_i++) _t[_i]=_s[(_i+16-4*(_i&3))&15]; ";
|
|
} else {
|
|
s += "for(int _i=0;_i<16;_i++) _t[_i]=_s[(_i+4*(_i&3))&15]; ";
|
|
}
|
|
s += "for(int _i=0;_i<16;_i++) _t[_i]=_sb[_t[_i]]; ";
|
|
s += "memcpy(c->vreg[" + std::to_string(rd) + "],_t,16); }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
if (opcode == 6 || opcode == 7) {
|
|
const char* mtx = (opcode == 7)
|
|
? "{14,11,13,9},{9,14,11,13},{13,9,14,11},{11,13,9,14}"
|
|
: "{2,3,1,1},{1,2,3,1},{1,1,2,3},{3,1,1,2}";
|
|
std::string s = "{ uint8_t _s[16],_r[16]; static const uint8_t _mx[4][4]={";
|
|
s += mtx;
|
|
s += "}; memcpy(_s,c->vreg[" + std::to_string(rn) + "],16); ";
|
|
s += "for(int _cl=0;_cl<16;_cl+=4) for(int _o=0;_o<4;_o++){ uint8_t _acc=0; ";
|
|
s += "for(int _k=0;_k<4;_k++){ uint8_t _x=_s[_cl+_k],_y=_mx[_o][_k],_pp=0; ";
|
|
s += "while(_y){ if(_y&1) _pp=(uint8_t)(_pp^_x); ";
|
|
s += "_x=(uint8_t)((_x<<1)^((_x>>7)*0x1B)); _y=(uint8_t)(_y>>1); } ";
|
|
s += "_acc=(uint8_t)(_acc^_pp); } _r[_cl+_o]=_acc; } ";
|
|
s += "memcpy(c->vreg[" + std::to_string(rd) + "],_r,16); }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// PMULL / PMULL2: carry-less multiply of the low or high 64-bit halves.
|
|
if ((i & 0xBF20FC00) == 0x0E20E000) {
|
|
const u32 Q = (i >> 30) & 1, U = (i >> 29) & 1;
|
|
const u32 size = (i >> 22) & 3;
|
|
const u32 rm = (i >> 16) & 31, rn = (i >> 5) & 31, rd = i & 31;
|
|
if (!U && size == 3) {
|
|
const int half = Q ? 1 : 0; // PMULL2 takes the top half of each source
|
|
std::string s = "{ uint64_t _a=c->vreg[" + std::to_string(rn) + "][" +
|
|
std::to_string(half) + "], _b=c->vreg[" + std::to_string(rm) + "][" +
|
|
std::to_string(half) + "]; uint64_t _lo=0,_hi=0; ";
|
|
s += "for(int _k=0;_k<64;_k++) if((_b>>_k)&1ULL){ _lo^=_a<<_k; ";
|
|
// Shifting by 64 is undefined, and that is exactly the k=0 case.
|
|
s += "if(_k) _hi^=_a>>(64-_k); } ";
|
|
s += "c->vreg[" + std::to_string(rd) + "][0]=_lo; c->vreg[" + std::to_string(rd) +
|
|
"][1]=_hi; }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// CRC32 / CRC32C. Computed a bit at a time rather than from a table: this
|
|
// is the same recurrence dynarmic's tables encode, and a cold instruction
|
|
// does not justify carrying 2 KB of tables in every generated image.
|
|
if ((i & 0x7FE0E000) == 0x1AC04000) {
|
|
const u32 sf = i >> 31;
|
|
const u32 rm = (i >> 16) & 31, rn = (i >> 5) & 31, rd = i & 31;
|
|
const u32 castagnoli = (i >> 12) & 1, sz = (i >> 10) & 3;
|
|
// The 64-bit variant is the sz=11 form only, and it needs sf set.
|
|
if (!(sz == 3 && !sf) && !(sz != 3 && sf) && rd != 31) {
|
|
const int nbytes = 1 << sz;
|
|
const char* poly = castagnoli ? "0x82F63B78UL" : "0xEDB88320UL";
|
|
std::string s = "{ uint32_t _crc=(uint32_t)" + Xz(rn) + "; uint64_t _v=" + Xz(rm) +
|
|
"; ";
|
|
s += "for(int _i=0;_i<" + std::to_string(nbytes) + ";_i++){ ";
|
|
s += "_crc^=(uint8_t)(_v>>(8*_i)); ";
|
|
s += "for(int _b=0;_b<8;_b++) _crc=(_crc>>1)^(" + std::string(poly) +
|
|
" & (uint32_t)(-(int32_t)(_crc&1u))); } ";
|
|
s += "c->x[" + std::to_string(rd) + "]=(uint64_t)_crc; }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// BSL / BIT / BIF: bitwise select. All three are the same operation with a
|
|
// different choice of which register supplies the mask and which the
|
|
// destination, so size picks the variant rather than an element width.
|
|
// U must be set; the U=0 half of this encoding is AND/BIC/ORR/ORN.
|
|
if ((i & 0xBF20FC00) == 0x2E201C00) {
|
|
const u32 Q = (i >> 30) & 1;
|
|
const u32 size = (i >> 22) & 3;
|
|
const u32 rm = (i >> 16) & 31, rn = (i >> 5) & 31, rd = i & 31;
|
|
if (size != 0) { // size 0 is EOR, handled with the other logicals
|
|
const int halves = Q ? 2 : 1;
|
|
std::string s = "{ ";
|
|
for (int h = 0; h < halves; ++h) {
|
|
const std::string k = "[" + std::to_string(h) + "]";
|
|
const std::string d = "c->vreg[" + std::to_string(rd) + "]" + k;
|
|
const std::string n = "c->vreg[" + std::to_string(rn) + "]" + k;
|
|
const std::string m = "c->vreg[" + std::to_string(rm) + "]" + k;
|
|
if (size == 1) {
|
|
// BSL: the destination is the mask.
|
|
s += d + " = (" + d + " & " + n + ") | (~" + d + " & " + m + "); ";
|
|
} else if (size == 2) {
|
|
// BIT: insert where the second source has bits set.
|
|
s += d + " = (" + d + " & ~" + m + ") | (" + n + " & " + m + "); ";
|
|
} else {
|
|
// BIF: insert where it has them clear.
|
|
s += d + " = (" + d + " & " + m + ") | (" + n + " & ~" + m + "); ";
|
|
}
|
|
}
|
|
if (!Q) {
|
|
s += "c->vreg[" + std::to_string(rd) + "][1]=0; ";
|
|
}
|
|
s += "}";
|
|
put(s);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// TBL / TBX: byte-wise table lookup. The table is 1-4 consecutive vector
|
|
// registers starting at Rn, wrapping at 32, and each byte of Rm indexes it.
|
|
// An index past the end gives zero for TBL and leaves the byte alone for
|
|
// TBX, which is the only difference between them.
|
|
if ((i & 0xBFE08C00) == 0x0E000000) {
|
|
const u32 Q = (i >> 30) & 1;
|
|
const u32 rm = (i >> 16) & 31, rn = (i >> 5) & 31, rd = i & 31;
|
|
const u32 len = (i >> 13) & 3, op = (i >> 12) & 1;
|
|
const int regs = (int)len + 1;
|
|
const int bytes = Q ? 16 : 8;
|
|
const int tbl_bytes = regs * 16;
|
|
std::string s = "{ uint8_t _t[" + std::to_string(tbl_bytes) + "],_x[" +
|
|
std::to_string(bytes) + "],_r[" + std::to_string(bytes) + "]; ";
|
|
for (int k = 0; k < regs; ++k) {
|
|
// The table wraps at v31, so a run starting near the top comes back
|
|
// round to v0 rather than reading off the end of the register file.
|
|
s += "memcpy(_t+" + std::to_string(k * 16) + ",c->vreg[" +
|
|
std::to_string((rn + (u32)k) & 31) + "],16); ";
|
|
}
|
|
s += "memcpy(_x,c->vreg[" + std::to_string(rm) + "]," + std::to_string(bytes) + "); ";
|
|
if (op) {
|
|
s += "memcpy(_r,c->vreg[" + std::to_string(rd) + "]," + std::to_string(bytes) + "); ";
|
|
}
|
|
s += "for(int _i=0;_i<" + std::to_string(bytes) + ";_i++){ unsigned _k=_x[_i]; ";
|
|
s += "if(_k<" + std::to_string(tbl_bytes) + "U) _r[_i]=_t[_k];";
|
|
if (!op) {
|
|
s += " else _r[_i]=0;";
|
|
}
|
|
s += " } ";
|
|
s += "c->vreg[" + std::to_string(rd) + "][0]=0; c->vreg[" + std::to_string(rd) +
|
|
"][1]=0; ";
|
|
s += "memcpy(c->vreg[" + std::to_string(rd) + "],_r," + std::to_string(bytes) + "); }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
|
|
// LD1-LD4 / ST1-ST4, one element per register, preserving every other lane.
|
|
// Bit 23 selects post-index; Rm=31 advances by the structure's byte size.
|
|
// Other Rm values name a register containing the increment.
|
|
if ((i & 0xBF000000) == 0x0D000000) {
|
|
const u32 Q = (i >> 30) & 1;
|
|
const bool post = ((i >> 23) & 1) != 0;
|
|
const bool load = ((i >> 22) & 1) != 0;
|
|
const u32 rm = (i >> 16) & 31;
|
|
const u32 opcode = (i >> 13) & 7, S = (i >> 12) & 1, size = (i >> 10) & 3;
|
|
const u32 rn = (i >> 5) & 31, rt = i & 31;
|
|
const u32 regs = (((opcode & 1) << 1) | ((i >> 21) & 1)) + 1;
|
|
int esz = 0, index = 0;
|
|
bool shaped = true;
|
|
if ((opcode >> 1) == 0) {
|
|
esz = 1;
|
|
index = (int)((Q << 3) | (S << 2) | size);
|
|
} else if ((opcode >> 1) == 1 && (size & 1) == 0) {
|
|
esz = 2;
|
|
index = (int)((Q << 2) | (S << 1) | (size >> 1));
|
|
} else if ((opcode >> 1) == 2 && size == 0) {
|
|
esz = 4;
|
|
index = (int)((Q << 1) | S);
|
|
} else if ((opcode >> 1) == 2 && size == 1 && S == 0) {
|
|
esz = 8;
|
|
index = (int)Q;
|
|
} else {
|
|
shaped = false; // Reserved shapes and the replicating forms below.
|
|
}
|
|
// Only the no-offset form may have a register field of zero meaning
|
|
// "no offset"; in the post-index form that field is Rm.
|
|
if (shaped && (post || rm == 0)) {
|
|
const int bits = esz * 8;
|
|
std::string s = "{ uint64_t _a=" + Xsp(rn) + "; ";
|
|
for (u32 r = 0; r < regs; ++r) {
|
|
const std::string vr = std::to_string((rt + r) & 31);
|
|
const std::string addr = "_a+" + std::to_string(r * esz);
|
|
if (load) {
|
|
s += "{ uint64_t _v=recomp_load" + std::to_string(bits) + "(c," + addr + "); ";
|
|
s += "memcpy((uint8_t*)c->vreg[" + vr + "]+" +
|
|
std::to_string(index * esz) + ",&_v," + std::to_string(esz) + "); ";
|
|
} else {
|
|
s += "{ uint64_t _v=0; memcpy(&_v,(const uint8_t*)c->vreg[" + vr +
|
|
"]+" + std::to_string(index * esz) + "," + std::to_string(esz) + "); ";
|
|
s += "recomp_store" + std::to_string(bits) + "(c," + addr + ",_v); ";
|
|
}
|
|
s += "} ";
|
|
}
|
|
if (post) {
|
|
const std::string step =
|
|
(rm == 31) ? (std::to_string(regs * esz) + "ULL") : ("c->x[" + std::to_string(rm) + "]");
|
|
s += "c->x[" + std::to_string(rn) + "]=_a+" + step + "; ";
|
|
}
|
|
s += "}";
|
|
put(s);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// SSHLL / USHLL: widen half the source elements and shift them left. SXTL
|
|
// and UXTL are these with a shift of zero, which is how the assembler spells
|
|
// them and why they never appeared as their own encoding.
|
|
if ((i & 0x9F00FC00) == 0x0F00A400) {
|
|
const u32 Q = (i >> 30) & 1, U = (i >> 29) & 1;
|
|
const u32 immh = (i >> 19) & 15, immb = (i >> 16) & 7;
|
|
const u32 rn = (i >> 5) & 31, rd = i & 31;
|
|
u32 size = 0;
|
|
bool shaped = true;
|
|
if (immh & 8) shaped = false; // reserved for this encoding
|
|
else if (immh & 4) size = 2;
|
|
else if (immh & 2) size = 1;
|
|
else if (immh & 1) size = 0;
|
|
else shaped = false;
|
|
if (shaped) {
|
|
const int sbits = 8 << size;
|
|
const u32 shift = ((immh << 3) | immb) - (u32)sbits;
|
|
const int ssz = sbits / 8;
|
|
const int lanes = 8 / ssz; // always half a register in
|
|
const int off = Q ? 8 : 0; // ...the top half when Q is set
|
|
const std::string sty =
|
|
(U ? std::string("uint") : std::string("int")) + std::to_string(sbits) + "_t";
|
|
const std::string dty = "uint" + std::to_string(sbits * 2) + "_t";
|
|
std::string s = "{ " + sty + " _a[" + std::to_string(lanes) + "]; " + dty + " _r[" +
|
|
std::to_string(lanes) + "]; ";
|
|
s += "memcpy(_a,(const uint8_t*)c->vreg[" + std::to_string(rn) + "]+" +
|
|
std::to_string(off) + ",8); ";
|
|
s += "for(int _i=0;_i<" + std::to_string(lanes) + ";_i++) _r[_i]=(" + dty + ")((" +
|
|
dty + ")_a[_i]<<" + std::to_string(shift) + "); ";
|
|
s += "memcpy(c->vreg[" + std::to_string(rd) + "],_r,16); }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// ZIP/UZP/TRN. All six are the same read of two registers with a different
|
|
// index pattern, so they share one emitter.
|
|
if ((i & 0xBF208C00) == 0x0E000800) {
|
|
const u32 Q = (i >> 30) & 1;
|
|
const u32 size = (i >> 22) & 3;
|
|
const u32 opcode = (i >> 12) & 7;
|
|
const u32 rm = (i >> 16) & 31, rn = (i >> 5) & 31, rd = i & 31;
|
|
const int esz = 1 << size;
|
|
const int bytes = Q ? 16 : 8;
|
|
const int lanes = bytes / esz;
|
|
const char* pattern = nullptr;
|
|
switch (opcode) {
|
|
case 1: pattern = "uzp1"; break;
|
|
case 2: pattern = "trn1"; break;
|
|
case 3: pattern = "zip1"; break;
|
|
case 5: pattern = "uzp2"; break;
|
|
case 6: pattern = "trn2"; break;
|
|
case 7: pattern = "zip2"; break;
|
|
default: break;
|
|
}
|
|
if (pattern && !(size == 3 && !Q)) {
|
|
const std::string ty = "uint" + std::to_string(esz * 8) + "_t";
|
|
const int half = lanes / 2;
|
|
std::string idx;
|
|
if (opcode == 3 || opcode == 7) {
|
|
// ZIP: interleave one half of each source.
|
|
const int base = (opcode == 7) ? half : 0;
|
|
idx = "_r[2*_i]=_a[" + std::to_string(base) + "+_i]; _r[2*_i+1]=_b[" +
|
|
std::to_string(base) + "+_i];";
|
|
} else if (opcode == 1 || opcode == 5) {
|
|
// UZP: take every other element, all of a then all of b.
|
|
const int first = (opcode == 5) ? 1 : 0;
|
|
idx = "_r[_i]=_a[2*_i+" + std::to_string(first) + "]; _r[" +
|
|
std::to_string(half) + "+_i]=_b[2*_i+" + std::to_string(first) + "];";
|
|
} else {
|
|
// TRN: pair up matching even or odd elements.
|
|
const int first = (opcode == 6) ? 1 : 0;
|
|
idx = "_r[2*_i]=_a[2*_i+" + std::to_string(first) + "]; _r[2*_i+1]=_b[2*_i+" +
|
|
std::to_string(first) + "];";
|
|
}
|
|
std::string s = "{ " + ty + " _a[" + std::to_string(lanes) + "],_b[" +
|
|
std::to_string(lanes) + "],_r[" + std::to_string(lanes) + "]; ";
|
|
s += "memcpy(_a,c->vreg[" + std::to_string(rn) + "]," + std::to_string(bytes) + "); ";
|
|
s += "memcpy(_b,c->vreg[" + std::to_string(rm) + "]," + std::to_string(bytes) + "); ";
|
|
s += "for(int _i=0;_i<" + std::to_string(half) + ";_i++){ " + idx + " } ";
|
|
s += "c->vreg[" + std::to_string(rd) + "][0]=0; c->vreg[" + std::to_string(rd) +
|
|
"][1]=0; ";
|
|
s += "memcpy(c->vreg[" + std::to_string(rd) + "],_r," + std::to_string(bytes) + "); }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// EXT: a window into Rn:Rm starting imm4 bytes in.
|
|
if ((i & 0xBFE08400) == 0x2E000000) {
|
|
const u32 Q = (i >> 30) & 1;
|
|
const u32 imm4 = (i >> 11) & 15;
|
|
const u32 rm = (i >> 16) & 31, rn = (i >> 5) & 31, rd = i & 31;
|
|
const int bytes = Q ? 16 : 8;
|
|
if ((int)imm4 < bytes) {
|
|
std::string s = "{ uint8_t _a[" + std::to_string(bytes) + "],_b[" +
|
|
std::to_string(bytes) + "],_r[" + std::to_string(bytes) + "]; ";
|
|
s += "memcpy(_a,c->vreg[" + std::to_string(rn) + "]," + std::to_string(bytes) + "); ";
|
|
s += "memcpy(_b,c->vreg[" + std::to_string(rm) + "]," + std::to_string(bytes) + "); ";
|
|
s += "for(int _i=0;_i<" + std::to_string(bytes) + ";_i++){ int _k=_i+" +
|
|
std::to_string(imm4) + "; _r[_i] = (_k<" + std::to_string(bytes) +
|
|
") ? _a[_k] : _b[_k-" + std::to_string(bytes) + "]; } ";
|
|
s += "c->vreg[" + std::to_string(rd) + "][0]=0; c->vreg[" + std::to_string(rd) +
|
|
"][1]=0; ";
|
|
s += "memcpy(c->vreg[" + std::to_string(rd) + "],_r," + std::to_string(bytes) + "); }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// FCMEQ / FCMGE / FCMGT, register forms. The compare-against-zero forms are
|
|
// handled with the other two-register-misc ops; these take a second vector.
|
|
// U and size<1> pick which comparison: the two bits are the operator.
|
|
if ((i & 0x9F20FC00) == 0x0E20E400) {
|
|
const u32 Q = (i >> 30) & 1, U = (i >> 29) & 1;
|
|
const u32 size = (i >> 22) & 3;
|
|
const u32 rm = (i >> 16) & 31, rn = (i >> 5) & 31, rd = i & 31;
|
|
const bool dbl = (size & 1) != 0;
|
|
const char* cmp = nullptr;
|
|
if (!U && !(size & 2)) cmp = "eq"; // FCMEQ
|
|
else if (U && !(size & 2)) cmp = "ge"; // FCMGE
|
|
else if (U && (size & 2)) cmp = "gt"; // FCMGT
|
|
if (cmp && !(dbl && !Q)) {
|
|
const std::string uty = dbl ? "uint64_t" : "uint32_t", count = dbl ? "2" : "4";
|
|
const int lanes = (Q ? 16 : 8) / (dbl ? 8 : 4);
|
|
put("{ " + uty + " _n[" + count + "],_m[" + count + "],_r[" + count +
|
|
"]={0}; memcpy(_n,c->vreg[" + std::to_string(rn) + "],16); memcpy(_m,c->vreg[" +
|
|
std::to_string(rm) + "],16); for(int _i=0;_i<" + std::to_string(lanes) + ";_i++) " +
|
|
EmitFPCompareMask(dbl, cmp, "_n[_i]", "_m[_i]", "_r[_i]=(" + uty + ")_v;") +
|
|
" memcpy(c->vreg[" + std::to_string(rd) + "],_r,16); }");
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// USHL/SSHL and URSHL/SRSHL, vector and scalar D. Only Rm's signed low
|
|
// byte controls the shift. Saturating neighboring opcodes remain separate.
|
|
if ((i & 0x8F20EC00) == 0x0E204400) {
|
|
const u32 Q = (i >> 30) & 1, U = (i >> 29) & 1;
|
|
const bool scalar = (i & 0x10000000) != 0, round = (i & 0x1000) != 0;
|
|
const u32 size = (i >> 22) & 3;
|
|
const u32 rm = (i >> 16) & 31, rn = (i >> 5) & 31, rd = i & 31;
|
|
const int esz = 1 << size;
|
|
const int ebits = esz * 8;
|
|
const int bytes = scalar ? 8 : Q ? 16 : 8;
|
|
if (scalar ? Q && size == 3 : !(size == 3 && !Q)) {
|
|
const int lanes = bytes / esz;
|
|
const std::string uty = "uint" + std::to_string(ebits) + "_t";
|
|
const std::string eb = std::to_string(ebits);
|
|
std::string s = "{ " + uty + " _a[" + std::to_string(lanes) + "],_m[" +
|
|
std::to_string(lanes) + "],_r[" + std::to_string(lanes) + "]; ";
|
|
s += "memcpy(_a,c->vreg[" + std::to_string(rn) + "]," + std::to_string(bytes) + "); ";
|
|
s += "memcpy(_m,c->vreg[" + std::to_string(rm) + "]," + std::to_string(bytes) + "); ";
|
|
s += "for(int _i=0;_i<" + std::to_string(lanes) + ";_i++){ ";
|
|
s += "int _s=(int)(_m[_i]&0xFF); if(_s>=128) _s-=256; uint64_t _v=_a[_i],_z; ";
|
|
// A shift of the element width or more is defined by the
|
|
// architecture but undefined in C, so both ends are special-cased.
|
|
s += "if(_s>=0) _z=(_s>=" + eb + ")?0:(_v<<_s); ";
|
|
s += "else { int _t=-_s; ";
|
|
s += "if(_t>" + eb + ") _z=" +
|
|
std::string(round || U ? "0" : "(_v>>(" + eb + "-1))?~0ULL:0") + "; ";
|
|
s += "else { _z=(_t==64)?0:(_v>>_t); ";
|
|
if (!U) s += "if(_v>>(" + eb + "-1)) _z|=~0ULL<<(" + eb + "-_t); ";
|
|
if (round) s += "_z+=(_v>>(_t-1))&1; ";
|
|
s += "} } _r[_i]=(" + uty + ")_z; } ";
|
|
s += "c->vreg[" + std::to_string(rd) + "][0]=0; c->vreg[" + std::to_string(rd) +
|
|
"][1]=0; ";
|
|
s += "memcpy(c->vreg[" + std::to_string(rd) + "],_r," + std::to_string(bytes) + "); }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// SHL by immediate. Bit 29 is U and must stay in the mask: the same opcode
|
|
// with U set is SLI, handled above. immh selects the element width and
|
|
// immh:immb encodes the shift as a bias above it.
|
|
// The immediate-shift translation above takes every vector SHL first when
|
|
// g_translate_all is set, so this one is reached by neither policy. Its gate
|
|
// once kept SHL on the JIT in Hybrid (2.13 ms/frame against 1.95, before
|
|
// ABI 6); see kHybridTranslatesGatedForms for why exports no longer do.
|
|
const bool kTranslateShiftLeftImmediate = g_translate_all;
|
|
if (kTranslateShiftLeftImmediate && (i & 0xBF80FC00) == 0x0F005400) {
|
|
const u32 Q = (i >> 30) & 1;
|
|
const u32 immh = (i >> 19) & 15, immb = (i >> 16) & 7;
|
|
const u32 rn = (i >> 5) & 31, rd = i & 31;
|
|
u32 size = 0;
|
|
bool ok = true;
|
|
if (immh & 8) size = 3;
|
|
else if (immh & 4) size = 2;
|
|
else if (immh & 2) size = 1;
|
|
else if (immh & 1) size = 0;
|
|
else ok = false; // immh 0000 is the modified-immediate space
|
|
// A 64-bit element only exists as 2D.
|
|
if (size == 3 && !Q) ok = false;
|
|
if (ok) {
|
|
const int ebits = 8 << size;
|
|
const u32 shift = ((immh << 3) | immb) - (u32)ebits;
|
|
const int esz = ebits / 8;
|
|
const int bytes = Q ? 16 : 8;
|
|
const int lanes = bytes / esz;
|
|
const std::string uty = "uint" + std::to_string(ebits) + "_t";
|
|
std::string s = "{ " + uty + " _a[" + std::to_string(lanes) + "],_r[" +
|
|
std::to_string(lanes) + "]; ";
|
|
s += "memcpy(_a,c->vreg[" + std::to_string(rn) + "]," + std::to_string(bytes) + "); ";
|
|
s += "for(int _i=0;_i<" + std::to_string(lanes) + ";_i++) _r[_i]=(" + uty + ")((" +
|
|
uty + ")_a[_i]<<" + std::to_string(shift) + "); ";
|
|
s += "c->vreg[" + std::to_string(rd) + "][0]=0; c->vreg[" + std::to_string(rd) +
|
|
"][1]=0; ";
|
|
s += "memcpy(c->vreg[" + std::to_string(rd) + "],_r," + std::to_string(bytes) + "); }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// MOVI/MVNI and ORR/BIC modified immediates. Odd cmode below 1100 reads
|
|
// and combines with the destination; other forms replace it. o2 is zero.
|
|
if ((i & 0x9FF80C00) == 0x0F000400) {
|
|
const u32 Q = (i >> 30) & 1, op = (i >> 29) & 1;
|
|
const u32 cmode = (i >> 12) & 15;
|
|
const u32 imm8 = ((((i >> 16) & 7) << 5) | ((i >> 5) & 31)) & 0xFF;
|
|
const u32 rd = i & 31;
|
|
const u32 hi3 = cmode >> 1;
|
|
bool ok = true;
|
|
u64 imm64 = 0;
|
|
if (hi3 <= 3) {
|
|
const u64 v = (u64)imm8 << (8 * hi3);
|
|
imm64 = (v << 32) | v;
|
|
} else if (hi3 == 4 || hi3 == 5) {
|
|
const u64 h = ((u64)imm8 << (8 * (hi3 - 4))) & 0xFFFF;
|
|
imm64 = (h << 48) | (h << 32) | (h << 16) | h;
|
|
} else if (hi3 == 6) {
|
|
const u64 v = (cmode & 1) ? (((u64)imm8 << 16) | 0xFFFF)
|
|
: (((u64)imm8 << 8) | 0xFF);
|
|
imm64 = (v << 32) | v;
|
|
} else if (cmode == 14) {
|
|
if (op == 0) {
|
|
for (int k = 0; k < 8; ++k) imm64 |= (u64)imm8 << (8 * k);
|
|
} else {
|
|
// Each bit of imm8 expands to a whole byte of the result.
|
|
for (int k = 0; k < 8; ++k) {
|
|
if ((imm8 >> k) & 1) imm64 |= 0xFFULL << (8 * k);
|
|
}
|
|
}
|
|
} else if (cmode == 15) {
|
|
// FMOV (vector, immediate). VFPExpandImm, built at double width and
|
|
// narrowed for the 32-bit form the way the scalar FMOV above does.
|
|
const u32 sgn = (imm8 >> 7) & 1, b6 = (imm8 >> 6) & 1;
|
|
const u64 e11 = ((u64)(b6 ^ 1) << 10) | (b6 ? (0xFFULL << 2) : 0ULL) |
|
|
((imm8 >> 4) & 3);
|
|
const u64 dbits = ((u64)sgn << 63) | (e11 << 52) | ((u64)(imm8 & 0xF) << 48);
|
|
if (op == 0) {
|
|
double dv;
|
|
memcpy(&dv, &dbits, 8);
|
|
const float fv = (float)dv;
|
|
u32 fb;
|
|
memcpy(&fb, &fv, 4);
|
|
imm64 = ((u64)fb << 32) | fb;
|
|
} else if (Q) {
|
|
imm64 = dbits; // the 64-bit form is 2D only
|
|
} else {
|
|
ok = false;
|
|
}
|
|
} else {
|
|
ok = false;
|
|
}
|
|
if (ok) {
|
|
// MVNI inverts, but cmode 1110 is MOVI in both op encodings.
|
|
if (op == 1 && cmode < 14) imm64 = ~imm64;
|
|
char lo[32];
|
|
snprintf(lo, sizeof lo, "0x%llxULL", (unsigned long long)imm64);
|
|
const bool combine = cmode < 12 && (cmode & 1);
|
|
const std::string assignment = combine ? (op ? "&=" : "|=") : "=";
|
|
put("c->vreg[" + std::to_string(rd) + "][0]" + assignment + lo + "; c->vreg[" +
|
|
std::to_string(rd) + "][1]" + (Q ? assignment + lo : "=0") + ";");
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// LD1R-LD4R: load consecutive elements and replicate each into its register.
|
|
// R and opcode<0> select the structure count; S must be zero.
|
|
if ((i & 0xBFDFD000) == 0x0D40C000 || (i & 0xBFC0D000) == 0x0DC0C000) {
|
|
const u32 Q = (i >> 30) & 1;
|
|
const bool post = ((i >> 23) & 1) != 0;
|
|
const u32 rm = (i >> 16) & 31;
|
|
const u32 size = (i >> 10) & 3;
|
|
const u32 rn = (i >> 5) & 31, rt = i & 31;
|
|
const u32 regs = ((((i >> 13) & 1) << 1) | ((i >> 21) & 1)) + 1;
|
|
const int esz = 1 << size;
|
|
const int bytes = Q ? 16 : 8;
|
|
const int lanes = bytes / esz;
|
|
const std::string uty = "uint" + std::to_string(esz * 8) + "_t";
|
|
std::string s = "{ uint64_t _a=" + Xsp(rn) + "; ";
|
|
for (u32 r = 0; r < regs; ++r) {
|
|
const std::string vr = std::to_string((rt + r) & 31);
|
|
s += "{ " + uty + " _e = (" + uty + ")recomp_load" +
|
|
std::to_string(esz * 8) + "(c,_a+" + std::to_string(r * esz) + "); ";
|
|
s += uty + " _r[" + std::to_string(lanes) + "]; ";
|
|
s += "for(int _i=0;_i<" + std::to_string(lanes) + ";_i++) _r[_i]=_e; ";
|
|
s += "c->vreg[" + vr + "][0]=0; c->vreg[" + vr + "][1]=0; ";
|
|
s += "memcpy(c->vreg[" + vr + "],_r," + std::to_string(bytes) + "); } ";
|
|
}
|
|
if (post) {
|
|
const std::string step =
|
|
(rm == 31) ? (std::to_string(regs * esz) + "ULL") : ("c->x[" + std::to_string(rm) + "]");
|
|
s += "c->x[" + std::to_string(rn) + "]=_a+" + step + "; ";
|
|
}
|
|
s += "}";
|
|
put(s);
|
|
return true;
|
|
}
|
|
|
|
// Scalar FP <-> fixed-point conversions (S/D, W/X). A strict-static
|
|
// export cannot leave these to a JIT. Do not gate architectural coverage
|
|
// on the former hybrid replay-performance switch.
|
|
// Use integer significands, not host FP scaling/casts: preserve guest FZ,
|
|
// rounding and cumulative status without intermediate overflow/rounding.
|
|
if ((i & 0x7F200000U) == 0x1E000000U) {
|
|
const u32 sf = i >> 31, ftype = (i >> 22) & 3;
|
|
const u32 rmode = (i >> 19) & 3, opcode = (i >> 16) & 7;
|
|
const u32 fbits = 64 - ((i >> 10) & 0x3F);
|
|
const u32 rn = (i >> 5) & 31, rd = i & 31;
|
|
const bool shaped = (ftype == 0 || ftype == 1) && (sf || fbits <= 32);
|
|
const bool to_fixed = rmode == 3 && (opcode == 0 || opcode == 1);
|
|
const bool to_float = rmode == 0 && (opcode == 2 || opcode == 3);
|
|
if (!shaped || (!to_fixed && !to_float)) {
|
|
put_unhandled();
|
|
return true;
|
|
}
|
|
const std::string fw = ftype == 1 ? "64" : "32";
|
|
const std::string iw = sf ? "64" : "32";
|
|
const std::string is_signed = (opcode & 1) ? "0" : "1";
|
|
const std::string args = "," + fw + "," + iw + "," + std::to_string(fbits) +
|
|
"," + is_signed + ",c->fpcr,&c->fpsr)";
|
|
if (to_fixed) {
|
|
const std::string call = "recomp_fp_to_fixed(c->vreg[" +
|
|
std::to_string(rn) + "][0]" + args;
|
|
// Rd=31 is WZR/XZR, NOT SP. Conversion exceptions still happen.
|
|
put(rd == 31 ? "(void)" + call + ";"
|
|
: "c->x[" + std::to_string(rd) + "]=" + call + ";");
|
|
} else {
|
|
put("{ uint64_t _r=recomp_fixed_to_fp(" + Xz(rn) + args +
|
|
"; c->vreg[" + std::to_string(rd) + "][0]=_r; c->vreg[" +
|
|
std::to_string(rd) + "][1]=0; }");
|
|
}
|
|
return true;
|
|
}
|
|
|
|
// FP <-> integer conversions and FMOV between register files. These share
|
|
// bit 21 with the FP arithmetic forms and are distinguished by bits 15..10
|
|
// being zero, so they must be decoded ahead of the arithmetic/compare block
|
|
// below or every one of them is mis-decoded as FCMP.
|
|
if ((i & 0x5F200000) == 0x1E200000 && ((i >> 21) & 1) && ((i >> 10) & 0x3F) == 0) {
|
|
const u32 sf = i >> 31, ftype = (i >> 22) & 3;
|
|
const u32 rmode = (i >> 19) & 3, opcode = (i >> 16) & 7;
|
|
const u32 rn = (i >> 5) & 31, rd = i & 31;
|
|
if (ftype == 0 || ftype == 1) {
|
|
const bool dbl = (ftype == 1);
|
|
const int fsz = dbl ? 8 : 4;
|
|
const std::string zero_d = "c->vreg[" + std::to_string(rd) + "][0]=0; c->vreg[" +
|
|
std::to_string(rd) + "][1]=0; ";
|
|
|
|
// SCVTF / UCVTF: integer register -> FP register, rounded in the
|
|
// guest FPCR mode with IXC, as the fixed-point forms with no
|
|
// fraction bits.
|
|
if (rmode == 0 && (opcode == 2 || opcode == 3)) {
|
|
put("{ uint64_t _r=recomp_fixed_to_fp(" + Xz(rn) + "," + std::to_string(fsz * 8) +
|
|
"," + (sf ? "64" : "32") + ",0," + (opcode == 2 ? "1" : "0") +
|
|
",c->fpcr,&c->fpsr); " + zero_d + "c->vreg[" + std::to_string(rd) + "][0]=_r; }");
|
|
return true;
|
|
}
|
|
|
|
// FCVT{N,P,M,Z}{S,U} and FCVTA{S,U}: FP register -> integer register.
|
|
// rmode names the rounding mode; FCVTA{S,U} is the odd one out,
|
|
// encoded as opcode 4/5 with rmode 0 and rounding halfway cases away
|
|
// from zero. Round, then saturate (NaN gives 0), in integer fields:
|
|
// a bare C cast is undefined out of range, and host rounding
|
|
// follows the host FP mode. IOC/IXC/IDC as the architecture.
|
|
const bool cvt_away = (rmode == 0 && (opcode == 4 || opcode == 5));
|
|
if (opcode == 0 || opcode == 1 || cvt_away) {
|
|
const bool is_signed = cvt_away ? (opcode == 4) : (opcode == 0);
|
|
const std::string call = "recomp_fp_to_int(c->vreg[" + std::to_string(rn) + "][0]&" +
|
|
(dbl ? "UINT64_MAX" : "UINT64_C(0xffffffff)") + "," + std::to_string(fsz * 8) + "," +
|
|
(sf ? "64" : "32") + "," + (is_signed ? "1" : "0") + "," +
|
|
std::to_string(cvt_away ? 4u : rmode) + ",c->fpcr,&c->fpsr)";
|
|
// Rd=31 is WZR/XZR; the conversion's exceptions still happen.
|
|
put(rd == 31 ? "(void)" + call + ";" : "c->x[" + std::to_string(rd) + "]=" + call + ";");
|
|
return true;
|
|
}
|
|
|
|
// FMOV between a general register and an FP register.
|
|
if (rmode == 0 && (opcode == 6 || opcode == 7)) {
|
|
if (opcode == 7) { // GPR -> FP
|
|
put("{ " + zero_d + "memcpy(&c->vreg[" + std::to_string(rd) + "][0],&" +
|
|
(rn == 31 ? std::string("(uint64_t){0}") : "c->x[" + std::to_string(rn) + "]") +
|
|
"," + std::to_string(fsz) + "); }");
|
|
return true;
|
|
}
|
|
if (rd != 31) { // FP -> GPR
|
|
put("{ uint64_t _r=0; memcpy(&_r,&c->vreg[" + std::to_string(rn) + "][0]," +
|
|
std::to_string(fsz) + "); c->x[" + std::to_string(rd) + "]=_r; }");
|
|
return true;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// FCSEL - the floating-point conditional select. Shares the scalar FP
|
|
// encoding but is picked out by bits 11..10 being 11, so it is matched
|
|
// before the arithmetic and compare forms below.
|
|
if ((i & 0x5F200C00) == 0x1E200C00) {
|
|
const u32 ftype = (i >> 22) & 3;
|
|
const u32 rm = (i >> 16) & 31, cond = (i >> 12) & 15;
|
|
const u32 rn = (i >> 5) & 31, rd = i & 31;
|
|
if (ftype == 0 || ftype == 1) {
|
|
const int fsz = (ftype == 1) ? 8 : 4;
|
|
// Selecting whole register halves rather than reinterpreting the
|
|
// value keeps this exact for NaN payloads too.
|
|
put("{ uint64_t _r = " + Cond(cond) + " ? c->vreg[" +
|
|
std::to_string(rn) + "][0] : c->vreg[" + std::to_string(rm) + "][0]; " +
|
|
(fsz == 4 ? "_r &= 0xFFFFFFFFULL; " : "") + "c->vreg[" + std::to_string(rd) +
|
|
"][0]=_r; c->vreg[" + std::to_string(rd) + "][1]=0; }");
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// Data-processing (1 source): RBIT, REV16, REV32, REV, CLZ and CLS.
|
|
// The mask must stop at bit 16: bits 15..10 are the opcode being switched
|
|
// on below, so including them would pin this to RBIT alone.
|
|
if ((i & 0x7FFF0000) == 0x5AC00000) {
|
|
const u32 sf = i >> 31, opcode = (i >> 10) & 0x3F;
|
|
const u32 rn = (i >> 5) & 31, rd = i & 31;
|
|
const u32 width = sf ? 64 : 32;
|
|
if (rd != 31) {
|
|
const std::string w = std::to_string(width);
|
|
const std::string src =
|
|
sf ? Xz(rn) : ("(" + Xz(rn) + " & 0xFFFFFFFFULL)");
|
|
std::string s;
|
|
switch (opcode) {
|
|
case 0: // RBIT - reverse every bit
|
|
s = "{ uint64_t _v=" + src + ", _r=0; for(int _i=0;_i<" + w +
|
|
";_i++){ _r=(_r<<1)|((_v>>_i)&1ULL); } c->x[" + std::to_string(rd) + "]=_r; }";
|
|
break;
|
|
case 1: // REV16 - reverse bytes within each halfword
|
|
s = "{ uint64_t _v=" + src + ", _r=0; for(int _i=0;_i<" + w +
|
|
"/8;_i++){ int _p=(_i^1)*8; _r|=((_v>>(_i*8))&0xFFULL)<<_p; } c->x[" +
|
|
std::to_string(rd) + "]=_r; }";
|
|
break;
|
|
case 2: // REV32 on 64-bit, REV on 32-bit: bytes within each word
|
|
if (!sf) {
|
|
s = "{ uint64_t _v=" + src + ", _r=0; for(int _i=0;_i<4;_i++){ "
|
|
"_r|=((_v>>(_i*8))&0xFFULL)<<((3-_i)*8); } c->x[" +
|
|
std::to_string(rd) + "]=_r; }";
|
|
} else {
|
|
s = "{ uint64_t _v=" + src + ", _r=0; for(int _i=0;_i<8;_i++){ "
|
|
"int _p=((_i&4)|(3-(_i&3)))*8; _r|=((_v>>(_i*8))&0xFFULL)<<_p; } c->x[" +
|
|
std::to_string(rd) + "]=_r; }";
|
|
}
|
|
break;
|
|
case 3: // REV (64-bit only) - reverse all bytes
|
|
if (sf) {
|
|
s = "{ uint64_t _v=" + src + ", _r=0; for(int _i=0;_i<8;_i++){ "
|
|
"_r|=((_v>>(_i*8))&0xFFULL)<<((7-_i)*8); } c->x[" +
|
|
std::to_string(rd) + "]=_r; }";
|
|
}
|
|
break;
|
|
case 4: // CLZ
|
|
s = "{ uint64_t _v=" + src + "; int _n=0; while(_n<" + w + " && !((_v>>(" + w +
|
|
"-1-_n))&1ULL)) _n++; c->x[" + std::to_string(rd) + "]=(uint64_t)_n; }";
|
|
break;
|
|
case 5: // CLS - leading sign bits, not counting the sign itself
|
|
s = "{ uint64_t _v=" + src + "; int _n=0; while(_n<" + w +
|
|
"-1 && (((_v>>(" + w + "-1-_n))&1ULL)==((_v>>(" + w +
|
|
"-2-_n))&1ULL))) _n++; c->x[" + std::to_string(rd) + "]=(uint64_t)_n; }";
|
|
break;
|
|
default: break;
|
|
}
|
|
if (!s.empty()) { put(s); return true; }
|
|
}
|
|
}
|
|
|
|
// MRS/MSR against the small set of system registers a user-mode program
|
|
// actually touches. Anything else stays on the fallback rather than being
|
|
// silently invented.
|
|
// SYS: 1101 0101 0000 1 op1 CRn CRm op2 Rt - the DC/IC/AT/TLBI space, which
|
|
// the MRS/MSR handler below does not cover (that one matches op0=3 only).
|
|
if ((i & 0xFFF80000) == 0xD5080000) {
|
|
const u32 sys_op1 = (i >> 16) & 7;
|
|
const u32 sys_crn = (i >> 12) & 0xF;
|
|
const u32 sys_crm = (i >> 8) & 0xF;
|
|
const u32 sys_op2 = (i >> 5) & 7;
|
|
|
|
// DC ZVA. The host config reports DCZID_EL0=4: 16 words, or 64 bytes.
|
|
// The operand may point anywhere within that naturally aligned block.
|
|
// Use guest-memory helpers so hosted writes retain normal translation.
|
|
if (sys_op1 == 3 && sys_crn == 7 && sys_crm == 4 && sys_op2 == 1) {
|
|
put("{ uint64_t _base=" + Xz(i & 31) +
|
|
"&~UINT64_C(63); for(unsigned _off=0;_off<64;_off+=8) "
|
|
"recomp_store64(c,_base+_off,UINT64_C(0)); }");
|
|
return true;
|
|
}
|
|
|
|
if ((i & 0xFFFFFFE0) == 0xD50B7520) {
|
|
snprintf(buf, sizeof buf, "c->pc=g_module_base+0x%llxULL; ", (unsigned long long)pc);
|
|
put(std::string(buf) + "recomp_ic_ivau(c," + Xz(i & 31) + ",g_recomp_guard_host_v2); return;");
|
|
return false;
|
|
}
|
|
// Data cache clean / invalidate by VA: DC CVAC, CVAU, CVAP, CVADP,
|
|
// CIVAC. Guest memory is host memory here with no emulated cache
|
|
// hierarchy, so there is nothing to write back or discard.
|
|
//
|
|
// This is not an assumption about what is safe to skip: suyu never sets
|
|
// Dynarmic's hook_data_cache_operations, so it stays false and the
|
|
// fallback JIT compiles these to nothing. Emitting nothing matches the
|
|
// other engine exactly.
|
|
//
|
|
// Deliberately narrow. DC ZVA above writes memory, whereas the
|
|
// following neighbour requires instruction-cache handling:
|
|
// IC IVAU (CRm=5) invalidates the instruction cache, which suyu does
|
|
// act on (InvalidateCacheRange, then halts the JIT).
|
|
const bool is_dc_clean_invalidate =
|
|
sys_op1 == 3 && sys_crn == 7 && sys_op2 == 1 &&
|
|
(sys_crm == 10 || sys_crm == 11 || sys_crm == 12 || sys_crm == 13 || sys_crm == 14);
|
|
if (is_dc_clean_invalidate) {
|
|
put("/* dc clean/invalidate: no cache to maintain */");
|
|
return true;
|
|
}
|
|
// Everything else in this space - the IC family, AT, TLBI -
|
|
// goes to the fallback engine.
|
|
}
|
|
if ((i & 0xFFF00000) == 0xD5300000 || (i & 0xFFF00000) == 0xD5100000) {
|
|
const bool is_read = ((i >> 21) & 1) != 0; // MRS reads, MSR writes
|
|
const u32 sysreg = (i >> 5) & 0x7FFF;
|
|
const u32 rt = i & 31;
|
|
// TPIDR_EL0 (thread pointer) and TPIDRRO_EL0 are the ones that matter
|
|
// for ordinary code; both live in the context already. They are kept
|
|
// in *separate* fields deliberately: TPIDR_EL0 is the guest's own
|
|
// thread pointer, while TPIDRRO_EL0 is written by the kernel and holds
|
|
// the thread-local region whose first bytes are the IPC message
|
|
// buffer. Folding them together corrupts both.
|
|
// NZCV. The flags live in the context as separate fields, so this is
|
|
// a pack and an unpack rather than a plain load and store.
|
|
constexpr u32 kNzcv = 0x5A10;
|
|
if (sysreg == kNzcv) {
|
|
if (is_read) {
|
|
if (rt != 31) {
|
|
put("c->x[" + std::to_string(rt) +
|
|
"] = ((uint64_t)(c->n&1)<<31)|((uint64_t)(c->z&1)<<30)|"
|
|
"((uint64_t)(c->c&1)<<29)|((uint64_t)(c->v&1)<<28);");
|
|
}
|
|
} else {
|
|
put("{ uint64_t _f=" + Xz(rt) +
|
|
"; c->n=(_f>>31)&1; c->z=(_f>>30)&1; c->c=(_f>>29)&1; c->v=(_f>>28)&1; }");
|
|
}
|
|
return true;
|
|
}
|
|
constexpr u32 kTpidrEl0 = 0x5E82;
|
|
constexpr u32 kTpidrroEl0 = 0x5E83;
|
|
if (sysreg == kTpidrEl0) {
|
|
if (is_read) {
|
|
if (rt != 31) put("c->x[" + std::to_string(rt) + "] = c->tpidr_el0;");
|
|
} else {
|
|
put("c->tpidr_el0 = " + Xz(rt) + ";");
|
|
}
|
|
return true;
|
|
}
|
|
// FPCR/FPSR. sysreg here is o0 op1 CRn CRm op2, so FPCR (op0=3, op1=3,
|
|
// CRn=4, CRm=4, op2=0) packs to 0x5A20 and FPSR to 0x5A21.
|
|
//
|
|
// These are stored and returned rather than acted on: the generated C
|
|
// computes in the host's default rounding mode and nothing here changes
|
|
// that. Keeping the value is still strictly better than dropping it -
|
|
// code that saves FPCR, changes it, and restores it now round-trips,
|
|
// and the value survives a transition to the JIT, which does honour it.
|
|
// A recompiler that actually implemented the rounding modes would set
|
|
// the host FP mode here instead.
|
|
constexpr u32 kFpcr = 0x5A20;
|
|
constexpr u32 kFpsr = 0x5A21;
|
|
if (sysreg == kFpcr || sysreg == kFpsr) {
|
|
const char* field = (sysreg == kFpcr) ? "fpcr" : "fpsr";
|
|
if (g_emit_fpx && sysreg == kFpcr) {
|
|
// FPX1: bit 32 of fpcr is the host's kill switch. The guest
|
|
// register is 32 bits, so it never sees the bit or clears it.
|
|
if (is_read) {
|
|
if (rt != 31) {
|
|
put("c->x[" + std::to_string(rt) + "] = c->fpcr & 0xffffffffULL;");
|
|
}
|
|
} else {
|
|
put("c->fpcr = (" + Xz(rt) + " & 0xffffffffULL) | (c->fpcr & RECOMP_FPX_INHIBIT);");
|
|
}
|
|
return true;
|
|
}
|
|
if (is_read) {
|
|
if (rt != 31) {
|
|
put("c->x[" + std::to_string(rt) + "] = c->" + std::string(field) + ";");
|
|
}
|
|
} else {
|
|
put("c->" + std::string(field) + " = " + Xz(rt) + ";");
|
|
}
|
|
return true;
|
|
}
|
|
// CNTPCT_EL0 / CNTVCT_EL0 / CNTFRQ_EL0. Packed the same way as the
|
|
// registers above: o0 op1 CRn CRm op2, so CNTPCT (op0=3, op1=3, CRn=14,
|
|
// CRm=0, op2=1) is 0x5F01.
|
|
//
|
|
// The counter is read from the emulator's own timing source, the same
|
|
// one the fallback JIT uses (DynarmicCallbacks64::GetCNTPCT ->
|
|
// CoreTiming::GetClockTicks). Two independent clocks would let the
|
|
// guest observe time moving backwards across an engine transition.
|
|
constexpr u32 kCntfrqEl0 = 0x5F00;
|
|
constexpr u32 kCntpctEl0 = 0x5F01;
|
|
constexpr u32 kCntvctEl0 = 0x5F02;
|
|
if (sysreg == kCntpctEl0 || sysreg == kCntvctEl0) {
|
|
// CNTVCT is the virtual counter. With no hypervisor offset it reads
|
|
// the same as the physical one, which is what the guest sees on a
|
|
// Switch and what the JIT reports.
|
|
if (is_read) {
|
|
if (rt != 31) {
|
|
put("c->x[" + std::to_string(rt) + "] = recomp_cntpct(c);");
|
|
} else {
|
|
put("(void)recomp_cntpct(c);");
|
|
}
|
|
} else {
|
|
// Writing the counter traps at EL0; swallow it.
|
|
put("(void)" + Xz(rt) + ";");
|
|
}
|
|
return true;
|
|
}
|
|
if (sysreg == kCntfrqEl0) {
|
|
if (is_read) {
|
|
if (rt != 31) {
|
|
// Core::Hardware::CNTFREQ. Fixed on this platform.
|
|
put("c->x[" + std::to_string(rt) + "] = 19200000ULL;");
|
|
}
|
|
} else {
|
|
put("(void)" + Xz(rt) + ";");
|
|
}
|
|
return true;
|
|
}
|
|
// CTR_EL0, the cache type register (op0=3 op1=3 CRn=0 CRm=0 op2=1).
|
|
//
|
|
// A constant, and read constantly: 31% of every transition to the
|
|
// fallback engine on a matrix-heavy title was this one instruction, most
|
|
// of a matrix-heavy title's remaining gap spent marshalling the whole
|
|
// context into dynarmic to fetch a number that never changes.
|
|
//
|
|
// The value is the one suyu configures dynarmic with
|
|
// (arm_dynarmic_64.cpp: config.ctr_el0), not a plausible-looking
|
|
// constant. Code that reads a cache line size once and relies on it
|
|
// later must not see it change when a thread crosses between engines.
|
|
constexpr u32 kCtrEl0 = 0x5801;
|
|
if (sysreg == kCtrEl0) {
|
|
if (is_read) {
|
|
if (rt != 31) {
|
|
put("c->x[" + std::to_string(rt) + "] = 0x8444c004ULL;");
|
|
}
|
|
} else {
|
|
// Read-only at EL0; a write traps on hardware.
|
|
put("(void)" + Xz(rt) + ";");
|
|
}
|
|
return true;
|
|
}
|
|
if (sysreg == kTpidrroEl0) {
|
|
if (is_read) {
|
|
if (rt != 31) put("c->x[" + std::to_string(rt) + "] = c->tpidrro_el0;");
|
|
} else {
|
|
// Read-only at EL0; a write traps on hardware. Swallow it
|
|
// rather than letting it destroy the kernel's TLS pointer.
|
|
put("(void)" + Xz(rt) + ";");
|
|
}
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// Scalar S/D register FP comparisons. Ordered comparisons signal every
|
|
// NaN; equality signals only sNaN. Baseline FZ/IDC, no host FP operations.
|
|
{
|
|
const u32 key = i & 0xFF20FC00;
|
|
const bool eq = key == 0x5E20E400 && !(i & 0x00800000);
|
|
const bool ordered = key == 0x7E20E400;
|
|
if (eq || ordered) {
|
|
const bool dbl = (i & 0x00400000) != 0, gt = (i & 0x00800000) != 0;
|
|
const unsigned rd=i&31,rn=(i>>5)&31,rm=(i>>16)&31;
|
|
const std::string ct=dbl?"uint64_t":"uint32_t",sz=dbl?"8":"4";
|
|
put("{ "+ct+" _aa,_bb;memcpy(&_aa,c->vreg["+std::to_string(rn)+"],"+sz+
|
|
");memcpy(&_bb,c->vreg["+std::to_string(rm)+"],"+sz+
|
|
");uint64_t _a=_aa,_b=_bb,_r=0;const uint64_t _sign="+
|
|
(dbl?"0x8000000000000000ULL":"0x80000000ULL")+",_exp="+
|
|
(dbl?"0x7ff0000000000000ULL":"0x7f800000ULL")+",_frac="+
|
|
(dbl?"0xfffffffffffffULL":"0x7fffffULL")+",_quiet="+
|
|
(dbl?"0x8000000000000ULL":"0x400000ULL")+";"
|
|
"if(c->fpcr&(1ULL<<24)) {if(!(_a&_exp)&&(_a&_frac)) {_a&=_sign;c->fpsr|=128;}"
|
|
"if(!(_b&_exp)&&(_b&_frac)) {_b&=_sign;c->fpsr|=128;}}"
|
|
"int _an=(_a&_exp)==_exp&&(_a&_frac),_bn=(_b&_exp)==_exp&&(_b&_frac);"
|
|
"if(_an||_bn) { if("+std::string(eq?"(_an&&!(_a&_quiet))||(_bn&&!(_b&_quiet))":"1")+
|
|
")c->fpsr|=1; }else {int _equal=_a==_b||!((_a|_b)&~_sign);"
|
|
"int _greater=!_equal&&(((_a^_b)&_sign)?!(_a&_sign):((_a&_sign)?_a<_b:_a>_b));"
|
|
"(void)_quiet;(void)_greater; if("+std::string(eq?"_equal":gt?"_greater":"_equal||_greater")+
|
|
")_r="+(dbl?"UINT64_MAX":"UINT32_MAX")+";}c->vreg["+std::to_string(rd)+
|
|
"][0]=_r;c->vreg["+std::to_string(rd)+"][1]=0;}");
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// Scalar FRECPS/FRSQRTS S/D, evaluated exactly before one rounding.
|
|
if ((i & 0xFF20FC00) == 0x5E20FC00) {
|
|
const bool dbl=(i&0x00400000)!=0,half=(i&0x00800000)!=0;
|
|
const unsigned rd=i&31,rn=(i>>5)&31,rm=(i>>16)&31;
|
|
const std::string ct=dbl?"uint64_t":"uint32_t",sz=dbl?"8":"4";
|
|
put("{ "+ct+" _aa,_bb;memcpy(&_aa,c->vreg["+std::to_string(rn)+"],"+sz+
|
|
");memcpy(&_bb,c->vreg["+std::to_string(rm)+"],"+sz+
|
|
");"+EmitFPStepValue(dbl,half)+"c->vreg["+std::to_string(rd)+"][0]=_v;c->vreg["+
|
|
std::to_string(rd)+"][1]=0;}");
|
|
return true;
|
|
}
|
|
|
|
// FSQRT vector/scalar and FABD vector/scalar, S/D. Integer significands
|
|
// make rounding independent of host FP modes. Baseline DN/FZ/RMode and
|
|
// FPSR flags are supported; FP access/exception traps and FEAT_AFP are not.
|
|
{
|
|
const bool sqrt_scalar = (i & 0xFFBFFC00) == 0x1E21C000;
|
|
const bool sqrt_vec = (i & 0xBFBFFC00) == 0x2EA1F800 || sqrt_scalar;
|
|
const bool abd_vec = (i & 0xBFA0FC00) == 0x2EA0D400;
|
|
const bool abd_scalar = (i & 0xFFA0FC00) == 0x7EA0D400;
|
|
const bool dbl = ((i >> 22) & 1) != 0;
|
|
const bool q = ((i >> 30) & 1) != 0;
|
|
if (((sqrt_vec || abd_vec) && (!dbl || q || sqrt_scalar)) || abd_scalar) {
|
|
const unsigned rd = i & 31, rn = (i >> 5) & 31, rm = (i >> 16) & 31;
|
|
const unsigned lanes = abd_scalar || sqrt_scalar ? 1 : (q ? 16 : 8) / (dbl ? 8 : 4);
|
|
const std::string ct = dbl ? "uint64_t" : "uint32_t";
|
|
put("{ " + ct + " _n[" + std::string(dbl ? "2" : "4") + "],_m[" +
|
|
(dbl ? "2" : "4") + "],_r[" + (dbl ? "2" : "4") +
|
|
"]={0}; memcpy(_n,c->vreg[" + std::to_string(rn) +
|
|
"],16); memcpy(_m,c->vreg[" + std::to_string(rm) + "],16);");
|
|
if (sqrt_vec && FpxOutline()) {
|
|
put("for(unsigned _lane=0;_lane<" + std::to_string(lanes) +
|
|
";++_lane) { uint64_t _a=_n[_lane],_v=0;" + EmitFPOutlined(dbl, "sqrt") +
|
|
"_r[_lane]=(" + ct + ")(_v); } memcpy(c->vreg[" + std::to_string(rd) + "],_r,16); }");
|
|
return true;
|
|
}
|
|
put("for(unsigned _lane=0;_lane<" + std::to_string(lanes) +
|
|
";++_lane) { uint64_t _a=_n[_lane],_b=" +
|
|
std::string(sqrt_vec ? "0" : "_m[_lane]") + ",_v=0;" +
|
|
(sqrt_vec ? EmitFPNativeValue(dbl, "sqrt") : std::string()) +
|
|
EmitFPSqrtAbdHead(dbl, sqrt_vec));
|
|
if (sqrt_vec) {
|
|
put(EmitFPSqrtTail(dbl));
|
|
} else {
|
|
put("else " + EmitFPAddSubValue(dbl, true));
|
|
}
|
|
put((sqrt_vec ? EmitFPNativeSuffix(dbl, "sqrt") : std::string()) +
|
|
"_r[_lane]=(" + ct + ")(_v" + (sqrt_vec ? std::string("") : "&~_sign") +
|
|
"); } memcpy(c->vreg[" + std::to_string(rd) + "],_r,16); }");
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// SIMD S/D min/max: Arm FPMin/FPMax/FPMinNum/FPMaxNum, with bitwise
|
|
// ordering so host NaN, signed-zero and denormal modes cannot intervene.
|
|
// Implements baseline FPCR.DN/FZ and cumulative FPSR.IOC/IDC. FP exception
|
|
// trap delivery and later FEAT_AFP modes are not implemented here.
|
|
{
|
|
const u32 vec_key = i & 0x9F20FC00;
|
|
const u32 pair_key = i & 0xFF3FFC00;
|
|
const u32 reduce_key = i & 0xFF7FFC00;
|
|
const bool vec = vec_key == 0x0E20F400 || vec_key == 0x0E20C400;
|
|
const bool scalar_pair = pair_key == 0x7E30F800 || pair_key == 0x7E30C800;
|
|
const bool reduce = reduce_key == 0x6E30F800 || reduce_key == 0x6E30C800;
|
|
const bool dbl = ((i >> 22) & 1) != 0;
|
|
const bool q = ((i >> 30) & 1) != 0;
|
|
if ((vec && (!dbl || q)) || scalar_pair || reduce) {
|
|
const bool minimum = ((i >> 23) & 1) != 0;
|
|
const bool numeric = ((i >> 12) & 3) == 0;
|
|
const bool pair = vec && ((i >> 29) & 1);
|
|
const u32 rn = (i >> 5) & 31, rm = (i >> 16) & 31, rd = i & 31;
|
|
const unsigned lanes = dbl ? 2 : 4;
|
|
const unsigned active = q ? lanes : lanes / 2;
|
|
const std::string ct = dbl ? "uint64_t" : "uint32_t";
|
|
put("{ " + ct + " _n[" + std::to_string(lanes) + "],_m[" +
|
|
std::to_string(lanes) + "],_r[" + std::to_string(lanes) +
|
|
"]={0}; memcpy(_n,c->vreg[" + std::to_string(rn) +
|
|
"],16); memcpy(_m,c->vreg[" + std::to_string(rm) + "],16);");
|
|
const auto element = [](const char* array, unsigned e) {
|
|
return std::string(array) + "[" + std::to_string(e) + "]";
|
|
};
|
|
const auto emit_pair = [&](const std::string& a, const std::string& b,
|
|
const std::string& dest) {
|
|
put(EmitFPMinMax(dbl, minimum, numeric, a, b, dest + "=(" + ct + ")_v;"));
|
|
};
|
|
if (reduce) {
|
|
// Architectural reduction is a balanced tree, not a left fold:
|
|
// the distinction affects signaling/quiet NaN selection.
|
|
emit_pair("_n[0]", "_n[1]", "_n[0]");
|
|
emit_pair("_n[2]", "_n[3]", "_n[2]");
|
|
emit_pair("_n[0]", "_n[2]", "_r[0]");
|
|
} else if (scalar_pair) {
|
|
emit_pair("_n[0]", "_n[1]", "_r[0]");
|
|
} else {
|
|
for (unsigned e = 0; e < active; ++e) {
|
|
const char* source = e < active / 2 ? "_n" : "_m";
|
|
emit_pair(pair ? element(source, 2 * (e % (active / 2))) : element("_n", e),
|
|
pair ? element(source, 2 * (e % (active / 2)) + 1) : element("_m", e),
|
|
element("_r", e));
|
|
}
|
|
}
|
|
put("memcpy(c->vreg[" + std::to_string(rd) + "],_r,16); }");
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// Absolute FP comparisons. IEEE magnitudes order as unsigned integers;
|
|
// both quiet and signaling NaNs raise IOC for ordered GE/GT comparisons.
|
|
{
|
|
const bool scalar = (i & 0xFF20FC00) == 0x7E20EC00;
|
|
const bool vector = (i & 0xBF20FC00) == 0x2E20EC00;
|
|
const bool dbl = (i & (1U << 22)) != 0, q = (i & (1U << 30)) != 0;
|
|
if (scalar || (vector && (!dbl || q))) {
|
|
const unsigned rd=i&31, rn=(i>>5)&31, rm=(i>>16)&31;
|
|
const unsigned lanes=dbl?2:4, active=scalar?1:q?lanes:lanes/2;
|
|
const std::string ct=dbl?"uint64_t":"uint32_t";
|
|
put("{ " + ct + " _n[" + std::to_string(lanes) + "],_m[" +
|
|
std::to_string(lanes) + "],_r[" + std::to_string(lanes) +
|
|
"]={0}; memcpy(_n,c->vreg[" + std::to_string(rn) +
|
|
"],16); memcpy(_m,c->vreg[" + std::to_string(rm) + "],16);");
|
|
put("for(unsigned _j=0;_j<" + std::to_string(active) + ";++_j) {"
|
|
" uint64_t _a=_n[_j]&" + std::string(dbl?"0x7fffffffffffffffULL":"0x7fffffffULL") +
|
|
",_b=_m[_j]&" + (dbl?"0x7fffffffffffffffULL":"0x7fffffffULL") +
|
|
"; const uint64_t _exp=" + (dbl?"0x7ff0000000000000ULL":"0x7f800000ULL") +
|
|
",_min=" + (dbl?"0x10000000000000ULL":"0x800000ULL") + ";"
|
|
" if(c->fpcr&(1ULL<<24)) { if(_a&&_a<_min) { _a=0; c->fpsr|=128; }"
|
|
" if(_b&&_b<_min) { _b=0; c->fpsr|=128; } }"
|
|
" if(_a>_exp||_b>_exp) c->fpsr|=1; else if(_a" +
|
|
std::string((i&(1U<<23))?">":">=") + "_b) _r[_j]=~(" + ct + ")0;"
|
|
" } memcpy(c->vreg[" + std::to_string(rd) + "],_r,16); }");
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// Vector float/fixed conversions and nearest integral conversions. Keep
|
|
// saturation and rounding in integer fields; invalid suppresses inexact.
|
|
{
|
|
const u32 key=i&0x9FBFFC00, fixed_key=i&0x9F80FC00;
|
|
const u32 scalar_key=i&0xDFBFFC00;
|
|
const bool scalar=scalar_key==0x5E21A800||scalar_key==0x5E21C800;
|
|
const bool nearest=key==0x0E21A800||scalar_key==0x5E21A800;
|
|
const bool away=key==0x0E21C800||scalar_key==0x5E21C800;
|
|
const bool fixed=fixed_key==0x0F00FC00, to_float=fixed_key==0x0F00E400;
|
|
const unsigned imm=(i>>16)&127;
|
|
const bool dbl=(fixed||to_float)?(imm&64)!=0:(i&(1U<<22))!=0;
|
|
const bool q=(i&(1U<<30))!=0, uns=(i&(1U<<29))!=0;
|
|
if((nearest||away||((fixed||to_float)&&imm>=32))&&(!dbl||q)) {
|
|
const unsigned width=dbl?64:32,frac=dbl?52:23,bias=dbl?1023:127;
|
|
const unsigned fbits=(fixed||to_float)?2*width-imm:0;
|
|
const unsigned rd=i&31,rn=(i>>5)&31,active=scalar?1:q?128/width:64/width;
|
|
const std::string ct=dbl?"uint64_t":"uint32_t";
|
|
const auto lit=[](uint64_t v){return std::to_string(v)+"ULL";};
|
|
put("{ "+ct+" _src["+std::to_string(128/width)+"],_dst["+
|
|
std::to_string(128/width)+"]={0}; memcpy(_src,c->vreg["+std::to_string(rn)+"],16);"
|
|
" for(unsigned _j=0;_j<"+std::to_string(active)+";++_j) { uint64_t _v=_src[_j],_r=0;");
|
|
if(to_float) {
|
|
put("int _neg="+std::string(uns?"0":"!!(_v&"+lit(1ULL<<(width-1))+")")+";"
|
|
" uint64_t _mag=_neg?((~_v+1)&"+lit(dbl?~0ULL:0xffffffffULL)+"):_v;"
|
|
" if(_mag) { unsigned _top=0; uint64_t _scan=_mag; while(_scan>>1) { _scan>>=1; ++_top; }"
|
|
" int _shift=(int)_top-"+std::to_string(frac)+";"
|
|
" uint64_t _whole=_shift>0?_mag>>_shift:_mag<<(-_shift);"
|
|
" uint64_t _lost=_shift>0?_mag&((1ULL<<_shift)-1):0; unsigned _mode=(unsigned)((c->fpcr>>22)&3);"
|
|
" int _up=_lost&&(_mode==0?(_lost>(1ULL<<(_shift-1))||(_lost==(1ULL<<(_shift-1))&&(_whole&1))):"
|
|
" _mode==1?!_neg:_mode==2?_neg:0); _whole+=_up; if(_whole>="+lit(1ULL<<(frac+1))+") { _whole>>=1; ++_top; }"
|
|
" _r=((uint64_t)_neg<<"+std::to_string(width-1)+")|((uint64_t)((int)_top-"+
|
|
std::to_string(fbits)+"+"+std::to_string(bias)+")<<"+std::to_string(frac)+")|(_whole&"+
|
|
lit((1ULL<<frac)-1)+"); if(_lost)c->fpsr|=16; }");
|
|
} else {
|
|
put("unsigned _exp=(unsigned)((_v>>"+std::to_string(frac)+")&"+std::to_string(2*bias+1)+");"
|
|
" uint64_t _mant=_v&"+lit((1ULL<<frac)-1)+"; int _neg=!!(_v&"+lit(1ULL<<(width-1))+");"
|
|
" int _invalid=0,_inexact=0; uint64_t _mag=0,_limit="+
|
|
std::string(uns?lit(dbl?~0ULL:0xffffffffULL):"(_neg?"+lit(1ULL<<(width-1))+":"+lit((1ULL<<(width-1))-1)+")")+";"
|
|
" if(_exp=="+std::to_string(2*bias+1)+") { _invalid=1; _mag=_mant?0:_limit;"
|
|
" if(_mant)_neg=0; } else if(!_exp&&_mant&&(c->fpcr&(1ULL<<24))) c->fpsr|=128;"
|
|
" else if(_exp||_mant) { int _e=(_exp?(int)_exp-"+std::to_string(bias)+":"+
|
|
std::to_string(1-(int)bias)+")+"+std::to_string(fbits)+"; if(_exp)_mant|="+lit(1ULL<<frac)+";"
|
|
" if(_e>="+std::to_string(width)+") { _invalid=1; _mag=_limit; } else {"
|
|
" int _shift="+std::to_string(frac)+"-_e; uint64_t _lost=_shift<=0?0:_shift<64?_mant&((1ULL<<_shift)-1):_mant;"
|
|
" _mag=_shift<=0?_mant<<(-_shift):_shift<64?_mant>>_shift:0; _inexact=_lost!=0;");
|
|
if(nearest||away)put("if(_lost&&_shift<64&&(_lost>(1ULL<<(_shift-1))||(_lost==(1ULL<<(_shift-1))&&"+
|
|
std::string(away?"1":"(_mag&1)")+"))) ++_mag;");
|
|
put("if(_mag>_limit) { _mag=_limit; _invalid=1; } } }");
|
|
if(uns)put("if(_neg&&_mag) { _mag=0; _invalid=1; }");
|
|
put("_r=_neg?0-_mag:_mag; if(_invalid)c->fpsr|=1; else if(_inexact)c->fpsr|=16;");
|
|
}
|
|
put("_dst[_j]=("+ct+")_r; } memcpy(c->vreg["+std::to_string(rd)+"],_dst,16); }");
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// Baseline half conversions observe AHP, but ignore FZ16 (FPUnpackCV /
|
|
// FPRoundCV). This is distinct from optional half arithmetic instructions.
|
|
{
|
|
const u32 key=i&0xFFFFFC00, vkey=i&0xBFFFFC00;
|
|
const bool scalar=key==0x1E624000||key==0x1E22C000||key==0x1E23C000||
|
|
key==0x1E63C000||key==0x1EE24000||key==0x1EE2C000;
|
|
const bool vw=vkey==0x0E217800, vn=vkey==0x0E216800;
|
|
if(scalar||vw||vn) {
|
|
const unsigned rd=i&31,rn=(i>>5)&31;
|
|
const bool upper=(i&(1U<<30))!=0;
|
|
const unsigned from=vw?16:vn?32:((i>>22)&3)==0?32:((i>>22)&3)==1?64:16;
|
|
const unsigned to=vw?32:vn?16:((i>>15)&3)==0?32:((i>>15)&3)==1?64:16;
|
|
const unsigned sf=from==16?10:from==32?23:52, df=to==16?10:to==32?23:52;
|
|
const unsigned sb=from==16?15:from==32?127:1023, db=to==16?15:to==32?127:1023;
|
|
const auto literal=[](uint64_t v){return std::to_string(v)+"ULL";};
|
|
const uint64_t sm=(1ULL<<sf)-1,dm=(1ULL<<df)-1;
|
|
const std::string src="uint"+std::to_string(from)+"_t",dst="uint"+std::to_string(to)+"_t";
|
|
put("{ "+src+" _src["+std::to_string(128/from)+"]; "+dst+" _dst["+
|
|
std::to_string(128/to)+"]={0}; memcpy(_src,c->vreg["+std::to_string(rn)+"],16);");
|
|
if(vn&&upper)put("memcpy(_dst,c->vreg["+std::to_string(rd)+"],16);");
|
|
put("for(unsigned _j=0;_j<"+std::to_string(scalar?1:4)+";++_j) { uint64_t _v=_src[_j+"+
|
|
std::to_string(vw&&upper?4:0)+"],_f=_v&"+literal(sm)+",_r; unsigned _e=(unsigned)((_v>>"+
|
|
std::to_string(sf)+")&"+std::to_string(2*sb+1)+"); uint64_t _sign=(_v>>"+
|
|
std::to_string(from-1)+")<<"+std::to_string(to-1)+"; unsigned _mode=(unsigned)((c->fpcr>>22)&3);"
|
|
" int _ahp="+std::string(to==16?"!!(c->fpcr&(1ULL<<26))":"0")+"; _r=_sign;");
|
|
put("if(_e=="+std::to_string(2*sb+1)+std::string(from==16?"&&!(c->fpcr&(1ULL<<26))":"")+
|
|
") { if(_f) { if(!(_f&"+literal(1ULL<<(sf-1))+")||_ahp) c->fpsr|=1; if(!_ahp) {"
|
|
" _r|="+literal((uint64_t)(2*db+1)<<df)+"|"+
|
|
std::string(sf>df?"(_f>>"+std::to_string(sf-df)+")":"(_f<<"+std::to_string(df-sf)+")")+
|
|
"|"+literal(1ULL<<(df-1))+"; if(c->fpcr&(1ULL<<25)) _r="+
|
|
literal(((uint64_t)(2*db+1)<<df)|(1ULL<<(df-1)))+"; } } else {"
|
|
" if(_ahp) { _r|="+literal((1ULL<<(to-1))-1)+"; c->fpsr|=1; } else _r|="+
|
|
literal((uint64_t)(2*db+1)<<df)+"; } }");
|
|
if(from!=16)put("else if(!_e&&_f&&(c->fpcr&(1ULL<<24))) c->fpsr|=128;");
|
|
put("else if(_e||_f) { int _unbiased=_e?(int)_e-"+std::to_string(sb)+":"+
|
|
std::to_string(1-(int)sb)+"; uint64_t _mant=_f|(_e?"+literal(1ULL<<sf)+":0);"
|
|
" while(!(_mant&"+literal(1ULL<<sf)+")) { _mant<<=1; --_unbiased; }");
|
|
if(to!=16)put("if(_unbiased<"+std::to_string(1-(int)db)+"&&(c->fpcr&(1ULL<<24))) c->fpsr|=8; else");
|
|
put("{ int _tiny=_unbiased<"+std::to_string(1-(int)db)+"; int _shift="+
|
|
std::to_string((int)sf-(int)df)+"+(_tiny?"+std::to_string(1-(int)db)+"-_unbiased:0);"
|
|
" uint64_t _whole=_shift<=0?_mant<<(-_shift):_shift<64?_mant>>_shift:0;"
|
|
" uint64_t _lost=_shift<=0?0:_shift<64?_mant&((1ULL<<_shift)-1):_mant;"
|
|
" int _up=_lost&&(_mode==0?(_shift<64&&(_lost>(1ULL<<(_shift-1))||"
|
|
" (_lost==(1ULL<<(_shift-1))&&(_whole&1)))):_mode==1?!_sign:_mode==2?!!_sign:0);"
|
|
" _whole+=_up; if(_whole>="+literal(1ULL<<(df+1))+") { _whole>>=1; ++_unbiased; }"
|
|
" if(_unbiased>"+std::to_string(db)+"+_ahp) { if(_ahp) { _r|="+
|
|
literal((1ULL<<(to-1))-1)+"; c->fpsr|=1; } else {"
|
|
" int _inf=_mode==0||(_mode==1&&!_sign)||(_mode==2&&_sign); _r|=_inf?"+
|
|
literal((uint64_t)(2*db+1)<<df)+":"+literal(((uint64_t)(2*db+1)<<df)-1)+"; c->fpsr|=20; } }"
|
|
" else { _r|=_tiny?_whole:((uint64_t)(_unbiased+"+std::to_string(db)+")<<"+
|
|
std::to_string(df)+")|(_whole&"+literal(dm)+"); if(_lost) { c->fpsr|=16; if(_tiny) c->fpsr|=8; } } } }"
|
|
" _dst[_j+"+std::to_string(vn&&upper?4:0)+"]=("+dst+")_r; } memcpy(c->vreg["+
|
|
std::to_string(rd)+"],_dst,16); }");
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// S->D vector widening and D->S narrowing. Half conversions are above;
|
|
// rounding-to-odd FCVTXN remains a separate, unmatched encoding.
|
|
if ((i & 0xBFFFFC00) == 0x0E617800 || (i & 0xBFFFFC00) == 0x0E616800) {
|
|
const unsigned rd = i & 31, rn = (i >> 5) & 31;
|
|
const bool upper = (i & (1U << 30)) != 0, widen = (i & 0x1000) != 0;
|
|
if (widen) {
|
|
put("{ uint32_t _src[4]; uint64_t _dst[2]; memcpy(_src,c->vreg[" +
|
|
std::to_string(rn) + "],16); for(unsigned _j=0;_j<2;++_j) {"
|
|
" uint32_t _v=_src[_j+" + std::to_string(upper ? 2 : 0) +
|
|
"],_f=_v&0x7fffff; unsigned _e=(_v>>23)&255;"
|
|
" uint64_t _r=(uint64_t)(_v&0x80000000U)<<32;"
|
|
" if(_e==255) { _r|=0x7ff0000000000000ULL; if(_f) {"
|
|
" if(!(_f&0x400000)) c->fpsr|=1; _r|=((uint64_t)_f<<29)|0x8000000000000ULL;"
|
|
" if(c->fpcr&(1ULL<<25)) _r=0x7ff8000000000000ULL; } }"
|
|
" else if(_e) _r|=((uint64_t)(_e+896)<<52)|((uint64_t)_f<<29);"
|
|
" else if(_f) { if(c->fpcr&(1ULL<<24)) c->fpsr|=128;"
|
|
" else { int _unbiased=-126; while(!(_f&0x800000)) { _f<<=1; --_unbiased; }"
|
|
" _r|=((uint64_t)(_unbiased+1023)<<52)|((uint64_t)(_f&0x7fffff)<<29); } }"
|
|
" _dst[_j]=_r; } memcpy(c->vreg[" + std::to_string(rd) + "],_dst,16); }");
|
|
} else {
|
|
put("{ uint64_t _src[2]; uint32_t _dst[2]; memcpy(_src,c->vreg[" +
|
|
std::to_string(rn) + "],16); for(unsigned _j=0;_j<2;++_j) {"
|
|
" uint64_t _v=_src[_j],_f=_v&0xfffffffffffffULL; unsigned _e=(unsigned)((_v>>52)&2047);"
|
|
" uint32_t _sign=(uint32_t)(_v>>32)&0x80000000U,_r=_sign;"
|
|
" unsigned _mode=(unsigned)((c->fpcr>>22)&3);"
|
|
" if(_e==2047) { _r|=0x7f800000U; if(_f) {"
|
|
" if(!(_f&0x8000000000000ULL)) c->fpsr|=1; _r|=(uint32_t)(_f>>29)|0x400000U;"
|
|
" if(c->fpcr&(1ULL<<25)) _r=0x7fc00000U; } }"
|
|
" else if(!_e&&_f&&(c->fpcr&(1ULL<<24))) c->fpsr|=128;"
|
|
" else if(_e||_f) { int _unbiased=_e?(int)_e-1023:-1022;"
|
|
" uint64_t _mant=_f|(_e?0x10000000000000ULL:0);"
|
|
" if(_unbiased < -126 && (c->fpcr&(1ULL<<24))) { c->fpsr|=8; }"
|
|
" else { unsigned _shift=_unbiased < -126?(unsigned)(-97-_unbiased):29;"
|
|
" uint64_t _whole=_shift<64?_mant>>_shift:0;"
|
|
" uint64_t _lost=_shift<64?_mant&((1ULL<<_shift)-1):_mant;"
|
|
" int _up=_lost&&(_mode==0?(_shift<64&&(_lost>(1ULL<<(_shift-1))||"
|
|
" (_lost==(1ULL<<(_shift-1))&&(_whole&1)))):_mode==1?!_sign:_mode==2?!!_sign:0);"
|
|
" _whole+=_up; if(_whole>=0x1000000ULL) { _whole>>=1; ++_unbiased; }"
|
|
" if(_unbiased>127) { int _inf=_mode==0||(_mode==1&&!_sign)||(_mode==2&&_sign);"
|
|
" _r|=_inf?0x7f800000U:0x7f7fffffU; c->fpsr|=20; }"
|
|
" else { _r|=_unbiased < -126?(uint32_t)_whole:"
|
|
" ((uint32_t)(_unbiased+127)<<23)|((uint32_t)_whole&0x7fffffU);"
|
|
" if(_lost) { c->fpsr|=16; if(_unbiased < -126) c->fpsr|=8; } } } }"
|
|
" _dst[_j]=_r; } memcpy((char*)c->vreg[" + std::to_string(rd) + "]+" +
|
|
std::to_string(upper ? 8 : 0) + ",_dst,8);");
|
|
if (!upper) put("c->vreg[" + std::to_string(rd) + "][1]=0;");
|
|
put("}");
|
|
}
|
|
return true;
|
|
}
|
|
|
|
// Vector and scalar integral rounding (FRINT{N,P,M,Z,A,X,I}), using integer
|
|
// IEEE-754 fields so guest rounding and subnormal modes do not depend on
|
|
// the host floating-point environment.
|
|
{
|
|
const u32 key = i & 0xBFBFFC00;
|
|
const u32 sop = (i >> 15) & 0x3F;
|
|
const bool scalar = (i & 0xFFA07C00) == 0x1E204000 && ((sop >= 8 && sop <= 12) || sop == 14 || sop == 15);
|
|
const bool rounding = key == 0x0E218800 || key == 0x0EA18800 ||
|
|
key == 0x0E219800 || key == 0x0EA19800 || key == 0x2E218800 ||
|
|
key == 0x2E219800 || key == 0x2EA19800;
|
|
const bool dbl = (i & (1U << 22)) != 0, q = (i & (1U << 30)) != 0;
|
|
if ((rounding && (!dbl || q)) || scalar) {
|
|
const unsigned rd = i & 31, rn = (i >> 5) & 31;
|
|
const unsigned lanes = dbl ? 2 : 4, active = scalar ? 1 : q ? lanes : lanes / 2;
|
|
const std::string ct = dbl ? "uint64_t" : "uint32_t";
|
|
const bool exact_flag = scalar ? sop == 14 : key == 0x2E219800; // FRINTX
|
|
const std::string mode = scalar
|
|
? (sop <= 12 ? std::to_string(sop - 8) : std::string("((c->fpcr>>22)&3)"))
|
|
: key == 0x0E218800 ? "0" : key == 0x0EA18800 ? "1" :
|
|
key == 0x0E219800 ? "2" : key == 0x0EA19800 ? "3" :
|
|
key == 0x2E218800 ? "4" : "((c->fpcr>>22)&3)";
|
|
put("{ " + ct + " _src[" + std::to_string(lanes) + "],_dst[" +
|
|
std::to_string(lanes) + "]={0}; memcpy(_src,c->vreg[" +
|
|
std::to_string(rn) + "],16); const unsigned _mode=" + mode + ";");
|
|
put("for(unsigned _j=0;_j<" + std::to_string(active) + ";++_j) {"
|
|
" uint64_t _v=_src[_j],_a,_r; const uint64_t _sign=" +
|
|
std::string(dbl ? "0x8000000000000000ULL" : "0x80000000ULL") +
|
|
",_exp=" + (dbl ? "0x7ff0000000000000ULL" : "0x7f800000ULL") +
|
|
",_frac=" + (dbl ? "0xfffffffffffffULL" : "0x7fffffULL") +
|
|
",_quiet=" + (dbl ? "0x8000000000000ULL" : "0x400000ULL") + ";");
|
|
put("if((c->fpcr&(1ULL<<24))&&!(_v&_exp)&&(_v&_frac))"
|
|
" { _v&=_sign; c->fpsr|=128; } _a=_v&~_sign; _r=_v;"
|
|
" if((_a&_exp)==_exp) { if(_a&_frac) {"
|
|
" if(!(_a&_quiet)) c->fpsr|=1; _r=_v|_quiet;"
|
|
" if(c->fpcr&(1ULL<<25)) _r=_exp|_quiet; } }"
|
|
" else { int _e=(int)(_a>>" + std::to_string(dbl ? 52 : 23) + ")-" +
|
|
std::to_string(dbl ? 1023 : 127) + "; if(_a&&_e<" +
|
|
std::to_string(dbl ? 52 : 23) + ") { int _up=0; if(_e<0) {"
|
|
" const uint64_t _half=" + (dbl ? "0x3fe0000000000000ULL" : "0x3f000000ULL") +
|
|
"; _up=(_mode==0?_a>_half:_mode==4?_a>=_half:"
|
|
" _mode==1?!(_v&_sign):_mode==2?!!(_v&_sign):0);"
|
|
" _r=(_v&_sign)|(_up?" + (dbl ? "0x3ff0000000000000ULL" : "0x3f800000ULL") + ":0);"
|
|
" } else { unsigned _shift=" + std::to_string(dbl ? 52 : 23) +
|
|
"-_e; uint64_t _unit=1ULL<<_shift,_mask=_unit-1,_lost=_a&_mask;"
|
|
" _up=_lost&&(_mode==0?(_lost>(_unit>>1)||(_lost==(_unit>>1)&&(_a&_unit))):"
|
|
" _mode==4?_lost>=(_unit>>1):_mode==1?!(_v&_sign):_mode==2?!!(_v&_sign):0);"
|
|
" _r=(_v&_sign)|((_a&~_mask)+(_up?_unit:0)); } }");
|
|
if (exact_flag) put("if(_r!=_v) c->fpsr|=16;");
|
|
put("} _dst[_j]=(" + ct + ")_r; } memcpy(c->vreg[" +
|
|
std::to_string(rd) + "],_dst,16); }");
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// Scalar fused multiply-add: negate inputs before the single rounding.
|
|
if ((i & 0xFF000000) == 0x1F000000 && ((i >> 22) & 3) <= 1) {
|
|
const bool dbl = ((i >> 22) & 1) != 0;
|
|
const bool negate_addend = ((i >> 21) & 1) != 0;
|
|
const bool negate_product = negate_addend != (((i >> 15) & 1) != 0);
|
|
const u32 rm = (i >> 16) & 31, ra = (i >> 10) & 31;
|
|
const u32 rn = (i >> 5) & 31, rd = i & 31;
|
|
const std::string sz = dbl ? "8" : "4";
|
|
put("{ uint64_t _a=0,_b=0,_z=0,_v; memcpy(&_a,c->vreg["+std::to_string(rn)+"],"+sz+
|
|
");memcpy(&_b,c->vreg["+std::to_string(rm)+"],"+sz+
|
|
");memcpy(&_z,c->vreg["+std::to_string(ra)+"],"+sz+");");
|
|
if(negate_product)put("_a^="+std::string(dbl?"0x8000000000000000ULL":"0x80000000ULL")+";");
|
|
if(negate_addend)put("_z^="+std::string(dbl?"0x8000000000000000ULL":"0x80000000ULL")+";");
|
|
put(EmitFPMulAddValue(dbl));
|
|
put("c->vreg["+std::to_string(rd)+"][0]=_v;c->vreg["+std::to_string(rd)+"][1]=0;}");
|
|
return true;
|
|
}
|
|
|
|
if((i&0xFFA0FC00)==0x1E200800||(i&0xFFA0FC00)==0x1E201800||
|
|
(i&0xFFA0FC00)==0x1E208800) {
|
|
const bool divide=(i&0x1000)!=0;
|
|
const bool negate=(i&0x8000)!=0;
|
|
const bool dbl=(i&(1U<<22))!=0;const unsigned rd=i&31,rn=(i>>5)&31,rm=(i>>16)&31;
|
|
const std::string sz=dbl?"8":"4";
|
|
put("{uint64_t _a=0,_b=0,_z,_v;memcpy(&_a,c->vreg["+std::to_string(rn)+"],"+sz+
|
|
");memcpy(&_b,c->vreg["+std::to_string(rm)+"],"+sz+");_z=(_a^_b)&"+
|
|
std::string(dbl?"0x8000000000000000ULL":"0x80000000ULL")+";");
|
|
if(divide)put("(void)_z;");
|
|
put(divide?EmitFPDivideValue(dbl):EmitFPMulAddValue(dbl,true));
|
|
// FNMUL negates the rounded product, including NaN/zero sign. Moving
|
|
// this sign change to an input would reverse directed rounding.
|
|
if(negate)put("_v^="+std::string(dbl?"0x8000000000000000ULL":"0x80000000ULL")+";");
|
|
put("c->vreg["+std::to_string(rd)+"][0]=_v;c->vreg["+std::to_string(rd)+"][1]=0;}");
|
|
return true;
|
|
}
|
|
|
|
// Scalar floating point. Only single (ftype 00) and double (ftype 01) are
|
|
// handled; half-precision and the SIMD vector forms still fall through.
|
|
// Values move through memcpy rather than type punning so this stays
|
|
// strictly conforming C.
|
|
if ((i & 0x5F000000) == 0x1E000000 && ((i >> 21) & 1)) {
|
|
const u32 ftype = (i >> 22) & 3;
|
|
const u32 rm = (i >> 16) & 31, rn = (i >> 5) & 31, rd = i & 31;
|
|
if (ftype == 0 || ftype == 1) {
|
|
const bool dbl = (ftype == 1);
|
|
const char* ct = dbl ? "double" : "float";
|
|
const int sz = dbl ? 8 : 4;
|
|
const std::string ld_n = std::string("{ ") + ct + " _a,_b,_r; memcpy(&_a,&c->vreg[" +
|
|
std::to_string(rn) + "][0]," + std::to_string(sz) + "); ";
|
|
const std::string ld_m = std::string("memcpy(&_b,&c->vreg[") + std::to_string(rm) +
|
|
"][0]," + std::to_string(sz) + "); ";
|
|
const std::string st_d = std::string("c->vreg[") + std::to_string(rd) +
|
|
"][0]=0; c->vreg[" + std::to_string(rd) +
|
|
"][1]=0; memcpy(&c->vreg[" + std::to_string(rd) + "][0],&_r," +
|
|
std::to_string(sz) + "); }";
|
|
|
|
// FMOV (immediate). This must be decoded before FCMP below: it
|
|
// also has bits 11..10 clear, so an FMOV whose imm8 and Rd happen
|
|
// to line up would otherwise be read as a compare and silently
|
|
// clobber the flags instead of loading a constant.
|
|
if (((i >> 10) & 7) == 4 && ((i >> 5) & 0x1F) == 0) {
|
|
const u32 imm8 = (i >> 13) & 0xFF;
|
|
// VFPExpandImm: sign, then exponent as NOT(b6) followed by b6
|
|
// repeated, then imm8<5:4>, then imm8<3:0> as the top of the
|
|
// mantissa. Built at double width and narrowed if needed.
|
|
const u32 sgn = (imm8 >> 7) & 1;
|
|
const u32 b6 = (imm8 >> 6) & 1;
|
|
const u64 e11 = ((u64)(b6 ^ 1) << 10) | (b6 ? (0xFFULL << 2) : 0ULL) |
|
|
((imm8 >> 4) & 3);
|
|
const u64 dbits = ((u64)sgn << 63) | (e11 << 52) | ((u64)(imm8 & 0xF) << 48);
|
|
char hb[64];
|
|
if (dbl) {
|
|
snprintf(hb, sizeof hb, "0x%llxULL", (unsigned long long)dbits);
|
|
} else {
|
|
double dv; memcpy(&dv, &dbits, 8);
|
|
const float fv = (float)dv;
|
|
u32 fbits; memcpy(&fbits, &fv, 4);
|
|
snprintf(hb, sizeof hb, "0x%xULL", fbits);
|
|
}
|
|
put("c->vreg[" + std::to_string(rd) + "][0]=" + hb + "; c->vreg[" +
|
|
std::to_string(rd) + "][1]=0;");
|
|
return true;
|
|
}
|
|
|
|
// FMAX/FMIN/FMAXNM/FMINNM (opcode 4-7 in bits 15..12): the SIMD
|
|
// forms' exact core. Host comparisons and fmax/fmin got signed
|
|
// zeros, NaN selection and the FPSR wrong.
|
|
if (((i >> 10) & 3) == 2 && ((i >> 12) & 15) >= 4 && ((i >> 12) & 15) <= 7) {
|
|
const u32 opcode = (i >> 12) & 15;
|
|
put("{ uint64_t _n=0,_m=0,_r; memcpy(&_n,c->vreg[" + std::to_string(rn) + "]," +
|
|
std::to_string(sz) + "); memcpy(&_m,c->vreg[" + std::to_string(rm) + "]," +
|
|
std::to_string(sz) + ");");
|
|
put(EmitFPMinMax(dbl, (opcode & 1) != 0, opcode >= 6, "_n", "_m", "_r=_v;"));
|
|
put("c->vreg[" + std::to_string(rd) + "][0]=_r; c->vreg[" + std::to_string(rd) +
|
|
"][1]=0; }");
|
|
return true;
|
|
}
|
|
|
|
// FCVT between precisions. It shares the one-source encoding but
|
|
// its source and destination widths differ, so it cannot use the
|
|
// shared load/store fragments below, which assume both are ftype.
|
|
// Opcode 0001xx, where the low two bits name the destination:
|
|
// 00 single, 01 double, 11 half. Half stays on the fallback.
|
|
if (((i >> 10) & 0x1F) == 0x10 && (((i >> 15) & 0x3C) == 0x04)) {
|
|
const u32 dst = (i >> 15) & 3;
|
|
if (ftype == 0 && dst == 1) { // single -> double
|
|
put("{ float _s; double _d; memcpy(&_s,&c->vreg[" + std::to_string(rn) +
|
|
"][0],4); _d=(double)_s; c->vreg[" + std::to_string(rd) +
|
|
"][1]=0; memcpy(&c->vreg[" + std::to_string(rd) + "][0],&_d,8); }");
|
|
return true;
|
|
}
|
|
if (ftype == 1 && dst == 0) { // double -> single
|
|
put("{ double _d; float _s; memcpy(&_d,&c->vreg[" + std::to_string(rn) +
|
|
"][0],8); _s=(float)_d; c->vreg[" + std::to_string(rd) +
|
|
"][0]=0; c->vreg[" + std::to_string(rd) +
|
|
"][1]=0; memcpy(&c->vreg[" + std::to_string(rd) + "][0],&_s,4); }");
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// One-source: FMOV/FABS/FNEG (opcode in bits 20..15 low bits).
|
|
// FSQRT and FRINT* are decoded exactly above.
|
|
if (((i >> 10) & 0x1F) == 0x10) {
|
|
const u32 opcode = (i >> 15) & 0x3F;
|
|
std::string expr;
|
|
switch (opcode) {
|
|
case 0: expr = "_a"; break; // FMOV
|
|
case 1: expr = dbl ? "fabs(_a)" : "fabsf(_a)"; break; // FABS
|
|
case 2: expr = "-_a"; break; // FNEG
|
|
default: break;
|
|
}
|
|
if (!expr.empty()) {
|
|
put(ld_n + "(void)_b; _r = " + expr + "; " + st_d);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// FCCMP / FCCMPE: compare when the condition holds, otherwise take
|
|
// the flags straight from nzcv. Bits 11..10 are 01 here, where an
|
|
// ordinary FCMP has 1000 in bits 13..10, so the two do not overlap.
|
|
// Bit 4 selects the signalling form.
|
|
if (((i >> 10) & 3) == 1) {
|
|
const u32 cond = (i >> 12) & 15, nzcv = i & 15;
|
|
std::string s = "{ if " + Cond(cond) + " ";
|
|
s += EmitFPCompareFlags(dbl, ((i >> 4) & 1) != 0,
|
|
"c->vreg[" + std::to_string(rn) + "]",
|
|
"c->vreg[" + std::to_string(rm) + "]");
|
|
s += " else { c->n=" + std::to_string((nzcv >> 3) & 1) + "; ";
|
|
s += "c->z=" + std::to_string((nzcv >> 2) & 1) + "; ";
|
|
s += "c->c=" + std::to_string((nzcv >> 1) & 1) + "; ";
|
|
s += "c->v=" + std::to_string(nzcv & 1) + "; } }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
|
|
// FCMP / FCMPE, including the compare-against-zero forms. The
|
|
// low five bits are opcode2: bit 3 selects the #0.0 variant (Rm is
|
|
// then not a register at all) and bit 4 selects the signalling
|
|
// form, which raises IOC for a quiet NaN as well. Requiring all
|
|
// five to be clear, as before, matched only a quarter of the
|
|
// compares in real code.
|
|
if (((i >> 10) & 0xF) == 8 && ((i >> 14) & 3) == 0 && (i & 7) == 0) {
|
|
const bool cmp_zero = ((i >> 3) & 1) != 0;
|
|
put(EmitFPCompareFlags(dbl, ((i >> 4) & 1) != 0,
|
|
"c->vreg[" + std::to_string(rn) + "]",
|
|
cmp_zero ? std::string() : "c->vreg[" + std::to_string(rm) + "]"));
|
|
return true;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Data-processing 2-source: UDIV, SDIV, and variable-shift forms LSLV/LSRV/ASRV/RORV.
|
|
// Encoding: sf 0 0 1 1 0 1 0 1 Rm opcode Rn Rd
|
|
if ((i & 0x5FE00000) == 0x1AC00000) {
|
|
const u32 sf = i >> 31;
|
|
const u32 rm = (i >> 16) & 31, opcode = (i >> 10) & 63;
|
|
const u32 rn = (i >> 5) & 31, rd = i & 31;
|
|
if (rd != 31) {
|
|
// Register 31 reads as XZR in this group, not SP.
|
|
const std::string xn = Xz(rn);
|
|
const std::string xm = Xz(rm);
|
|
std::string s;
|
|
switch (opcode) {
|
|
case 2: // UDIV - divide by 0 yields 0 per ARM spec
|
|
if (sf) s = "{ uint64_t _r=" + xm + "?" + xn + "/" + xm + ":0; c->x[" + std::to_string(rd) + "]=_r; }";
|
|
else s = "{ uint32_t _a=(uint32_t)" + xn + ",_b=(uint32_t)" + xm + "; c->x[" + std::to_string(rd) + "]=(uint64_t)(_b?_a/_b:0); }";
|
|
break;
|
|
case 3: // SDIV
|
|
if (sf) s = "{ int64_t _a=(int64_t)" + xn + ",_b=(int64_t)" + xm + "; c->x[" + std::to_string(rd) + "]=(uint64_t)(_b?_a/_b:0); }";
|
|
else s = "{ int32_t _a=(int32_t)(uint32_t)" + xn + ",_b=(int32_t)(uint32_t)" + xm + "; c->x[" + std::to_string(rd) + "]=(uint64_t)(uint32_t)(_b?_a/_b:0); }";
|
|
break;
|
|
case 8: // LSLV
|
|
if (sf) s = "{ c->x[" + std::to_string(rd) + "]=" + xn + "<<(" + xm + "&63); }";
|
|
else s = "{ c->x[" + std::to_string(rd) + "]=(uint64_t)(uint32_t)((uint32_t)" + xn + "<<(" + xm + "&31)); }";
|
|
break;
|
|
case 9: // LSRV
|
|
if (sf) s = "{ c->x[" + std::to_string(rd) + "]=" + xn + ">>(" + xm + "&63); }";
|
|
else s = "{ c->x[" + std::to_string(rd) + "]=(uint64_t)((uint32_t)" + xn + ">>(" + xm + "&31)); }";
|
|
break;
|
|
case 10: // ASRV
|
|
if (sf) s = "{ c->x[" + std::to_string(rd) + "]=(uint64_t)((int64_t)" + xn + ">>(" + xm + "&63)); }";
|
|
else s = "{ c->x[" + std::to_string(rd) + "]=(uint64_t)(uint32_t)((int32_t)(uint32_t)" + xn + ">>(" + xm + "&31)); }";
|
|
break;
|
|
case 11: // RORV
|
|
if (sf) s = "{ uint64_t _a=" + xn + ",_s=" + xm + "&63; c->x[" + std::to_string(rd) + "]=_s?(_a>>_s)|(_a<<(64-_s)):_a; }";
|
|
else s = "{ uint32_t _a=(uint32_t)" + xn + ",_s=(uint32_t)" + xm + "&31; c->x[" + std::to_string(rd) + "]=(uint64_t)(uint32_t)(_s?(_a>>_s)|(_a<<(32-_s)):_a); }";
|
|
break;
|
|
}
|
|
if (!s.empty()) { put(s); return true; }
|
|
}
|
|
}
|
|
|
|
// Advanced SIMD two-register misc, floating-point compare against zero:
|
|
// FCMGT, FCMGE, FCMEQ, FCMLE and FCMLT. A true lane is all-ones, which is
|
|
// what the following select/AND normally consumes. Only the bit-23-clear
|
|
// group is decoded; the rest stays on the fallback.
|
|
{
|
|
const bool vec_misc = (i & 0x9F3E0C00) == 0x0E200800;
|
|
// Bit 29 is U and must stay out of the mask, or only the U=0 half of
|
|
// each pair (FCMEQ but not FCMLE, FCMGT but not FCMGE) would match.
|
|
const bool scl_misc = (i & 0xDF3E0C00) == 0x5E200800;
|
|
if (vec_misc || scl_misc) {
|
|
const u32 Q = (i >> 30) & 1, U = (i >> 29) & 1;
|
|
const u32 opcode = (i >> 12) & 0x1F;
|
|
const bool dbl = ((i >> 22) & 1) != 0;
|
|
const u32 rn = (i >> 5) & 31, rd = i & 31;
|
|
const char* cmp = nullptr;
|
|
// NOT and RBIT: both opcode 5 with U set, told apart by size.
|
|
if (opcode == 0x05 && U && vec_misc) {
|
|
const u32 size = (i >> 22) & 3;
|
|
const int bytes = Q ? 16 : 8;
|
|
if (size <= 1) {
|
|
std::string s = "{ uint8_t _a[" + std::to_string(bytes) + "]; ";
|
|
s += "memcpy(_a,c->vreg[" + std::to_string(rn) + "]," + std::to_string(bytes) +
|
|
"); ";
|
|
if (size == 0) {
|
|
s += "for(int _i=0;_i<" + std::to_string(bytes) +
|
|
";_i++) _a[_i]=(uint8_t)~_a[_i]; ";
|
|
} else {
|
|
s += "for(int _i=0;_i<" + std::to_string(bytes) + ";_i++){ uint8_t _v=_a[_i]; ";
|
|
s += "_v=(uint8_t)((_v>>4)|(_v<<4)); ";
|
|
s += "_v=(uint8_t)(((_v&0xCC)>>2)|((_v&0x33)<<2)); ";
|
|
s += "_v=(uint8_t)(((_v&0xAA)>>1)|((_v&0x55)<<1)); _a[_i]=_v; } ";
|
|
}
|
|
s += "c->vreg[" + std::to_string(rd) + "][0]=0; c->vreg[" + std::to_string(rd) +
|
|
"][1]=0; ";
|
|
s += "memcpy(c->vreg[" + std::to_string(rd) + "],_a," + std::to_string(bytes) +
|
|
"); }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
}
|
|
// The integer compares against zero share this class; their
|
|
// opcodes sit just below the floating-point ones.
|
|
if (opcode >= 0x08 && opcode <= 0x0A && !(opcode == 0x0A && U)) {
|
|
const u32 size = (i >> 22) & 3;
|
|
const int esz = 1 << size;
|
|
const int bytes = scl_misc ? esz : (Q ? 16 : 8);
|
|
const int lanes = bytes / esz;
|
|
const std::string uty = "uint" + std::to_string(esz * 8) + "_t";
|
|
const std::string ity = "int" + std::to_string(esz * 8) + "_t";
|
|
const char* op = (opcode == 0x08) ? (U ? ">=" : ">")
|
|
: (opcode == 0x09) ? (U ? "<=" : "==")
|
|
: "<";
|
|
if (!(size == 3 && !Q && !scl_misc)) {
|
|
std::string s = "{ " + ity + " _a[" + std::to_string(lanes) + "]; " + uty +
|
|
" _r[" + std::to_string(lanes) + "]; ";
|
|
s += "memcpy(_a,c->vreg[" + std::to_string(rn) + "]," + std::to_string(bytes) +
|
|
"); ";
|
|
s += "for(int _i=0;_i<" + std::to_string(lanes) + ";_i++) _r[_i]=(_a[_i]" +
|
|
std::string(op) + "0)?(" + uty + ")~(" + uty + ")0:(" + uty + ")0; ";
|
|
s += "c->vreg[" + std::to_string(rd) + "][0]=0; c->vreg[" + std::to_string(rd) +
|
|
"][1]=0; ";
|
|
s += "memcpy(c->vreg[" + std::to_string(rd) + "],_r," + std::to_string(bytes) +
|
|
"); }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
}
|
|
if (opcode == 0x0C) cmp = U ? ">=" : ">"; // FCMGE / FCMGT
|
|
else if (opcode == 0x0D) cmp = U ? "<=" : "=="; // FCMLE / FCMEQ
|
|
else if (opcode == 0x0E && !U) cmp = "<"; // FCMLT
|
|
// SCVTF / UCVTF: convert the integer in each lane to a float of the
|
|
// same width. This is the vector counterpart of the general-register
|
|
// conversion handled further up - the operand is a lane here, not a
|
|
// general register.
|
|
// Bit 23 selects reciprocal/reciprocal-square-root estimates,
|
|
// not integer conversion. U selects the square-root form.
|
|
if (opcode == 0x1D && (i & (1U << 23)) != 0 && (!dbl || Q)) {
|
|
const int fsz = dbl ? 8 : 4;
|
|
const int bytes = scl_misc ? fsz : (Q ? 16 : 8);
|
|
const int lanes = bytes / fsz;
|
|
const std::string ct = dbl ? "uint64_t" : "uint32_t";
|
|
std::string s = "{ " + ct + " _a[" + std::to_string(lanes) + "],_r[" +
|
|
std::to_string(lanes) + "]={0}; memcpy(_a,c->vreg[" +
|
|
std::to_string(rn) + "]," + std::to_string(bytes) + "); ";
|
|
s += "for(int _i=0;_i<" + std::to_string(lanes) +
|
|
";++_i){uint64_t _v=0,_x=_a[_i];";
|
|
s += U ? EmitFPRSqrtEstimateValue(dbl) : EmitFPRecipEstimateValue(dbl);
|
|
s += "_r[_i]=(" + ct + ")_v;} c->vreg[" + std::to_string(rd) +
|
|
"][0]=0;c->vreg[" + std::to_string(rd) + "][1]=0;memcpy(c->vreg[" +
|
|
std::to_string(rd) + "],_r," + std::to_string(bytes) + ");}";
|
|
put(s);
|
|
return true;
|
|
}
|
|
if (opcode == 0x1D && (i & (1U << 23)) == 0) {
|
|
// SCVTF/UCVTF per lane, rounded in the guest FPCR mode with IXC.
|
|
const int fsz = dbl ? 8 : 4;
|
|
const int bytes = scl_misc ? fsz : (Q ? 16 : 8);
|
|
const int lanes = bytes / fsz;
|
|
const std::string uty = dbl ? "uint64_t" : "uint32_t", count = dbl ? "2" : "4";
|
|
const std::string w = std::to_string(fsz * 8);
|
|
put("{ " + uty + " _a[" + count + "],_r[" + count + "]={0}; memcpy(_a,c->vreg[" +
|
|
std::to_string(rn) + "],16); for(int _i=0;_i<" + std::to_string(lanes) +
|
|
";_i++) _r[_i]=(" + uty + ")recomp_fixed_to_fp(_a[_i]," + w + "," + w + ",0," +
|
|
(U ? "0" : "1") + ",c->fpcr,&c->fpsr); memcpy(c->vreg[" + std::to_string(rd) +
|
|
"],_r,16); }");
|
|
return true;
|
|
}
|
|
// FABS/FNEG share opcode 0x0F; U selects negation. Opcode 0x0E
|
|
// is FCMLT and must reach the comparison below. Move sign bits
|
|
// directly so NaN payloads and FPSR remain untouched.
|
|
if ((i & 0x9FBFFC00) == 0x0EA0F800 && (!dbl || Q)) {
|
|
const int fsz = dbl ? 8 : 4;
|
|
const int bytes = Q ? 16 : 8;
|
|
const int lanes = bytes / fsz;
|
|
const std::string ct = dbl ? "uint64_t" : "uint32_t";
|
|
const std::string sign = dbl ? "0x8000000000000000ULL" : "0x80000000U";
|
|
const std::string expr = U ? "_a[_i]^" + sign : "_a[_i]&~" + sign;
|
|
std::string s = "{ " + ct + " _a[" + std::to_string(lanes) +
|
|
"],_r[" + std::to_string(lanes) + "]; ";
|
|
s += "memcpy(_a,c->vreg[" + std::to_string(rn) + "]," + std::to_string(bytes) +
|
|
"); ";
|
|
s += "for(int _i=0;_i<" + std::to_string(lanes) + ";_i++) _r[_i]=" + expr + "; ";
|
|
s += "c->vreg[" + std::to_string(rd) + "][0]=0; c->vreg[" + std::to_string(rd) +
|
|
"][1]=0; ";
|
|
s += "memcpy(c->vreg[" + std::to_string(rd) + "],_r," + std::to_string(bytes) +
|
|
"); }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
|
|
// XTN / XTN2: take the low half of each element. XTN2 writes the
|
|
// top half of the destination and leaves the bottom alone, which is
|
|
// the only reason this is not a plain narrowing.
|
|
if (opcode == 0x12 && !U && !scl_misc) {
|
|
const u32 size = (i >> 22) & 3;
|
|
if (size != 3) {
|
|
const int dsz = 1 << size; // destination element
|
|
const int lanes = 8 / dsz; // always half a register out
|
|
const std::string sty = "uint" + std::to_string(dsz * 16) + "_t";
|
|
const std::string dty = "uint" + std::to_string(dsz * 8) + "_t";
|
|
std::string s = "{ " + sty + " _a[" + std::to_string(lanes) + "]; " + dty +
|
|
" _r[" + std::to_string(lanes) + "]; ";
|
|
s += "memcpy(_a,c->vreg[" + std::to_string(rn) + "],16); ";
|
|
s += "for(int _i=0;_i<" + std::to_string(lanes) + ";_i++) _r[_i]=(" + dty +
|
|
")_a[_i]; ";
|
|
if (!Q) {
|
|
s += "c->vreg[" + std::to_string(rd) + "][0]=0; c->vreg[" +
|
|
std::to_string(rd) + "][1]=0; ";
|
|
}
|
|
s += "memcpy((uint8_t*)c->vreg[" + std::to_string(rd) + "]+" +
|
|
std::to_string(Q ? 8 : 0) + ",_r,8); }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// FCVTZS / FCVTZU (bit 23 set) and FCVTMS / FCVTMU (bit 23 clear),
|
|
// vector and scalar: round toward zero or minus infinity, then
|
|
// saturate, the same contract as the general-register forms.
|
|
if (opcode == 0x1B) {
|
|
const int fsz = dbl ? 8 : 4;
|
|
const int bytes = scl_misc ? fsz : (Q ? 16 : 8);
|
|
const int lanes = bytes / fsz;
|
|
const std::string uty = dbl ? "uint64_t" : "uint32_t", count = dbl ? "2" : "4";
|
|
const std::string w = std::to_string(fsz * 8);
|
|
put("{ " + uty + " _a[" + count + "],_r[" + count + "]={0}; memcpy(_a,c->vreg[" +
|
|
std::to_string(rn) + "],16); for(int _i=0;_i<" + std::to_string(lanes) +
|
|
";_i++) _r[_i]=(" + uty + ")recomp_fp_to_int(_a[_i]," + w + "," + w + "," +
|
|
(U ? "0" : "1") + "," + ((i & (1U << 23)) ? "3" : "2") +
|
|
",c->fpcr,&c->fpsr); memcpy(c->vreg[" + std::to_string(rd) + "],_r,16); }");
|
|
return true;
|
|
}
|
|
|
|
// REV16 / REV32 / REV64: reverse bytes within each container.
|
|
// opcode 0 is REV64, 1 is REV16, and REV32 is opcode 0 with U set;
|
|
// size gives the element width being reversed inside.
|
|
if ((opcode == 0 || opcode == 1) && !scl_misc) {
|
|
const u32 size = (i >> 22) & 3;
|
|
const int esz = 1 << size; // byte width of an element
|
|
const int container = (opcode == 1) ? 2 : (U ? 4 : 8);
|
|
if (esz < container) {
|
|
const int bytes = Q ? 16 : 8;
|
|
std::string s = "{ uint8_t _a[" + std::to_string(bytes) + "],_r[" +
|
|
std::to_string(bytes) + "]; ";
|
|
s += "memcpy(_a,c->vreg[" + std::to_string(rn) + "]," +
|
|
std::to_string(bytes) + "); ";
|
|
s += "for(int _i=0;_i<" + std::to_string(bytes) + ";_i++){ ";
|
|
s += "int _base=_i-(_i%" + std::to_string(container) + "); ";
|
|
s += "int _off=_i-_base; ";
|
|
s += "_r[_i]=_a[_base+(" + std::to_string(container) + "-" +
|
|
std::to_string(esz) + "-(_off-(_off%" + std::to_string(esz) +
|
|
")))+(_off%" + std::to_string(esz) + ")]; } ";
|
|
s += "c->vreg[" + std::to_string(rd) + "][0]=0; c->vreg[" +
|
|
std::to_string(rd) + "][1]=0; ";
|
|
s += "memcpy(c->vreg[" + std::to_string(rd) + "],_r," +
|
|
std::to_string(bytes) + "); }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// CNT: set bits per byte. Defined for byte elements only.
|
|
if (opcode == 0x05 && ((i >> 22) & 3) == 0 && !U) {
|
|
const int bytes = Q ? 16 : 8;
|
|
std::string s = "{ uint8_t _a[" + std::to_string(bytes) + "],_r[" +
|
|
std::to_string(bytes) + "]; ";
|
|
s += "memcpy(_a,c->vreg[" + std::to_string(rn) + "]," +
|
|
std::to_string(bytes) + "); ";
|
|
s += "for(int _i=0;_i<" + std::to_string(bytes) + ";_i++){ ";
|
|
s += "uint8_t _v=_a[_i]; _v=(uint8_t)(_v-((_v>>1)&0x55)); ";
|
|
s += "_v=(uint8_t)((_v&0x33)+((_v>>2)&0x33)); ";
|
|
s += "_r[_i]=(uint8_t)((_v+(_v>>4))&0x0F); } ";
|
|
s += "c->vreg[" + std::to_string(rd) + "][0]=0; c->vreg[" +
|
|
std::to_string(rd) + "][1]=0; ";
|
|
s += "memcpy(c->vreg[" + std::to_string(rd) + "],_r," +
|
|
std::to_string(bytes) + "); }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
|
|
// ABS / NEG, lanewise over integer elements. U picks NEG.
|
|
if (opcode == 0x0B) {
|
|
const u32 size = (i >> 22) & 3;
|
|
const int esz = 1 << size;
|
|
const int bytes = scl_misc ? 8 : (Q ? 16 : 8);
|
|
// The 64-bit element only exists as 2D or as the scalar form.
|
|
const bool shaped = scl_misc ? (size == 3) : !(size == 3 && !Q);
|
|
if (shaped) {
|
|
const int lanes = bytes / esz;
|
|
const std::string uty = "uint" + std::to_string(esz * 8) + "_t";
|
|
const std::string ity = "int" + std::to_string(esz * 8) + "_t";
|
|
std::string s = "{ " + uty + " _a[" + std::to_string(lanes) + "],_r[" +
|
|
std::to_string(lanes) + "]; ";
|
|
s += "memcpy(_a,c->vreg[" + std::to_string(rn) + "]," +
|
|
std::to_string(bytes) + "); ";
|
|
// Unsigned throughout: negating the minimum signed value wraps
|
|
// on the architecture and is undefined on a signed C type.
|
|
if (U) {
|
|
s += "for(int _i=0;_i<" + std::to_string(lanes) + ";_i++) _r[_i]=(" +
|
|
uty + ")(0-_a[_i]); ";
|
|
} else {
|
|
s += "for(int _i=0;_i<" + std::to_string(lanes) + ";_i++) _r[_i]=((" +
|
|
ity + ")_a[_i]<0) ? (" + uty + ")(0-_a[_i]) : _a[_i]; ";
|
|
}
|
|
s += "c->vreg[" + std::to_string(rd) + "][0]=0; c->vreg[" +
|
|
std::to_string(rd) + "][1]=0; ";
|
|
s += "memcpy(c->vreg[" + std::to_string(rd) + "],_r," +
|
|
std::to_string(bytes) + "); }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
}
|
|
if (cmp) {
|
|
// Against +0.0: LE and LT are GE and GT with the operands
|
|
// swapped, which is how the architecture defines them.
|
|
const int fsz = dbl ? 8 : 4;
|
|
// A scalar form touches one lane; a vector form covers the
|
|
// whole selected width.
|
|
const int bytes = scl_misc ? fsz : (Q ? 16 : 8);
|
|
const int lanes = bytes / fsz;
|
|
const std::string uty = dbl ? "uint64_t" : "uint32_t", count = dbl ? "2" : "4";
|
|
const std::string op = cmp;
|
|
const bool swap = op == "<=" || op == "<";
|
|
const char* kind = op == "==" ? "eq" : (op == ">=" || op == "<=") ? "ge" : "gt";
|
|
put("{ " + uty + " _n[" + count + "],_r[" + count + "]={0}; memcpy(_n,c->vreg[" +
|
|
std::to_string(rn) + "],16); for(int _i=0;_i<" + std::to_string(lanes) + ";_i++) " +
|
|
EmitFPCompareMask(dbl, kind, swap ? "0" : "_n[_i]", swap ? "_n[_i]" : "0",
|
|
"_r[_i]=(" + uty + ")_v;") +
|
|
" memcpy(c->vreg[" + std::to_string(rd) + "],_r,16); }");
|
|
return true;
|
|
}
|
|
}
|
|
}
|
|
|
|
// FMUL by indexed element. Rm is only four bits here, extended by M, and
|
|
// the lane index is split across H and L - reading Rm as the usual five
|
|
// bits would silently address the wrong register.
|
|
{
|
|
// Bit 29 (U) selects FMULX, a different operation - keep it out of FMUL.
|
|
// Opcode 1001 is FMUL, 0001 FMLA, 0101 FMLS; the three share everything
|
|
// except whether the product replaces the destination or accumulates
|
|
// into it.
|
|
const u32 idxop = (i >> 12) & 0xF;
|
|
const bool idx_shape = (idxop == 0x9 || idxop == 0x1 || idxop == 0x5);
|
|
const bool vec_idx = idx_shape && (i & 0xBF000400) == 0x0F000000;
|
|
const bool scl_idx = idx_shape && (i & 0xFF000400) == 0x5F000000;
|
|
if ((vec_idx || scl_idx) && ((i >> 23) & 1) == 1) {
|
|
const u32 Q = (i >> 30) & 1;
|
|
const bool dbl = ((i >> 22) & 1) != 0;
|
|
const u32 rn = (i >> 5) & 31, rd = i & 31;
|
|
const u32 rm = ((i >> 16) & 15) | (((i >> 20) & 1) << 4);
|
|
const u32 H = (i >> 11) & 1, L = (i >> 21) & 1;
|
|
const u32 index = dbl ? H : ((H << 1) | L);
|
|
const char* ct = dbl ? "uint64_t" : "uint32_t";
|
|
const int fsz = dbl ? 8 : 4;
|
|
const int bytes = scl_idx ? fsz : (Q ? 16 : 8);
|
|
const int lanes = bytes / fsz;
|
|
if (!(dbl && L)) { // L must be zero for the 64-bit form
|
|
std::string s = "{ " + std::string(ct) + " _a[" + std::to_string(lanes) +
|
|
"],_r[" + std::to_string(lanes) + "],_m; ";
|
|
s += "memcpy(_a,c->vreg[" + std::to_string(rn) + "]," + std::to_string(bytes) + "); ";
|
|
s += "memcpy(&_m,(const uint8_t*)c->vreg[" + std::to_string(rm) + "]+" +
|
|
std::to_string(index * fsz) + "," + std::to_string(fsz) + "); ";
|
|
s += "memcpy(_r,c->vreg["+std::to_string(rd)+"],"+std::to_string(bytes)+");";
|
|
// Local scalar names belong to an inner block; source arrays
|
|
// are captured before entering it so every alias remains safe.
|
|
s += "for(int _i=0;_i<"+std::to_string(lanes)+";++_i){uint64_t _n=_a[_i],_z0=_r[_i];"
|
|
"{uint64_t _a=_n,_b=_m,_z=_z0,_v;";
|
|
const std::string sign=dbl?"0x8000000000000000ULL":"0x80000000ULL";
|
|
if(idxop==9)s+="_z=(_a^_b)&"+sign+";";
|
|
if(idxop==5)s+="_a^="+sign+";";
|
|
s+=EmitFPMulAddValue(dbl,idxop==9);
|
|
s+="_r[_i]=("+std::string(ct)+")_v;}}";
|
|
s += "c->vreg[" + std::to_string(rd) + "][0]=0; c->vreg[" + std::to_string(rd) +
|
|
"][1]=0; ";
|
|
s += "memcpy(c->vreg[" + std::to_string(rd) + "],_r," + std::to_string(bytes) + "); }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
}
|
|
}
|
|
|
|
// MUL / MLA / MLS by indexed element. Same shape as the floating-point
|
|
// group above, but size selects an integer width, and for 16-bit elements
|
|
// M is part of the lane index rather than the top bit of Rm.
|
|
{
|
|
const u32 idxop = (i >> 12) & 0xF;
|
|
const u32 U = (i >> 29) & 1;
|
|
const bool widening = idxop == 0x2 || idxop == 0x6 || idxop == 0xA;
|
|
const bool shaped = widening || (idxop == 0x8 && !U) || ((idxop == 0x0 || idxop == 0x4) && U);
|
|
if (shaped && (i & 0x9F00F400) == 0x0F000000 + (idxop << 12)) {
|
|
const u32 Q = (i >> 30) & 1, size = (i >> 22) & 3;
|
|
const u32 rn = (i >> 5) & 31, rd = i & 31;
|
|
const u32 H = (i >> 11) & 1, L = (i >> 21) & 1, M = (i >> 20) & 1;
|
|
u32 rm = 0, index = 0;
|
|
bool ok = true;
|
|
if (size == 1) {
|
|
rm = (i >> 16) & 15;
|
|
index = (H << 2) | (L << 1) | M;
|
|
} else if (size == 2) {
|
|
rm = ((i >> 16) & 15) | (M << 4);
|
|
index = (H << 1) | L;
|
|
} else {
|
|
ok = false;
|
|
}
|
|
if (ok && widening) {
|
|
const int esz = 1 << size, lanes = 8 / esz;
|
|
const std::string sty = std::string(U ? "uint" : "int") + std::to_string(esz * 8) + "_t";
|
|
const std::string dty = "uint" + std::to_string(esz * 16) + "_t";
|
|
const std::string wide = U ? "uint64_t" : "int64_t";
|
|
std::string s = "{ " + sty + " _a[" + std::to_string(lanes) + "],_m; " +
|
|
dty + " _r[" + std::to_string(lanes) + "]; ";
|
|
s += "memcpy(_a,&c->vreg[" + std::to_string(rn) + "][" + std::to_string(Q) + "],8); ";
|
|
s += "memcpy(&_m,(const uint8_t*)c->vreg[" + std::to_string(rm) + "]+" +
|
|
std::to_string(index * esz) + "," + std::to_string(esz) + "); ";
|
|
if (idxop != 0xA) s += "memcpy(_r,c->vreg[" + std::to_string(rd) + "],16); ";
|
|
s += "for(int _i=0;_i<" + std::to_string(lanes) + ";++_i) _r[_i]=(" + dty + ")(";
|
|
if (idxop != 0xA) s += "(uint64_t)_r[_i]" + std::string(idxop == 2 ? "+" : "-");
|
|
s += "(uint64_t)((" + wide + ")_a[_i]*(" + wide + ")_m)); ";
|
|
s += "memcpy(c->vreg[" + std::to_string(rd) + "],_r,16); }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
if (ok) {
|
|
const int esz = 1 << size;
|
|
const int bytes = Q ? 16 : 8;
|
|
const int lanes = bytes / esz;
|
|
const std::string uty = "uint" + std::to_string(esz * 8) + "_t";
|
|
std::string s = "{ " + uty + " _a[" + std::to_string(lanes) + "],_r[" +
|
|
std::to_string(lanes) + "],_m; ";
|
|
s += "memcpy(_a,c->vreg[" + std::to_string(rn) + "]," + std::to_string(bytes) +
|
|
"); ";
|
|
s += "memcpy(&_m,(const uint8_t*)c->vreg[" + std::to_string(rm) + "]+" +
|
|
std::to_string(index * esz) + "," + std::to_string(esz) + "); ";
|
|
if (idxop == 0x8) {
|
|
s += "for(int _i=0;_i<" + std::to_string(lanes) + ";_i++) _r[_i]=(" + uty +
|
|
")((uint64_t)_a[_i]*(uint64_t)_m); ";
|
|
} else {
|
|
s += "memcpy(_r,c->vreg[" + std::to_string(rd) + "]," + std::to_string(bytes) +
|
|
"); ";
|
|
s += "for(int _i=0;_i<" + std::to_string(lanes) + ";_i++) _r[_i]=(" + uty +
|
|
")(_r[_i]" + std::string(idxop == 0x0 ? "+" : "-") + "(" + uty +
|
|
")((uint64_t)_a[_i]*(uint64_t)_m)); ";
|
|
}
|
|
s += "c->vreg[" + std::to_string(rd) + "][0]=0; c->vreg[" + std::to_string(rd) +
|
|
"][1]=0; ";
|
|
s += "memcpy(c->vreg[" + std::to_string(rd) + "],_r," + std::to_string(bytes) +
|
|
"); }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Integer leading-bit counts, halving arithmetic, absolute differences,
|
|
// and narrowing high-half arithmetic. These operations do not affect QC.
|
|
{
|
|
const bool leading = (i & 0x9F3FFC00) == 0x0E204800;
|
|
const u32 half_op = i & 0x9F20FC00;
|
|
const bool halving = half_op == 0x0E200400 || half_op == 0x0E201400 || half_op == 0x0E202400;
|
|
const bool difference = (i & 0x9F20F400) == 0x0E207400;
|
|
const bool high_narrow = (i & 0x9F20DC00) == 0x0E204000;
|
|
const u32 size = (i >> 22) & 3;
|
|
if ((leading || halving || difference || high_narrow) && size < 3) {
|
|
const u32 Q = (i >> 30) & 1, U = (i >> 29) & 1;
|
|
const u32 rn = (i >> 5) & 31, rm = (i >> 16) & 31, rd = i & 31;
|
|
const int esz = 1 << size, bits = esz * 8, bytes = Q ? 16 : 8;
|
|
const int lanes = high_narrow ? 8 / esz : bytes / esz;
|
|
const std::string dest = "uint" + std::to_string(bits) + "_t";
|
|
const std::string src = std::string(high_narrow || leading || U ? "uint" : "int") +
|
|
std::to_string(high_narrow ? bits * 2 : bits) + "_t";
|
|
std::string s = "{ " + src + " _a[" + std::to_string(lanes) + "]; " + dest +
|
|
" _r[" + std::to_string(lanes) + "]; ";
|
|
s += "memcpy(_a,c->vreg[" + std::to_string(rn) + "]," + std::to_string(high_narrow ? 16 : bytes) + "); ";
|
|
if (!leading) {
|
|
s += src + " _b[" + std::to_string(lanes) + "]; memcpy(_b,c->vreg[" +
|
|
std::to_string(rm) + "]," + std::to_string(high_narrow ? 16 : bytes) + "); ";
|
|
}
|
|
const bool accumulate = difference && (i & 0x800);
|
|
if (accumulate) s += "memcpy(_r,c->vreg[" + std::to_string(rd) + "]," + std::to_string(bytes) + "); ";
|
|
s += "for(int _j=0;_j<" + std::to_string(lanes) + ";++_j) { ";
|
|
if (leading) {
|
|
s += "uint64_t _x=_a[_j]; unsigned _n=0; ";
|
|
if (!U) s += "if(_x&(UINT64_C(1)<<" + std::to_string(bits - 1) + ")) _x=(" + dest + ")~_x; ";
|
|
s += "for(int _b=" + std::to_string(bits - (U ? 1 : 2)) + ";_b>=0;--_b) { if((_x>>_b)&1) break; ++_n; } _r[_j]=(" + dest + ")_n; ";
|
|
} else if (high_narrow) {
|
|
s += "uint64_t _x=(uint64_t)_a[_j]" + std::string(i & 0x2000 ? "-" : "+") + "(uint64_t)_b[_j]";
|
|
if (U) s += "+(UINT64_C(1)<<" + std::to_string(bits - 1) + ")";
|
|
s += "; _r[_j]=(" + dest + ")(_x>>" + std::to_string(bits) + "); ";
|
|
} else if (halving) {
|
|
s += "int64_t _x=(int64_t)_a[_j]" + std::string(half_op == 0x0E202400 ? "-" : "+") + "(int64_t)_b[_j]";
|
|
if (half_op == 0x0E201400) s += "+1";
|
|
s += "; _r[_j]=(" + dest + ")(_x<0?-((-_x+1)/2):_x/2); ";
|
|
} else {
|
|
s += "int64_t _x=(int64_t)_a[_j]-(int64_t)_b[_j]; uint64_t _abs=(uint64_t)(_x<0?-_x:_x); ";
|
|
s += "_r[_j]=(" + dest + ")(" + std::string(accumulate ? "(uint64_t)_r[_j]+" : "") + "_abs); ";
|
|
}
|
|
s += "} ";
|
|
if (high_narrow) {
|
|
if (!Q) s += "c->vreg[" + std::to_string(rd) + "][1]=0; ";
|
|
s += "memcpy(&c->vreg[" + std::to_string(rd) + "][" + std::to_string(Q) + "],_r,8); }";
|
|
} else s += "memset(c->vreg[" + std::to_string(rd) + "],0,16); memcpy(c->vreg[" +
|
|
std::to_string(rd) + "],_r," + std::to_string(bytes) + "); }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// Integer widening add/subtract and pairwise long add/accumulate.
|
|
// Unsigned arithmetic at 64 bits implements modular destination writes,
|
|
// including signed wide additions that would otherwise overflow C types.
|
|
{
|
|
const bool widen = (i & 0x9F20CC00) == 0x0E200000;
|
|
const bool paired = (i & 0x9F3FBC00) == 0x0E202800;
|
|
const bool absolute = (i & 0x9F20DC00) == 0x0E205000;
|
|
const u32 size = (i >> 22) & 3;
|
|
if ((widen || paired || absolute) && size < 3) {
|
|
const u32 Q = (i >> 30) & 1, U = (i >> 29) & 1;
|
|
const u32 rn = (i >> 5) & 31, rm = (i >> 16) & 31, rd = i & 31;
|
|
const bool wide = widen && (i & 0x1000), sub = widen && (i & 0x2000);
|
|
const bool accumulate = (paired && (i & 0x4000)) || (absolute && !(i & 0x2000));
|
|
const int esz = 1 << size, bytes = paired ? (Q ? 16 : 8) : 16;
|
|
const int lanes = bytes / (2 * esz);
|
|
const std::string narrow = std::string(U ? "uint" : "int") + std::to_string(esz * 8) + "_t";
|
|
const std::string dest = "uint" + std::to_string(esz * 16) + "_t";
|
|
std::string s = "{ " + dest + " _r[8]={0}; ";
|
|
if (!wide) s += narrow + " _a[16]; ";
|
|
if (widen || absolute) s += narrow + " _b[8]; ";
|
|
if (wide || accumulate) s += dest + " _w[8]; ";
|
|
if (wide) s += "memcpy(_w,c->vreg[" + std::to_string(rn) + "],16); ";
|
|
else s += "memcpy(_a," + std::string(paired ? "c->vreg[" : "&c->vreg[") +
|
|
std::to_string(rn) + (paired ? "]," : "][" + std::to_string(Q) + "],") +
|
|
std::to_string(paired ? bytes : 8) + "); ";
|
|
if (widen || absolute) s += "memcpy(_b,&c->vreg[" + std::to_string(rm) + "][" + std::to_string(Q) + "],8); ";
|
|
if (accumulate) s += "memcpy(_w,c->vreg[" + std::to_string(rd) + "]," + std::to_string(bytes) + "); ";
|
|
s += "for(int _j=0;_j<" + std::to_string(lanes) + ";++_j) { ";
|
|
if (absolute) s += "int64_t _x=(int64_t)_a[_j]-(int64_t)_b[_j]; ";
|
|
s += "_r[_j]=(" + dest + ")(";
|
|
if (accumulate) s += "(uint64_t)_w[_j]+";
|
|
if (absolute) s += "(uint64_t)(_x<0?-_x:_x)";
|
|
else s += "(uint64_t)" + std::string(paired ? "_a[2*_j]" : wide ? "_w[_j]" : "_a[_j]") +
|
|
(sub ? "-" : "+") + "(uint64_t)" + (paired ? "_a[2*_j+1]" : "_b[_j]");
|
|
s += "); } ";
|
|
s += "memset(c->vreg[" + std::to_string(rd) + "],0,16); memcpy(c->vreg[" +
|
|
std::to_string(rd) + "],_r," + std::to_string(bytes) + "); }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// SADDLV / UADDLV: sum every lane into one element of twice the width.
|
|
if ((i & 0x9F3FFC00) == 0x0E303800) {
|
|
const u32 Q = (i >> 30) & 1, U = (i >> 29) & 1, size = (i >> 22) & 3;
|
|
const u32 rn = (i >> 5) & 31, rd = i & 31;
|
|
const int esz = 1 << size;
|
|
const int bytes = Q ? 16 : 8;
|
|
if (size < 3 && !(size == 2 && !Q)) {
|
|
const int lanes = bytes / esz;
|
|
const std::string sty =
|
|
std::string(U ? "uint" : "int") + std::to_string(esz * 8) + "_t";
|
|
const std::string dty =
|
|
std::string(U ? "uint" : "int") + std::to_string(esz * 16) + "_t";
|
|
std::string s = "{ " + sty + " _a[" + std::to_string(lanes) + "]; " + dty + " _s=0; ";
|
|
s += "memcpy(_a,c->vreg[" + std::to_string(rn) + "]," + std::to_string(bytes) + "); ";
|
|
s += "for(int _i=0;_i<" + std::to_string(lanes) + ";_i++) _s=(" + dty + ")(_s+(" + dty +
|
|
")_a[_i]); ";
|
|
s += "c->vreg[" + std::to_string(rd) + "][0]=0; c->vreg[" + std::to_string(rd) +
|
|
"][1]=0; ";
|
|
s += "memcpy(c->vreg[" + std::to_string(rd) + "],&_s," + std::to_string(esz * 2) +
|
|
"); }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// Advanced SIMD copy: DUP, INS, SMOV and UMOV. These move single lanes
|
|
// between vector registers and the general registers, which is how any
|
|
// scalar value gets into or out of vector code - so they turn up
|
|
// constantly. imm5 encodes both the element width and the lane index:
|
|
// the position of its lowest set bit gives the width, and the bits above
|
|
// it give the index.
|
|
{
|
|
const bool vector_copy = (i & 0x9FE08400) == 0x0E000400;
|
|
const bool scalar_copy = (i & 0xFFE08400) == 0x5E000400;
|
|
if (vector_copy || scalar_copy) {
|
|
const u32 Q = (i >> 30) & 1, op = (i >> 29) & 1;
|
|
const u32 imm5 = (i >> 16) & 0x1F, imm4 = (i >> 11) & 0xF;
|
|
const u32 rn = (i >> 5) & 31, rd = i & 31;
|
|
int size = -1;
|
|
u32 index = 0;
|
|
if (imm5 & 1) { size = 0; index = imm5 >> 1; }
|
|
else if (imm5 & 2) { size = 1; index = imm5 >> 2; }
|
|
else if (imm5 & 4) { size = 2; index = imm5 >> 3; }
|
|
else if (imm5 & 8) { size = 3; index = imm5 >> 4; }
|
|
if (size >= 0) {
|
|
const int esz = 1 << size;
|
|
const std::string sn = std::to_string(rn), sd = std::to_string(rd);
|
|
const std::string off = std::to_string(index * esz);
|
|
// Read the source lane into _e as a byte copy; this avoids any
|
|
// assumption about how the 128-bit register is split in two.
|
|
const std::string read_lane =
|
|
"uint8_t _s[16]; memcpy(_s,c->vreg[" + sn + "],16); uint64_t _e=0; "
|
|
"memcpy(&_e,_s+" + off + "," + std::to_string(esz) + "); ";
|
|
|
|
if (scalar_copy && !op && imm4 == 0) {
|
|
// DUP (scalar): the named lane alone, rest of the register zeroed.
|
|
put("{ " + read_lane + "c->vreg[" + sd + "][0]=_e; c->vreg[" + sd +
|
|
"][1]=0; }");
|
|
return true;
|
|
}
|
|
if (vector_copy && !op && (imm4 == 0 || imm4 == 1)) {
|
|
// DUP (element) or DUP (general): every lane takes the value.
|
|
const int lanes = (Q ? 16 : 8) / esz;
|
|
std::string s = "{ ";
|
|
s += (imm4 == 0) ? read_lane
|
|
: ("uint64_t _e=" + Xz(rn) + "; ");
|
|
s += "uint8_t _d[16]; memset(_d,0,16); ";
|
|
s += "for(int _i=0;_i<" + std::to_string(lanes) + ";_i++) memcpy(_d+_i*" +
|
|
std::to_string(esz) + ",&_e," + std::to_string(esz) + "); ";
|
|
s += "memcpy(c->vreg[" + sd + "],_d,16); }";
|
|
put(s);
|
|
return true;
|
|
}
|
|
if (vector_copy && !op && (imm4 == 5 || imm4 == 7) && rd != 31) {
|
|
// SMOV / UMOV: one lane out into a general register.
|
|
std::string s = "{ " + read_lane;
|
|
if (imm4 == 5) {
|
|
// Sign-extend from the element width.
|
|
const char* st = size == 0 ? "int8_t"
|
|
: size == 1 ? "int16_t"
|
|
: "int32_t";
|
|
s += "uint64_t _r=(uint64_t)(int64_t)(" + std::string(st) + ")_e; ";
|
|
if (!Q) s += "_r &= 0xFFFFFFFFULL; ";
|
|
s += "c->x[" + sd + "]=_r; }";
|
|
} else {
|
|
s += "c->x[" + sd + "]=_e; }";
|
|
}
|
|
put(s);
|
|
return true;
|
|
}
|
|
if (vector_copy && !op && imm4 == 3) {
|
|
// INS (general): overwrite one lane, leave the others alone.
|
|
put("{ uint8_t _d[16]; memcpy(_d,c->vreg[" + sd + "],16); uint64_t _e=" +
|
|
Xz(rn) + "; memcpy(_d+" + off + ",&_e," + std::to_string(esz) +
|
|
"); memcpy(c->vreg[" + sd + "],_d,16); }");
|
|
return true;
|
|
}
|
|
if (vector_copy && op) {
|
|
// INS (element): imm4 holds the source lane, above the bits
|
|
// the element width occupies.
|
|
const u32 src_index = imm4 >> size;
|
|
put("{ uint8_t _s[16],_d[16]; memcpy(_s,c->vreg[" + sn +
|
|
"],16); memcpy(_d,c->vreg[" + sd + "],16); memcpy(_d+" + off + ",_s+" +
|
|
std::to_string(src_index * esz) + "," + std::to_string(esz) +
|
|
"); memcpy(c->vreg[" + sd + "],_d,16); }");
|
|
return true;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// MLA / MLS (vector): accumulate the low element-width product modulo 2^bits.
|
|
// Widen before multiplication to avoid signed promotion overflow for H lanes.
|
|
if ((i & 0x9F20FC00) == 0x0E209400 && ((i >> 22) & 3) < 3) {
|
|
const u32 size = (i >> 22) & 3, Q = (i >> 30) & 1;
|
|
const u32 rm = (i >> 16) & 31, rn = (i >> 5) & 31, rd = i & 31;
|
|
const bool subtract = ((i >> 29) & 1) != 0;
|
|
const int bytes = Q ? 16 : 8, lanes = bytes >> size;
|
|
const std::string ty = "uint" + std::to_string(8 << size) + "_t";
|
|
std::string s = "{ " + ty + " _a[" + std::to_string(lanes) + "],_b[" +
|
|
std::to_string(lanes) + "],_d[" + std::to_string(lanes) + "],_r[" +
|
|
std::to_string(lanes) + "]; ";
|
|
s += "memcpy(_a,c->vreg[" + std::to_string(rn) + "]," + std::to_string(bytes) + "); ";
|
|
s += "memcpy(_b,c->vreg[" + std::to_string(rm) + "]," + std::to_string(bytes) + "); ";
|
|
s += "memcpy(_d,c->vreg[" + std::to_string(rd) + "]," + std::to_string(bytes) + "); ";
|
|
s += "for(int _i=0;_i<" + std::to_string(lanes) + ";++_i) _r[_i]=(" + ty +
|
|
")((uint64_t)_d[_i]" + (subtract ? "-" : "+") +
|
|
"(uint64_t)_a[_i]*(uint64_t)_b[_i]); ";
|
|
s += "memcpy(c->vreg[" + std::to_string(rd) + "],_r," + std::to_string(bytes) + "); ";
|
|
if (!Q) s += "c->vreg[" + std::to_string(rd) + "][1]=0; ";
|
|
put(s + "}");
|
|
return true;
|
|
}
|
|
|
|
// Advanced SIMD three-register same (0x0E/0x4E/0x6E group).
|
|
// Bit pattern: Q U 0 1 1 1 0 size 1 Rm opcode 1 Rn Rd
|
|
// Fixed bits: [28:24]=01110, bit[21]=1, bit[10]=1
|
|
// Handles the most common vector operations needed by games.
|
|
// Element operations use memcpy to stay strictly conforming C.
|
|
if ((i & 0x9F200400) == 0x0E200400) {
|
|
const u32 Q = (i >> 30) & 1; // 0=64-bit half-register, 1=128-bit full
|
|
const u32 U = (i >> 29) & 1;
|
|
const u32 size = (i >> 22) & 3; // 0=B 1=H 2=S 3=D (integer); sz for FP
|
|
const u32 rm = (i >> 16) & 31;
|
|
const u32 opc5 = (i >> 11) & 31; // bits[15:11]
|
|
const u32 rn = (i >> 5) & 31;
|
|
const u32 rd = i & 31;
|
|
const int vbytes = Q ? 16 : 8;
|
|
const int esz = 1 << size; // bytes per integer element
|
|
const int nelems = vbytes / esz;
|
|
|
|
// Bitwise ops (opcode=3): AND/BIC/ORR/ORN (U=0); EOR/BSL/BIT/BIF (U=1).
|
|
// These operate on the full register; element size encodes which variant.
|
|
if (opc5 == 3) {
|
|
// Build body string; if left empty, fall through to unhandled.
|
|
std::string body;
|
|
const std::string vd0 = "c->vreg[" + std::to_string(rd) + "][0]";
|
|
const std::string vd1 = "c->vreg[" + std::to_string(rd) + "][1]";
|
|
const std::string vn0 = "c->vreg[" + std::to_string(rn) + "][0]";
|
|
const std::string vn1 = "c->vreg[" + std::to_string(rn) + "][1]";
|
|
const std::string vm0 = "c->vreg[" + std::to_string(rm) + "][0]";
|
|
const std::string vm1 = "c->vreg[" + std::to_string(rm) + "][1]";
|
|
if (!U) {
|
|
const char* op0 = nullptr; const char* op1 = nullptr;
|
|
if (size == 0) { op0="&"; op1="&"; } // AND
|
|
else if (size == 1) { op0="&~"; op1="&~"; } // BIC
|
|
else if (size == 2) { op0="|"; op1="|"; } // ORR
|
|
else { op0="|~"; op1="|~"; } // ORN
|
|
body = vd0 + "=" + vn0 + op0 + vm0 + "; " + vd1 + "=" + vn1 + op1 + vm1 + "; ";
|
|
} else {
|
|
if (size == 0) { // EOR
|
|
body = vd0 + "=" + vn0 + "^" + vm0 + "; " + vd1 + "=" + vn1 + "^" + vm1 + "; ";
|
|
} else if (size == 1) { // BSL: Vd = (Vd & Vn) | (~Vd & Vm)
|
|
body = vd0 + "=(" + vd0 + "&" + vn0 + ")|(~" + vd0 + "&" + vm0 + "); ";
|
|
body += vd1 + "=(" + vd1 + "&" + vn1 + ")|(~" + vd1 + "&" + vm1 + "); ";
|
|
}
|
|
// BIT (size=2) / BIF (size=3): leave on fallback
|
|
}
|
|
if (!body.empty()) {
|
|
std::string s = "{ " + body;
|
|
if (!Q) s += vd1 + "=0; ";
|
|
s += "}";
|
|
put(s);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// Integer ADD (opc5=16, U=0) and SUB (opc5=16, U=1), element-wise.
|
|
if (opc5 == 16) {
|
|
const char* iop = U ? "-" : "+";
|
|
const int b8 = 8 * esz;
|
|
const std::string ty = "uint" + std::to_string(b8) + "_t";
|
|
std::string s = "{ " + ty + " _a[" + std::to_string(nelems) + "],_b[" + std::to_string(nelems) + "],_r[" + std::to_string(nelems) + "]; ";
|
|
s += "memcpy(_a,c->vreg[" + std::to_string(rn) + "]," + std::to_string(vbytes) + "); ";
|
|
s += "memcpy(_b,c->vreg[" + std::to_string(rm) + "]," + std::to_string(vbytes) + "); ";
|
|
s += "for(int _i=0;_i<" + std::to_string(nelems) + ";_i++) _r[_i]=(" + ty + ")(_a[_i]" + iop + "_b[_i]); ";
|
|
s += "memcpy(c->vreg[" + std::to_string(rd) + "],_r," + std::to_string(vbytes) + "); ";
|
|
if (!Q) s += "c->vreg[" + std::to_string(rd) + "][1]=0; ";
|
|
s += "}";
|
|
put(s);
|
|
return true;
|
|
}
|
|
|
|
// MUL (opcode 10011, U=0) - element-wise multiply, integer only.
|
|
// U=1 at the same opcode is PMUL (polynomial), which is not the same
|
|
// operation and stays on the fallback.
|
|
if (opc5 == 0x13 && !U && size < 3) {
|
|
const int b8 = 8 * esz;
|
|
const std::string ty = "uint" + std::to_string(b8) + "_t";
|
|
std::string s = "{ " + ty + " _a[" + std::to_string(nelems) + "],_b[" + std::to_string(nelems) + "],_r[" + std::to_string(nelems) + "]; ";
|
|
s += "memcpy(_a,c->vreg[" + std::to_string(rn) + "]," + std::to_string(vbytes) + "); ";
|
|
s += "memcpy(_b,c->vreg[" + std::to_string(rm) + "]," + std::to_string(vbytes) + "); ";
|
|
s += "for(int _i=0;_i<" + std::to_string(nelems) + ";_i++) _r[_i]=(" + ty + ")(_a[_i]*_b[_i]); ";
|
|
s += "memcpy(c->vreg[" + std::to_string(rd) + "],_r," + std::to_string(vbytes) + "); ";
|
|
if (!Q) s += "c->vreg[" + std::to_string(rd) + "][1]=0; ";
|
|
s += "}";
|
|
put(s);
|
|
return true;
|
|
}
|
|
|
|
// FP three-register same. Here the two "size" bits mean something
|
|
// different from the integer forms: bit 23 is an opcode-extension bit
|
|
// (it picks FSUB over FADD, FMLS over FMLA) and bit 22 is sz, the
|
|
// element width. Treating bit 23 as part of an element size is what
|
|
// makes FADD look like an unrelated instruction, so the two are split
|
|
// apart explicitly here.
|
|
{
|
|
const u32 a = (i >> 23) & 1; // opcode extension, not a size bit
|
|
const bool dbl = ((i >> 22) & 1) != 0; // sz: 0 = float, 1 = double
|
|
const int fsz = dbl ? 8 : 4;
|
|
const int fne = vbytes / fsz;
|
|
// opcode 11001: FMLA (a=0) / FMLS (a=1), both U=0.
|
|
if (opc5 == 0x19 && !U) {
|
|
const std::string ty=dbl?"uint64_t":"uint32_t";
|
|
put("{"+ty+" _n["+std::to_string(fne)+"],_m["+std::to_string(fne)+"],_d["+std::to_string(fne)+"];"
|
|
"memcpy(_n,c->vreg["+std::to_string(rn)+"],"+std::to_string(vbytes)+");"
|
|
"memcpy(_m,c->vreg["+std::to_string(rm)+"],"+std::to_string(vbytes)+");"
|
|
"memcpy(_d,c->vreg["+std::to_string(rd)+"],"+std::to_string(vbytes)+");"
|
|
"for(unsigned _j=0;_j<"+std::to_string(fne)+";++_j){uint64_t _a=_n[_j],_b=_m[_j],_z=_d[_j],_v;");
|
|
if(a)put("_a^="+std::string(dbl?"0x8000000000000000ULL":"0x80000000ULL")+";");
|
|
put(EmitFPMulAddValue(dbl));
|
|
put("_d[_j]=("+ty+")_v;}memcpy(c->vreg["+std::to_string(rd)+"],_d,"+std::to_string(vbytes)+");");
|
|
if(!Q)put("c->vreg["+std::to_string(rd)+"][1]=0;");
|
|
put("}");
|
|
return true;
|
|
}
|
|
// opcode 11011 with U=1 and a=0 is FMUL. The U=0 encoding at the
|
|
// same opcode is FMULX, which differs on infinity times zero, so
|
|
// it is deliberately left alone rather than aliased to FMUL.
|
|
if (opc5 == 0x1B && U && !a) {
|
|
const std::string ty=dbl?"uint64_t":"uint32_t";
|
|
put("{"+ty+" _n["+std::to_string(fne)+"],_m["+std::to_string(fne)+"],_d["+std::to_string(fne)+"];"
|
|
"memcpy(_n,c->vreg["+std::to_string(rn)+"],"+std::to_string(vbytes)+");"
|
|
"memcpy(_m,c->vreg["+std::to_string(rm)+"],"+std::to_string(vbytes)+");"
|
|
"for(unsigned _j=0;_j<"+std::to_string(fne)+";++_j){uint64_t _a=_n[_j],_b=_m[_j],_z=(_a^_b)&"+
|
|
std::string(dbl?"0x8000000000000000ULL":"0x80000000ULL")+",_v;");
|
|
put(EmitFPMulAddValue(dbl,true));
|
|
put("_d[_j]=("+ty+")_v;}memcpy(c->vreg["+std::to_string(rd)+"],_d,"+std::to_string(vbytes)+");");
|
|
if(!Q)put("c->vreg["+std::to_string(rd)+"][1]=0;");
|
|
put("}");
|
|
return true;
|
|
}
|
|
// opcode 11111 with U=1 and a=0 is FDIV.
|
|
if (opc5 == 0x1F && U && !a) {
|
|
const std::string ty=dbl?"uint64_t":"uint32_t";
|
|
put("{"+ty+" _n["+std::to_string(fne)+"],_m["+std::to_string(fne)+"],_d["+std::to_string(fne)+"];"
|
|
"memcpy(_n,c->vreg["+std::to_string(rn)+"],"+std::to_string(vbytes)+");"
|
|
"memcpy(_m,c->vreg["+std::to_string(rm)+"],"+std::to_string(vbytes)+");"
|
|
"for(unsigned _j=0;_j<"+std::to_string(fne)+";++_j){uint64_t _a=_n[_j],_b=_m[_j],_v;");
|
|
put(EmitFPDivideValue(dbl));
|
|
put("_d[_j]=("+ty+")_v;}memcpy(c->vreg["+std::to_string(rd)+"],_d,"+std::to_string(vbytes)+");");
|
|
if(!Q)put("c->vreg["+std::to_string(rd)+"][1]=0;");
|
|
put("}");
|
|
return true;
|
|
}
|
|
}
|
|
}
|
|
|
|
// SIMD/FP load/store pair with V=1 (LDP/STP for S, D, Q registers).
|
|
// Same general format as the integer pair handler, but with SIMD registers.
|
|
// Sizes: opc=00→32-bit(S), opc=01→64-bit(D), opc=10→128-bit(Q).
|
|
if ((i & 0x3E000000) == 0x2C000000) {
|
|
const u32 opc = i >> 30;
|
|
const bool is_load = (i >> 22) & 1;
|
|
const u32 mode = (i >> 23) & 3; // 1=post, 2=signed-offset, 3=pre
|
|
const u32 rt2 = (i >> 10) & 31, rn = (i >> 5) & 31, rt = i & 31;
|
|
s32 imm7 = (s32)((i >> 15) & 0x7F);
|
|
if (imm7 & 0x40) imm7 |= ~0x7F;
|
|
if ((opc <= 2) && mode >= 1 && mode <= 3) {
|
|
const u32 sz = opc == 0 ? 4 : opc == 1 ? 8 : 16;
|
|
const s64 off = (s64)imm7 * (s64)sz;
|
|
const int bits = sz * 8;
|
|
std::string s = "{ uint64_t _b=c->x[" + std::to_string(rn) + "]; int64_t _o=" + std::to_string((long long)off) + "; ";
|
|
// std::string, not const char*: this gets concatenated with byte
|
|
// offsets below, and as a pointer that silently becomes pointer
|
|
// arithmetic on the literal rather than building an expression.
|
|
const std::string addr = (mode == 1) ? std::string("_b") : std::string("(_b+_o)");
|
|
if (is_load) {
|
|
if (sz <= 8) {
|
|
s += "{ uint64_t _p0,_p1; recomp_ldp" + std::to_string(bits) + "(c," + addr +
|
|
",&_p0,&_p1); ";
|
|
// The pair helper zero-extends S values; write the entire low half.
|
|
s += "c->vreg[" + std::to_string(rt) + "][0]=_p0; c->vreg[" +
|
|
std::to_string(rt) + "][1]=0; ";
|
|
s += "c->vreg[" + std::to_string(rt2) + "][0]=_p1; c->vreg[" +
|
|
std::to_string(rt2) + "][1]=0; }";
|
|
} else {
|
|
// 128-bit: one pair per register.
|
|
s += "recomp_ldp64(c," + addr + ",&c->vreg[" + std::to_string(rt) +
|
|
"][0],&c->vreg[" + std::to_string(rt) + "][1]); ";
|
|
s += "recomp_ldp64(c," + addr + "+16,&c->vreg[" + std::to_string(rt2) +
|
|
"][0],&c->vreg[" + std::to_string(rt2) + "][1]); ";
|
|
}
|
|
} else {
|
|
if (sz <= 8) {
|
|
s += "{ uint64_t _p0=0,_p1=0; memcpy(&_p0,&c->vreg[" + std::to_string(rt) +
|
|
"][0]," + std::to_string(sz) + "); memcpy(&_p1,&c->vreg[" +
|
|
std::to_string(rt2) + "][0]," + std::to_string(sz) + "); ";
|
|
s += "recomp_stp" + std::to_string(bits) + "(c," + addr + ",_p0,_p1); }";
|
|
} else {
|
|
s += "recomp_stp64(c," + addr + ",c->vreg[" + std::to_string(rt) +
|
|
"][0],c->vreg[" + std::to_string(rt) + "][1]); ";
|
|
s += "recomp_stp64(c," + addr + "+16,c->vreg[" + std::to_string(rt2) +
|
|
"][0],c->vreg[" + std::to_string(rt2) + "][1]); ";
|
|
}
|
|
}
|
|
if (mode == 1 || mode == 3) s += "c->x[" + std::to_string(rn) + "]=_b+_o; ";
|
|
s += "}";
|
|
put(s);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// SIMD/FP single-register load and store: the V=1 counterparts of the
|
|
// integer forms handled further up. Access width is a scale value where
|
|
// 0..3 mean 1/2/4/8 bytes and 4 means a whole 16-byte Q register; the
|
|
// 16-byte case is encoded as size==00 with the high bit of opc set, which
|
|
// is why the width cannot simply be read off the size field.
|
|
const auto simd_scale = [](u32 size, u32 opc) {
|
|
return (size == 0 && (opc & 2)) ? 4 : (int)size;
|
|
};
|
|
const auto simd_mem = [&](u32 rt, const std::string& ea, int scale, bool is_load) {
|
|
const int nb = 1 << scale;
|
|
const std::string v0 = "c->vreg[" + std::to_string(rt) + "][0]";
|
|
const std::string v1 = "c->vreg[" + std::to_string(rt) + "][1]";
|
|
if (is_load) {
|
|
if (nb == 16) {
|
|
return "recomp_ldp64(c," + ea + ",&" + v0 + ",&" + v1 + "); ";
|
|
}
|
|
// Narrower loads zero the rest of the register, as the architecture
|
|
// requires - the destination is written whole, not merged into.
|
|
return "{ uint64_t _v=recomp_load" + std::to_string(nb * 8) + "(c," + ea + "); " +
|
|
v0 + "=_v; " + v1 + "=0; } ";
|
|
}
|
|
if (nb == 16) {
|
|
return "recomp_stp64(c," + ea + "," + v0 + "," + v1 + "); ";
|
|
}
|
|
return "{ uint64_t _v=0; memcpy(&_v,&" + v0 + "," + std::to_string(nb) + "); recomp_store" +
|
|
std::to_string(nb * 8) + "(c," + ea + ",_v); } ";
|
|
};
|
|
|
|
// LDR/STR (SIMD, register offset): [Xn, Xm{, extend {amount}}].
|
|
if ((i & 0x3F200C00) == 0x3C200800) {
|
|
const u32 size = i >> 30, opc = (i >> 22) & 3;
|
|
const u32 rn = (i >> 5) & 31, rt = i & 31;
|
|
const u32 option = (i >> 13) & 7, S = (i >> 12) & 1;
|
|
const int scale = simd_scale(size, opc);
|
|
// Index register 31 is XZR, not SP.
|
|
const std::string rm_str = Xz((i >> 16) & 31);
|
|
std::string idx;
|
|
switch (option) {
|
|
case 2: idx = "(uint64_t)(uint32_t)" + rm_str; break; // UXTW
|
|
case 3: idx = rm_str; break; // LSL/UXTX
|
|
case 6: idx = "(uint64_t)(int64_t)(int32_t)(uint32_t)" + rm_str; break; // SXTW
|
|
case 7: idx = rm_str; break; // SXTX
|
|
default: break;
|
|
}
|
|
if (!idx.empty()) {
|
|
// The shift amount, when enabled, is the access scale - not the
|
|
// size field, which disagrees with it for the 16-byte form.
|
|
if (S && scale) idx = "((" + idx + ")<<" + std::to_string(scale) + ")";
|
|
const std::string ea = "(c->x[" + std::to_string(rn) + "]+" + idx + ")";
|
|
put("{ " + simd_mem(rt, ea, scale, (opc & 1) != 0) + "}");
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// LDR/STR (SIMD, unsigned 12-bit immediate offset) - the commonest form.
|
|
if ((i & 0x3F000000) == 0x3D000000) {
|
|
const u32 size = i >> 30, opc = (i >> 22) & 3;
|
|
const u32 imm12 = (i >> 10) & 0xFFF, rn = (i >> 5) & 31, rt = i & 31;
|
|
const int scale = simd_scale(size, opc);
|
|
const std::string ea = "(c->x[" + std::to_string(rn) + "]+" +
|
|
std::to_string((unsigned long long)imm12 << scale) + "ULL)";
|
|
put("{ " + simd_mem(rt, ea, scale, (opc & 1) != 0) + "}");
|
|
return true;
|
|
}
|
|
|
|
// LDUR/STUR (SIMD) and the pre/post-indexed immediate forms.
|
|
if ((i & 0x3F000000) == 0x3C000000 && ((i >> 24) & 1) == 0) {
|
|
const u32 size = i >> 30, opc = (i >> 22) & 3, mode = (i >> 10) & 3;
|
|
const u32 rn = (i >> 5) & 31, rt = i & 31;
|
|
s32 imm9 = (s32)((i >> 12) & 0x1FF);
|
|
if (imm9 & 0x100) imm9 |= ~0x1FF;
|
|
if (mode == 0 || mode == 1 || mode == 3) {
|
|
const int scale = simd_scale(size, opc);
|
|
std::string s = "{ uint64_t _b=c->x[" + std::to_string(rn) + "]; int64_t _o=" +
|
|
std::to_string((long long)imm9) + "; ";
|
|
// Post-index accesses the unmodified base; the other two add the
|
|
// offset first. Pre- and post-index both write the base back.
|
|
const std::string ea = (mode == 1) ? "_b" : "(_b+_o)";
|
|
s += simd_mem(rt, ea, scale, (opc & 1) != 0);
|
|
if (mode == 1 || mode == 3) s += "c->x[" + std::to_string(rn) + "]=_b+_o; ";
|
|
s += "}";
|
|
put(s);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// An instruction the decoder doesn't know is reported and stepped over
|
|
// rather than ending the block. recomp_unhandled is a non-fatal stub, and
|
|
// ending the block here would set pc to the following instruction - an
|
|
// address that block discovery never marked as a block start, so the
|
|
// dispatcher could not resolve it and execution would stop dead at the
|
|
// first unimplemented opcode instead of continuing past it.
|
|
// Leaving the block the moment the handler halts is what makes the
|
|
// hand-off to the fallback engine correct: the remaining instructions in
|
|
// this block must not run twice, since the fallback resumes at this same
|
|
// PC and will execute them itself.
|
|
// ---- EXTR (and therefore ROR immediate) ----------------------------
|
|
//
|
|
// ROR Rd, Rn, #imm is EXTR with Rn == Rm, and between the two widths it was
|
|
// 13.1% of every unhandled instruction in a second title - the single
|
|
// largest gap after EXT, and pure integer work. Compilers emit it for
|
|
// rotates in hashing and bit-twiddling code.
|
|
//
|
|
// sf 0 0 100111 N 0 Rm imms Rn Rd. N must track sf: N==sf is the only
|
|
// allocated combination, and imms must be < 32 in the 32-bit form.
|
|
if ((i & 0x7FA00000) == 0x13800000) {
|
|
const u32 sf = i >> 31, N = (i >> 22) & 1;
|
|
const u32 rm = (i >> 16) & 31, imms = (i >> 10) & 63;
|
|
const u32 rn = (i >> 5) & 31, rd = i & 31;
|
|
const u32 width = sf ? 64u : 32u;
|
|
if (N == sf && imms < width) {
|
|
if (rd != 31) {
|
|
std::string s2 = "{ uint64_t _hi=" + (sf ? Xz(rn) : ("(uint64_t)" + Wz(rn))) +
|
|
", _lo=" + (sf ? Xz(rm) : ("(uint64_t)" + Wz(rm))) + ", _r; ";
|
|
if (imms == 0) {
|
|
// Shifting by the full width is undefined in C, and lsb==0
|
|
// is the common case (a plain MOV-through-EXTR).
|
|
s2 += "_r = _lo; ";
|
|
} else {
|
|
s2 += "_r = (_lo >> " + std::to_string(imms) + ") | (_hi << " +
|
|
std::to_string(width - imms) + "); ";
|
|
}
|
|
if (!sf) s2 += "_r &= 0xFFFFFFFFULL; ";
|
|
s2 += "c->x[" + std::to_string(rd) + "] = _r; }";
|
|
put(s2);
|
|
} else {
|
|
put("/* extr -> xzr */");
|
|
}
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// ---- ADC / ADCS / SBC / SBCS ---------------------------------------
|
|
//
|
|
// sf op S 11010000 Rm 000000 Rn Rd. op: 0 = ADC, 1 = SBC. Small but real -
|
|
// 1.4% of the gap - and it appears in multi-word arithmetic where getting
|
|
// the carry wrong is silent.
|
|
if ((i & 0x1FE0FC00) == 0x1A000000) {
|
|
const u32 sf = i >> 31, op = (i >> 30) & 1, S = (i >> 29) & 1;
|
|
const u32 rm = (i >> 16) & 31, rn = (i >> 5) & 31, rd = i & 31;
|
|
const char* sign = sf ? "0x8000000000000000ULL" : "0x80000000ULL";
|
|
std::string s2 = "{ uint64_t _a=" + (sf ? Xz(rn) : ("(uint64_t)" + Wz(rn))) +
|
|
", _b=" + (sf ? Xz(rm) : ("(uint64_t)" + Wz(rm))) +
|
|
", _ci=(uint64_t)c->c, _r; ";
|
|
// SBC is ADC against the inverted operand, which is also what makes the
|
|
// borrow come out right: carry-clear means borrow.
|
|
if (op) s2 += sf ? "_b = ~_b; " : "_b = (~_b) & 0xFFFFFFFFULL; ";
|
|
s2 += "_r = _a + _b + _ci; ";
|
|
if (!sf) s2 += "_r &= 0xFFFFFFFFULL; ";
|
|
if (S) {
|
|
s2 += std::string("c->z = (_r == 0); c->n = (_r & ") + sign + ") ? 1 : 0; ";
|
|
if (sf) {
|
|
// Unsigned overflow with a carry-in: the sum wraps either when
|
|
// it lands below _a, or when it lands exactly on _a with the
|
|
// carry set (_b == ~0 and _ci == 1).
|
|
s2 += "c->c = (_r < _a) || (_ci && _r == _a); ";
|
|
} else {
|
|
s2 += "c->c = ((_a + _b + _ci) > 0xFFFFFFFFULL); ";
|
|
}
|
|
s2 += std::string("c->v = ((~(_a ^ _b) & (_a ^ _r)) & ") + sign + ") ? 1 : 0; ";
|
|
}
|
|
if (rd != 31) {
|
|
s2 += "c->x[" + std::to_string(rd) + "] = _r; ";
|
|
}
|
|
s2 += "}";
|
|
put(s2);
|
|
return true;
|
|
}
|
|
|
|
// ---- LDPSW ----------------------------------------------------------
|
|
//
|
|
// Load pair of 32-bit words, each sign-extended to 64. opc=01 V=0 L=1, so
|
|
// it sits outside the LDP/STP handler above, which only covers opc 00/10.
|
|
// 3.1% of the gap, and it turns up wherever a pair of ints is read into
|
|
// 64-bit registers at once.
|
|
//
|
|
// 0110100xx1 ... : signed offset 0x69400000, post-index 0x68C00000,
|
|
// pre-index 0x69C00000. imm7 is scaled by 4.
|
|
if ((i & 0xFFC00000) == 0x69400000 || (i & 0xFFC00000) == 0x68C00000 ||
|
|
(i & 0xFFC00000) == 0x69C00000) {
|
|
const u32 rt = i & 31, rn = (i >> 5) & 31, rt2 = (i >> 10) & 31;
|
|
const s32 imm7 = ((s32)(((i >> 15) & 0x7F) << 25) >> 25) * 4;
|
|
const bool post = (i & 0xFFC00000) == 0x68C00000;
|
|
const bool pre = (i & 0xFFC00000) == 0x69C00000;
|
|
|
|
std::string s2 = "{ uint64_t _base=" + Xsp(rn) + "; uint64_t _addr = _base";
|
|
if (!post) s2 += " + (int64_t)" + std::to_string(imm7);
|
|
s2 += "; ";
|
|
// Both words are read before either register is written: Rn may be Rt
|
|
// or Rt2, and writing early would corrupt the address for the second
|
|
// load.
|
|
s2 += "uint64_t _v1 = (uint64_t)(int64_t)(int32_t)(uint32_t)recomp_load32(c,_addr); ";
|
|
s2 += "uint64_t _v2 = (uint64_t)(int64_t)(int32_t)(uint32_t)recomp_load32(c,_addr+4); ";
|
|
if (rt != 31) s2 += "c->x[" + std::to_string(rt) + "] = _v1; ";
|
|
if (rt2 != 31) s2 += "c->x[" + std::to_string(rt2) + "] = _v2; ";
|
|
if (post || pre) {
|
|
s2 += "c->x[" + std::to_string(rn) + "] = _base + (int64_t)" +
|
|
std::to_string(imm7) + "; ";
|
|
}
|
|
s2 += "}";
|
|
put(s2);
|
|
return true;
|
|
}
|
|
|
|
put_unhandled();
|
|
return true;
|
|
}
|
|
|
|
|
|
const char* RuntimeH();
|
|
const char* RuntimeC();
|
|
|
|
/// One bucket of the unhandled-instruction histogram.
|
|
struct UnhandledSite {
|
|
size_t count = 0;
|
|
u32 example_insn = 0; ///< a representative full encoding, for hand-decoding
|
|
u64 example_pc = 0; ///< where that example lives, for disassembling in context
|
|
};
|
|
|
|
/// AArch64 top-level encoding group, from bits 28:25 of the instruction.
|
|
/// Table C4-1 in the ARM ARM. Coarse, but enough to say "the gap is SIMD"
|
|
/// versus "the gap is loads and stores" at a glance.
|
|
inline const char* EncodingGroupName(u32 op0) {
|
|
switch (op0 & 0xF) {
|
|
case 0x0: return "reserved/sme";
|
|
case 0x1: case 0x3: return "unallocated";
|
|
case 0x2: return "sve";
|
|
case 0x8: case 0x9: return "dp-immediate";
|
|
case 0xA: case 0xB: return "branch/exception/system";
|
|
case 0x4: case 0x6: case 0xC: case 0xE: return "load/store";
|
|
case 0x5: case 0xD: return "dp-register";
|
|
case 0x7: case 0xF: return "dp-simd/fp";
|
|
default: return "unknown";
|
|
}
|
|
}
|
|
|
|
// Stats returned to the caller for manifest/reporting.
|
|
struct RecompileStats {
|
|
size_t blocks = 0;
|
|
size_t instructions = 0; ///< n_bytes/4: every word in .text, padding included
|
|
size_t translated_terminators = 0;
|
|
size_t emitted = 0; ///< instructions actually walked inside discovered blocks
|
|
size_t unhandled = 0; ///< of those, how many fell through to recomp_unhandled
|
|
|
|
/// Unhandled counts keyed by encoding group (bits 28:25).
|
|
std::map<u32, size_t> unhandled_by_group;
|
|
/// Unhandled counts keyed by the top 10 bits, which usually identifies the
|
|
/// opcode family while dropping register operands. This is the list that
|
|
/// says which instruction to implement next.
|
|
std::map<u32, UnhandledSite> unhandled_by_signature;
|
|
|
|
/// Fraction of walked instructions the decoder could not translate.
|
|
double UnhandledFraction() const {
|
|
return emitted ? double(unhandled) / double(emitted) : 0.0;
|
|
}
|
|
};
|
|
|
|
// Emit a buildable C project that statically recompiles `text` (raw AArch64 .text at `base`).
|
|
// Layout written into out_dir:
|
|
// CMakeLists.txt, main.c, recomp_export.c, recomp_runtime.{c,h} <- hand-readable top level
|
|
// src/recompiled_<mod>.c, src/recompiled_<mod>_<n>.c <- generated translation units
|
|
// data/{text,rodata,data}.bin <- bundled guest segments
|
|
// build/ <- cmake's one canonical build dir
|
|
// The translation units are kept in src/ rather than the module root: a large
|
|
// title emits hundreds of them, and loose in the root they bury the four files
|
|
// anyone actually needs to look at.
|
|
// Optional rodata/data parameters bundle those segments so the exported exe is self-contained.
|
|
inline RecompileStats EmitProject(const std::string& mod, const u8* text, size_t n_bytes, u64 base,
|
|
const std::string& out_dir, bool source_only,
|
|
const u8* rodata = nullptr, size_t rodata_size = 0,
|
|
const u8* data_seg = nullptr, size_t data_size = 0,
|
|
// Guest address to begin executing at. Defaults to `base`, but a
|
|
// real NSO starts with a MOD0 header rather than code, so the
|
|
// loader passes the module's actual entry here.
|
|
u64 entry_pc = 0,
|
|
// Human-readable game name, shown by the generated executable so
|
|
// it identifies itself rather than printing raw addresses.
|
|
const std::string& display_title = {},
|
|
// Addresses of this module's exported dynsym symbols, so
|
|
// block discovery seeds a root at each even when nothing in
|
|
// this module's own .text branches there directly.
|
|
const std::vector<u64>* extra_roots = nullptr,
|
|
// Internal fixture control; production exports use 32 MiB.
|
|
size_t unit_source_limit = 32u << 20) {
|
|
RecompileStats stats;
|
|
auto blocks = DiscoverBlocks(text, n_bytes, base, entry_pc, extra_roots);
|
|
const u32* p = reinterpret_cast<const u32*>(text);
|
|
stats.blocks = blocks.size();
|
|
stats.instructions = n_bytes / 4;
|
|
|
|
// Block bodies are split across several translation units. A whole game is
|
|
// millions of blocks, which as one .c file runs to hundreds of megabytes -
|
|
// enough that a compiler needs hours and a great deal of memory, or gives
|
|
// up outright. Splitting keeps each unit to a sane size and lets the build
|
|
// compile them in parallel.
|
|
constexpr size_t kBlocksPerUnit = 20000;
|
|
// Retain only the current bounded unit and one block's rendered body.
|
|
// Buffering a whole module would require gigabytes on top of game data.
|
|
const auto make_dir = [](const std::string& dir) {
|
|
std::error_code ec;
|
|
std::filesystem::create_directories(Utf8Path(dir), ec);
|
|
};
|
|
make_dir(out_dir);
|
|
// Every generated translation unit lands here, keeping the module root down
|
|
// to the few files a human reads.
|
|
const std::string src_dir = out_dir + "/src";
|
|
make_dir(src_dir);
|
|
// Render one complete block before deciding which unit owns it. Block counts
|
|
// alone do not bound source size: exact vector FP bodies can expand one
|
|
// guest instruction into kilobytes of C. Retain every block, but isolate a
|
|
// single block exceeding the budget rather than silently dropping code.
|
|
std::string unit_body;
|
|
size_t unit_first = 0;
|
|
size_t unit_blocks = 0;
|
|
std::string rcu;
|
|
rcu.reserve(4096);
|
|
|
|
// Hoisted out of the per-instruction loop: a fresh std::string per guest
|
|
// instruction is one allocation per instruction across the whole title.
|
|
std::string body;
|
|
body.reserve(4096);
|
|
char namebuf[64];
|
|
|
|
// Each unit carries the dispatch-table entries for its own blocks. Putting
|
|
// the whole table in one file instead meant a single translation unit that
|
|
// grew with the module - hundreds of megabytes of forward declarations and
|
|
// one array initialiser for a large title, compiled by one cl.exe with no
|
|
// parallelism and enormous peak memory, which is both the longest pole in
|
|
// the build and a plausible way for it to die outright. Here the entries
|
|
// sit next to the definitions they point at, so they cost no extra
|
|
// declarations and compile across every core with the bodies.
|
|
std::vector<u64> seg_hi;
|
|
const auto emit_seg = [&](size_t u) {
|
|
const size_t lo = unit_first;
|
|
const size_t hi = lo + unit_blocks;
|
|
char line[160];
|
|
snprintf(line, sizeof line, "\nconst struct _recomp_ent _seg_%s_%zu[] = {\n", mod.c_str(),
|
|
u);
|
|
unit_body += line;
|
|
for (size_t i = lo; i < hi; ++i) {
|
|
FuncNameTo(namebuf, mod.c_str(), blocks[i].vaddr);
|
|
snprintf(line, sizeof line, " {0x%llxULL, 0x%llxULL, %s},\n",
|
|
(unsigned long long)blocks[i].vaddr,
|
|
(unsigned long long)(blocks[i].vaddr + (u64)(blocks[i].count - 1) * 4),
|
|
namebuf);
|
|
unit_body += line;
|
|
}
|
|
// C has no empty initialiser list; a zero-length segment still needs a
|
|
// well-formed array, and its declared count of 0 keeps it unsearchable.
|
|
if (lo >= hi) {
|
|
unit_body += " {0ULL, 0ULL, 0}\n";
|
|
seg_hi.push_back(0);
|
|
} else {
|
|
const auto& last = blocks[hi - 1];
|
|
seg_hi.push_back(last.vaddr + (u64)(last.count - 1) * 4);
|
|
}
|
|
snprintf(line, sizeof line, "};\nconst unsigned _segn_%s_%zu = %zuU;\n", mod.c_str(), u,
|
|
hi - lo);
|
|
unit_body += line;
|
|
};
|
|
const auto finish_unit = [&] {
|
|
const size_t u = seg_hi.size();
|
|
emit_seg(u);
|
|
std::ofstream output(Utf8Path(src_dir + "/recompiled_" + mod + "_" +
|
|
std::to_string(u) + ".c"), std::ios::binary);
|
|
output << "/* auto-generated by suyu static recompiler - DO NOT EDIT */\n"
|
|
"#include \"recomp_runtime.h\"\n#include <stdint.h>\n"
|
|
"#include <string.h>\n#include <math.h>\n"
|
|
"struct _recomp_ent{uint64_t lo,hi; BlockFn fn;};\n\n";
|
|
if (u == 0) output << "int g_recomp_guard_host_v2=0;\n";
|
|
if (EmitGuardGen()) {
|
|
output << "static uint32_t recomp_gg_seen[" << std::max<size_t>(unit_blocks, 1)
|
|
<< "];\n";
|
|
}
|
|
output.write(unit_body.data(), static_cast<std::streamsize>(unit_body.size()));
|
|
unit_body.clear();
|
|
unit_first += unit_blocks;
|
|
unit_blocks = 0;
|
|
};
|
|
|
|
// Every block's address, so a direct branch can be turned into a direct call
|
|
// when its target is a block we actually emitted.
|
|
std::unordered_set<u64> chain_blocks;
|
|
chain_blocks.reserve(blocks.size() * 2);
|
|
for (const auto& b : blocks) {
|
|
chain_blocks.insert(b.vaddr);
|
|
}
|
|
g_chain_blocks = &chain_blocks;
|
|
g_chain_mod = mod.c_str();
|
|
struct ChainScope {
|
|
~ChainScope() {
|
|
g_chain_blocks = nullptr;
|
|
g_chain_mod = nullptr;
|
|
}
|
|
} chain_scope;
|
|
|
|
for (const auto& b : blocks) {
|
|
rcu.clear();
|
|
FuncNameTo(namebuf, mod.c_str(), b.vaddr);
|
|
rcu += "void ";
|
|
rcu += namebuf;
|
|
rcu += "(GuestContext* c){\n";
|
|
const u32 first = (u32)((b.vaddr - base) / 4);
|
|
rcu += " static const uint32_t _expected[]={";
|
|
if (EmitGuardGen()) {
|
|
// GG1 header: word count, module-relative PC low and high.
|
|
char header[64];
|
|
snprintf(header, sizeof header, "%uU,0x%08xU,0x%08xU,", b.count,
|
|
(unsigned)(b.vaddr & 0xffffffffu), (unsigned)(b.vaddr >> 32));
|
|
rcu += header;
|
|
}
|
|
for (u32 k = 0; k < b.count; ++k) {
|
|
char word[32]; snprintf(word, sizeof word, "0x%08xU,", p[first + k]); rcu += word;
|
|
}
|
|
// Verify each entry, including side entries and direct chains. Do not
|
|
// cache this in a function-static variable: multiple guest cores can
|
|
// enter the same generated block. Per-entry verification also handles
|
|
// process reuse and remapped addresses without an unsafe shared epoch.
|
|
if (EmitGuardGen()) {
|
|
// GG1: the same check, skipped while this block's seen word (its
|
|
// slot of the unit's recomp_gg_seen) matches the module generation.
|
|
rcu += "};\n RECOMP_GG_GUARD(_expected,@GG_SLOT@U);\n";
|
|
} else {
|
|
rcu += "};\n recomp_code_guard(c,g_module_base+" + std::to_string(b.vaddr) +
|
|
"ULL,_expected," + std::to_string(b.count) + "U,g_recomp_guard_host_v2);\n";
|
|
}
|
|
rcu += " if(c->halted) return;\n";
|
|
// Lookup indexes every emitted instruction, not only block starts. An
|
|
// indirect transfer can therefore enter the middle of this function.
|
|
// Direct chains publish their exact target PC before calling, so any
|
|
// address outside this block is an invariant violation, never a request
|
|
// to start silently at the first instruction.
|
|
{
|
|
char addr[32];
|
|
rcu += " { uint64_t _entry=c->pc-g_module_base; if(_entry!=0x";
|
|
snprintf(addr, sizeof addr, "%llx", (unsigned long long)b.vaddr);
|
|
rcu += addr;
|
|
rcu += "ULL){ if(_entry<0x";
|
|
rcu += addr;
|
|
rcu += "ULL || _entry>0x";
|
|
snprintf(addr, sizeof addr, "%llx",
|
|
(unsigned long long)(b.vaddr + (u64)(b.count - 1) * 4));
|
|
rcu += addr;
|
|
rcu += "ULL || ((_entry-0x";
|
|
snprintf(addr, sizeof addr, "%llx", (unsigned long long)b.vaddr);
|
|
rcu += addr;
|
|
rcu += "ULL)&3ULL)!=0){ recomp_unhandled(c,0U,c->pc); return; } switch((unsigned)((_entry-0x";
|
|
rcu += addr;
|
|
rcu += "ULL)>>2)){\n";
|
|
for (u32 k = 1; k < b.count; ++k) {
|
|
char entry[96];
|
|
snprintf(entry, sizeof entry, " case %uU: goto _recomp_entry_%u;\n", k, k);
|
|
rcu += entry;
|
|
}
|
|
rcu += " default: recomp_unhandled(c,0U,c->pc); return;\n } } }\n";
|
|
}
|
|
bool open = true;
|
|
for (u32 k = 0; k < b.count; ++k) {
|
|
if (k > 0) {
|
|
char label[48];
|
|
snprintf(label, sizeof label, "_recomp_entry_%u: ;\n", k);
|
|
rcu += label;
|
|
}
|
|
body.clear();
|
|
const u32 insn = p[first + k];
|
|
const u64 insn_pc = b.vaddr + (u64)k * 4;
|
|
bool unhandled = false;
|
|
open = Translate(insn, insn_pc, body, &unhandled);
|
|
rcu += body;
|
|
++stats.emitted;
|
|
if (unhandled) {
|
|
++stats.unhandled;
|
|
++stats.unhandled_by_group[(insn >> 25) & 0xF];
|
|
auto& site = stats.unhandled_by_signature[insn & 0xFFC00000u];
|
|
if (site.count == 0) {
|
|
site.example_insn = insn;
|
|
site.example_pc = insn_pc;
|
|
}
|
|
++site.count;
|
|
}
|
|
if (!open) ++stats.translated_terminators;
|
|
}
|
|
// A block that ends by running off its own end still has to hand the
|
|
// dispatcher an absolute guest address, exactly as every branch
|
|
// terminator above does. Emitting the bare module-relative offset here
|
|
// was silently catastrophic: the host dispatcher picks the owning image
|
|
// by "greatest base not exceeding the PC", so a small unbased value has
|
|
// no owner at all and falls through to the "try every image at the raw
|
|
// PC" path - which happily returns *some other module's* block at the
|
|
// same offset and runs it against this module's register state.
|
|
if (open) {
|
|
char tail[80];
|
|
snprintf(tail, sizeof tail, " c->pc=g_module_base+0x%llxULL; return;\n",
|
|
(unsigned long long)(b.vaddr + b.size));
|
|
rcu += tail;
|
|
}
|
|
rcu += "}\n\n";
|
|
// Reserve conservative space for the unit header and dispatch entries.
|
|
// The actual table lines fit in emit_seg's 160-byte buffer.
|
|
const size_t projected = unit_body.size() + rcu.size() +
|
|
(unit_blocks + 1) * 160 + 512;
|
|
if (unit_blocks && (unit_blocks == kBlocksPerUnit || projected > unit_source_limit)) {
|
|
finish_unit();
|
|
}
|
|
if (EmitGuardGen()) {
|
|
const size_t slot = rcu.find("@GG_SLOT@");
|
|
rcu.replace(slot, sizeof("@GG_SLOT@") - 1, std::to_string(unit_blocks));
|
|
}
|
|
unit_body += rcu;
|
|
++unit_blocks;
|
|
}
|
|
finish_unit();
|
|
const size_t unit_count = seg_hi.size();
|
|
|
|
// The dispatch table is now just a directory of the per-unit segments
|
|
// emitted above: a few lines per unit instead of two entries per block, so
|
|
// this file stays a few kilobytes no matter how large the module is.
|
|
std::string rc;
|
|
rc.reserve(unit_count * 200 + 4096);
|
|
rc += "/* auto-generated by suyu static recompiler - DO NOT EDIT */\n"
|
|
"#include \"recomp_runtime.h\"\n#include <stdint.h>\n\n"
|
|
"struct _recomp_ent{uint64_t lo,hi; BlockFn fn;};\n\n";
|
|
{
|
|
char line[192];
|
|
for (size_t u = 0; u < unit_count; ++u) {
|
|
snprintf(line, sizeof line,
|
|
"extern const struct _recomp_ent _seg_%s_%zu[];\nextern const unsigned "
|
|
"_segn_%s_%zu;\n",
|
|
mod.c_str(), u, mod.c_str(), u);
|
|
rc += line;
|
|
}
|
|
rc += "\nstatic const struct _recomp_ent* const _segs[] = {\n";
|
|
for (size_t u = 0; u < unit_count; ++u) {
|
|
snprintf(line, sizeof line, " _seg_%s_%zu,\n", mod.c_str(), u);
|
|
rc += line;
|
|
}
|
|
rc += "};\nstatic const unsigned* const _segn[] = {\n";
|
|
for (size_t u = 0; u < unit_count; ++u) {
|
|
snprintf(line, sizeof line, " &_segn_%s_%zu,\n", mod.c_str(), u);
|
|
rc += line;
|
|
}
|
|
// Highest guest address in each segment. Segments follow the block list,
|
|
// which is sorted ascending, so they are disjoint and ordered - this
|
|
// lets the lookup binary-search for the owning segment rather than
|
|
// walking every one of them on each dispatch.
|
|
rc += "};\nstatic const uint64_t _seg_hi[] = {\n";
|
|
for (size_t u = 0; u < unit_count; ++u) {
|
|
snprintf(line, sizeof line, " 0x%llxULL,\n", (unsigned long long)seg_hi[u]);
|
|
rc += line;
|
|
}
|
|
rc += "};\n";
|
|
}
|
|
rc += "#define _NSEG (sizeof(_segs)/sizeof(_segs[0]))\n"
|
|
"/* Direct block index.\n"
|
|
"\n"
|
|
" Every block boundary lands in recomp_lookup - tens of millions of\n"
|
|
" times a second - and it used to run two binary searches, one over\n"
|
|
" segments and one over a segment's entries, which for the main image\n"
|
|
" means about twenty probes over 600k entries per dispatch.\n"
|
|
"\n"
|
|
" Guest instructions are 4-byte aligned and a module's blocks span one\n"
|
|
" contiguous address range, so (pc - lo) >> 2 indexes a flat array\n"
|
|
" directly. One bounds check and one load, no search.\n"
|
|
"\n"
|
|
" Built once from the sorted tables at load time and read-only after,\n"
|
|
" so it needs no locking and no per-thread state - a thread-local\n"
|
|
" cache was tried instead and cost 34%% in the race phase, because the\n"
|
|
" footprint per thread stops being free once the title is actually\n"
|
|
" using its threads.\n"
|
|
"\n"
|
|
" Costs one pointer per guest instruction word in the module, so about\n"
|
|
" 24 MB for a 3M-instruction image. If the allocation fails the binary\n"
|
|
" search below still answers, just slowly. */\n"
|
|
"static BlockFn* _idx = 0;\n"
|
|
"static uint64_t _idx_lo = 0, _idx_hi = 0;\n"
|
|
"static int _idx_tried = 0;\n"
|
|
"static void _build_idx(void){\n"
|
|
" size_t i, j, n;\n"
|
|
" if(_idx_tried) return;\n"
|
|
" _idx_tried = 1;\n"
|
|
" if(_NSEG == 0 || *_segn[0] == 0) return;\n"
|
|
" _idx_lo = _segs[0][0].lo;\n"
|
|
" _idx_hi = _seg_hi[_NSEG-1];\n"
|
|
" if(_idx_hi < _idx_lo) return;\n"
|
|
" n = (size_t)((_idx_hi - _idx_lo) >> 2) + 1;\n"
|
|
" _idx = (BlockFn*)calloc(n, sizeof(BlockFn));\n"
|
|
" if(!_idx) return;\n"
|
|
" for(i=0;i<_NSEG;i++){\n"
|
|
" const struct _recomp_ent* t=_segs[i];\n"
|
|
" unsigned m=*_segn[i];\n"
|
|
" for(j=0;j<m;j++){ uint64_t k,count=((t[j].hi-t[j].lo)>>2)+1;\n"
|
|
" for(k=0;k<count;k++) _idx[(size_t)(((t[j].lo+4*k)-_idx_lo)>>2)]=t[j].fn; }\n"
|
|
" }\n"
|
|
"}\n"
|
|
"/* Called from recomp_image_set_base, which the loader runs once per\n"
|
|
" module before any guest thread exists - so the build below is not\n"
|
|
" racing anything. */\n"
|
|
"void recomp_build_index(void){ _build_idx(); }\n"
|
|
"/* Module-relative view of the block index, for the host dispatcher.\n"
|
|
" The index itself is static to this unit, so the exported wrapper in\n"
|
|
" recomp_export.c has to come through here. */\n"
|
|
"int _recomp_index_view(uint64_t* lo, uint64_t* hi, BlockFn** idx){\n"
|
|
" _build_idx();\n"
|
|
" if(!_idx) return 0;\n"
|
|
" *lo = _idx_lo; *hi = _idx_hi; *idx = _idx;\n"
|
|
" return 1;\n"
|
|
"}\n"
|
|
"\n"
|
|
"BlockFn recomp_lookup(uint64_t pc){\n"
|
|
" if(_idx && pc>=_idx_lo && pc<=_idx_hi && ((pc-_idx_lo)&3ULL)==0) return _idx[(size_t)((pc-_idx_lo)>>2)];\n"
|
|
" {\n"
|
|
" unsigned slo=0, shi=(unsigned)_NSEG;\n"
|
|
" while(slo<shi){ unsigned m=slo+(shi-slo)/2; if(_seg_hi[m]<pc) slo=m+1; else shi=m; }\n"
|
|
" if(slo>=_NSEG) return 0;\n"
|
|
" {\n"
|
|
" const struct _recomp_ent* t=_segs[slo];\n"
|
|
" unsigned n=*_segn[slo], lo=0, hi=n;\n"
|
|
" while(lo<hi){ unsigned m=lo+(hi-lo)/2; if(t[m].lo<=pc) lo=m+1; else hi=m; }\n"
|
|
" if(lo==0) return 0; --lo;\n"
|
|
" return (pc<=t[lo].hi && ((pc-t[lo].lo)&3ULL)==0)?t[lo].fn:0;\n"
|
|
" }\n"
|
|
" }\n}\n";
|
|
|
|
const std::string title_str = display_title.empty() ? mod : display_title;
|
|
std::ostringstream mc;
|
|
mc << "#include \"recomp_runtime.h\"\n#include <stdio.h>\n#include <stdlib.h>\n#include <string.h>\n";
|
|
mc << "#ifdef HAVE_SDL2\n#include <SDL2/SDL.h>\n#endif\n\n";
|
|
mc << "#define GUEST_MEM_SIZE (256ULL * 1024 * 1024) /* 256 MB */\n\n";
|
|
mc << "static const char* kGameTitle = \"" << title_str << "\";\n\n";
|
|
mc << "#ifdef HAVE_SDL2\n";
|
|
mc << "static SDL_Window* g_sdl_window = NULL;\n";
|
|
mc << "static SDL_Renderer* g_sdl_renderer = NULL;\n";
|
|
mc << "static SDL_Texture* g_sdl_texture = NULL;\n";
|
|
mc << "static int g_fb_w = 1280, g_fb_h = 720;\n\n";
|
|
mc << "/* Called from the SVC handler when the guest flushes a framebuffer.\n";
|
|
mc << " fb: CPU-accessible RGBA8 pixels, w x h. */\n";
|
|
mc << "void recomp_sdl_blit(const void* fb, int w, int h) {\n";
|
|
mc << " if(!g_sdl_renderer) return;\n";
|
|
mc << " if(w!=g_fb_w || h!=g_fb_h || !g_sdl_texture) {\n";
|
|
mc << " if(g_sdl_texture) SDL_DestroyTexture(g_sdl_texture);\n";
|
|
mc << " g_sdl_texture = SDL_CreateTexture(g_sdl_renderer,\n";
|
|
mc << " SDL_PIXELFORMAT_RGBA32, SDL_TEXTUREACCESS_STREAMING, w, h);\n";
|
|
mc << " g_fb_w=w; g_fb_h=h;\n";
|
|
mc << " }\n";
|
|
mc << " SDL_UpdateTexture(g_sdl_texture, NULL, fb, w*4);\n";
|
|
mc << " SDL_RenderClear(g_sdl_renderer);\n";
|
|
mc << " SDL_RenderCopy(g_sdl_renderer, g_sdl_texture, NULL, NULL);\n";
|
|
mc << " SDL_RenderPresent(g_sdl_renderer);\n}\n\n";
|
|
mc << "static int sdl_pump_events(void) {\n";
|
|
mc << " SDL_Event e;\n";
|
|
mc << " while(SDL_PollEvent(&e)) {\n";
|
|
mc << " if(e.type==SDL_QUIT) return 0;\n";
|
|
mc << " if(e.type==SDL_KEYDOWN && e.key.keysym.sym==SDLK_ESCAPE) return 0;\n";
|
|
mc << " }\n return 1;\n}\n#endif /* HAVE_SDL2 */\n\n";
|
|
mc << "int main(int argc, char** argv){\n";
|
|
mc << " printf(\"=== %s ===\\n\", kGameTitle);\n\n";
|
|
mc << "#ifdef HAVE_SDL2\n";
|
|
mc << " if(SDL_Init(SDL_INIT_VIDEO) == 0) {\n";
|
|
mc << " g_sdl_window = SDL_CreateWindow(kGameTitle,\n";
|
|
mc << " SDL_WINDOWPOS_CENTERED, SDL_WINDOWPOS_CENTERED, 1280, 720, SDL_WINDOW_RESIZABLE);\n";
|
|
mc << " if(g_sdl_window)\n";
|
|
mc << " g_sdl_renderer = SDL_CreateRenderer(g_sdl_window, -1, SDL_RENDERER_ACCELERATED);\n";
|
|
mc << " } else {\n";
|
|
mc << " fprintf(stderr, \"[recomp] SDL2 init failed: %s (running headless)\\n\", SDL_GetError());\n";
|
|
mc << " }\n#endif\n\n";
|
|
mc << " uint8_t* mem = (uint8_t*)calloc(1, (size_t)GUEST_MEM_SIZE);\n";
|
|
mc << " if(!mem){ fprintf(stderr,\"Failed to allocate guest memory\\n\"); return 1; }\n";
|
|
mc << " GuestContext c; memset(&c,0,sizeof c);\n";
|
|
mc << " c.mem=mem; c.mem_size=GUEST_MEM_SIZE;\n";
|
|
mc << " c.mem_base_vaddr=0x" << std::hex << base << std::dec << "ULL;\n";
|
|
mc << " c.heap_base=c.mem_base_vaddr+GUEST_MEM_SIZE/2;\n";
|
|
mc << " c.heap_cur=c.heap_base; c.heap_end=c.mem_base_vaddr+GUEST_MEM_SIZE;\n";
|
|
mc << " c.x[31]=c.mem_base_vaddr + GUEST_MEM_SIZE - 16; /* SP */\n";
|
|
mc << " c.pc=0x" << std::hex << (entry_pc ? entry_pc : base) << std::dec << "ULL;\n\n";
|
|
mc << " recomp_save_init(&c, argv[0]);\n\n";
|
|
mc << " { char data_dir[512];\n";
|
|
mc << " snprintf(data_dir,sizeof data_dir,\"%s\",argv[0]);\n";
|
|
mc << " char* sl=strrchr(data_dir,'\\\\'); if(!sl) sl=strrchr(data_dir,'/'); if(sl) *(sl+1)=0; else data_dir[0]=0;\n";
|
|
mc << " strncat(data_dir,\"data\",sizeof(data_dir)-strlen(data_dir)-1);\n";
|
|
mc << " recomp_load_segments(&c,data_dir);\n }\n\n";
|
|
mc << " { uint64_t sz=0;\n";
|
|
mc << " if(recomp_save_exists(&c,\"autosave.bin\")){\n";
|
|
mc << " recomp_save_read(&c,\"autosave.bin\",c.mem,(uint64_t)GUEST_MEM_SIZE,&sz);\n";
|
|
mc << " printf(\"[recomp] Restored autosave (%llu bytes)\\n\",(unsigned long long)sz);\n";
|
|
mc << " }\n }\n\n";
|
|
mc << " printf(\"[recomp] Starting at pc=0x%llx\\n\",(unsigned long long)c.pc);\n";
|
|
mc << " /* Main loop: pump SDL events while the guest runs */\n";
|
|
mc << "#ifdef HAVE_SDL2\n";
|
|
mc << " while(!c.halted) {\n";
|
|
mc << " if(!sdl_pump_events()) break;\n";
|
|
mc << " recomp_run(&c); /* runs until SVC or halt */\n";
|
|
mc << " }\n";
|
|
mc << "#else\n";
|
|
mc << " recomp_run(&c);\n";
|
|
mc << "#endif\n\n";
|
|
mc << " recomp_save_write(&c,\"autosave.bin\",c.mem,(uint64_t)GUEST_MEM_SIZE);\n";
|
|
mc << " printf(\"[recomp] halted at pc=0x%llx\\n\",(unsigned long long)c.pc);\n";
|
|
mc << "#ifdef HAVE_SDL2\n";
|
|
mc << " if(g_sdl_texture) SDL_DestroyTexture(g_sdl_texture);\n";
|
|
mc << " if(g_sdl_renderer) SDL_DestroyRenderer(g_sdl_renderer);\n";
|
|
mc << " if(g_sdl_window) SDL_DestroyWindow(g_sdl_window);\n";
|
|
mc << " SDL_Quit();\n";
|
|
mc << "#endif\n";
|
|
mc << " free(mem);\n return 0;\n}\n";
|
|
|
|
std::ostringstream cm;
|
|
// The generated units live in src/ and include "recomp_runtime.h" from the
|
|
// module root, so the root has to be on the include path.
|
|
cm << "cmake_minimum_required(VERSION 3.13)\n"
|
|
<< "# When this directory is pulled into a bigger build (suyu-cmd linking the\n"
|
|
<< "# modules statically) it must not start a project of its own - it just\n"
|
|
<< "# contributes targets. Standalone it is still a complete project.\n"
|
|
<< "if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)\n"
|
|
<< " project(suyu_recompiled C)\n"
|
|
<< "else()\n"
|
|
<< " enable_language(C)\n"
|
|
<< "endif()\n"
|
|
<< "set(CMAKE_C_STANDARD 11)\n"
|
|
<< "include_directories(${CMAKE_CURRENT_SOURCE_DIR})\n"
|
|
<< "# A host project (suyu) may apply C++ flags to every language; this\n"
|
|
<< "# directory is plain C, and MSVC rejects /std:c11 together with\n"
|
|
<< "# /std:c++20 outright.\n"
|
|
<< "get_directory_property(_recomp_opts COMPILE_OPTIONS)\n"
|
|
<< "if(_recomp_opts)\n"
|
|
<< " list(FILTER _recomp_opts EXCLUDE REGEX \"std:c\\\\+\\\\+|std=c\\\\+\\\\+|EHsc|permissive\")\n"
|
|
<< " set_directory_properties(PROPERTIES COMPILE_OPTIONS \"${_recomp_opts}\")\n"
|
|
<< "endif()\n\n"
|
|
<< "# RECOMP_STATIC_ONLY: build just the static library this module\n"
|
|
<< "# contributes to a host executable, skipping the portable standalone exe\n"
|
|
<< "# and the loadable shared image.\n"
|
|
<< "if(NOT RECOMP_STATIC_ONLY)\n"
|
|
<< "# Optional: SDL2 window for display output.\n"
|
|
<< "# Install SDL2 (e.g. vcpkg install sdl2) to enable the game window.\n"
|
|
<< "find_package(SDL2 QUIET)\n"
|
|
<< "endif()\n\n"
|
|
<< "# RECOMP_BUILD_STATIC_LIB: the per-module static library exists only so\n"
|
|
<< "# suyu can link every module into its own single-file launcher. Nothing\n"
|
|
<< "# else consumes it, and it recompiles every generated unit a third time,\n"
|
|
<< "# so a standalone build of this project turns it off.\n"
|
|
<< "option(RECOMP_BUILD_STATIC_LIB \"build the static library for a host link\" ON)\n\n"
|
|
// Job count has to be bounded by memory, not cores. A single generated
|
|
// unit here reaches 100+ MB of C, and the compiler needs several GB of
|
|
// heap to get through one; handing a 32-core machine 32 of them at once
|
|
// is how a build dies with "C1060: compiler is out of heap space" after
|
|
// an hour of work. Budget ~8 GB per concurrent compile and take the
|
|
// lower of that and the core count.
|
|
<< "cmake_host_system_information(RESULT _recomp_ram_mb QUERY TOTAL_PHYSICAL_MEMORY)\n"
|
|
<< "cmake_host_system_information(RESULT _recomp_cores QUERY NUMBER_OF_LOGICAL_CORES)\n"
|
|
<< "math(EXPR _recomp_ram_jobs \"${_recomp_ram_mb} / 8192\")\n"
|
|
<< "if(_recomp_ram_jobs LESS 1)\n set(_recomp_ram_jobs 1)\nendif()\n"
|
|
<< "if(_recomp_ram_jobs LESS _recomp_cores)\n"
|
|
<< " set(_recomp_default_jobs ${_recomp_ram_jobs})\n"
|
|
<< "else()\n"
|
|
<< " set(_recomp_default_jobs ${_recomp_cores})\n"
|
|
<< "endif()\n"
|
|
<< "set(RECOMP_JOBS ${_recomp_default_jobs} CACHE STRING\n"
|
|
" \"concurrent compiles of the generated units (memory-bound, not core-bound)\")\n"
|
|
<< "message(STATUS \"recompiled: building generated units with ${RECOMP_JOBS} \"\n"
|
|
" \"concurrent compiles (${_recomp_ram_mb} MB RAM, ${_recomp_cores} cores)\")\n"
|
|
// /MP is an MSVC compiler flag and means nothing to Ninja, which
|
|
// schedules with its own -j and would happily start one compile per core.
|
|
// A job pool caps these targets alone, so the surrounding build (suyu's
|
|
// own tree, when it links these in) still runs at full width.
|
|
<< "if(CMAKE_GENERATOR MATCHES \"Ninja\")\n"
|
|
<< " get_property(_recomp_pools GLOBAL PROPERTY JOB_POOLS)\n"
|
|
<< " if(NOT \"${_recomp_pools}\" MATCHES \"recomp_compile=\")\n"
|
|
<< " set_property(GLOBAL APPEND PROPERTY JOB_POOLS recomp_compile=${RECOMP_JOBS})\n"
|
|
<< " endif()\n"
|
|
<< " set(RECOMP_JOB_POOL recomp_compile)\n"
|
|
<< "else()\n"
|
|
<< " set(RECOMP_JOB_POOL \"\")\n"
|
|
<< "endif()\n\n"
|
|
<< "# Generated translation units, all under src/.\nset(RECOMP_SOURCES\n"
|
|
<< " src/recompiled_" << mod << ".c";
|
|
for (size_t u = 0; u < unit_count; ++u) {
|
|
cm << "\n src/recompiled_" << mod << "_" << u << ".c";
|
|
}
|
|
cm << ")\n\n"
|
|
<< "# Generated block bodies are huge flat switch/if chains translated\n"
|
|
<< "# straight from machine code. They were compiled at -O1 on the belief\n"
|
|
<< "# that -O2 gains nothing here; measured on MK8D, -O2 (/O2 with MSVC)\n"
|
|
<< "# races ~13% faster on x64 Windows and ~11% on arm64 macOS for ~25%\n"
|
|
<< "# longer compiles. MSVC needs /bigobj at /O2: the units exceed the\n"
|
|
<< "# default COFF section limit (C1128).\n"
|
|
<< "if(MSVC)\n"
|
|
// /MP is what actually decides wall-clock time here. The Visual Studio
|
|
// generator compiles the files of a single project strictly in
|
|
// sequence, so a module with 100+ generated translation units serialises
|
|
// onto one core no matter what --parallel is passed to `cmake --build`.
|
|
// /MP fans them out across every core. Ninja parallelises on its own and
|
|
// warns about /MP, so gate it on the generator, not just on MSVC.
|
|
// The warning suppressions are the noisy ones the translation
|
|
// unavoidably produces (constant conditionals, provably-taken
|
|
// divide/shift paths); silencing them keeps cl.exe from spending real
|
|
// time formatting hundreds of thousands of diagnostics.
|
|
// /we4189 (from the top-level target's inherited warning-as-error
|
|
// set) turns "unused local" into a hard build failure; the codegen
|
|
// legitimately computes and drops _r on some flag-only paths, so
|
|
// downgrade it back to a warning for generated sources specifically.
|
|
<< " set(_recomp_msvc_opts \"/O2\" \"/bigobj\" \"/WX-\" \"/wd4127\" \"/wd4723\" \"/wd4102\" "
|
|
"\"/wd4101\" \"/wd4189\" \"/wd4456\" \"/wd4457\" \"/wd4459\")\n"
|
|
// FPX1's fast paths need IEEE arithmetic; fast-math compiles them out
|
|
// (RECOMP_FPX_HOST 0), so pin the model rather than inherit one.
|
|
<< (g_emit_fpx ? " list(APPEND _recomp_msvc_opts \"/fp:precise\")\n" : "")
|
|
<< " if(NOT CMAKE_GENERATOR MATCHES \"Ninja\")\n"
|
|
// Bare /MP means "one compile per core", which is exactly the
|
|
// oversubscription that exhausts memory on these units.
|
|
<< " list(APPEND _recomp_msvc_opts \"/MP${RECOMP_JOBS}\")\n"
|
|
<< " endif()\n"
|
|
<< " set_source_files_properties(${RECOMP_SOURCES} PROPERTIES COMPILE_OPTIONS "
|
|
"\"${_recomp_msvc_opts}\")\n"
|
|
<< "else()\n"
|
|
// Overridable. -O2 by default: it measured ~11% faster than -O1 on arm64
|
|
// macOS, and GCC does not vectorise at all below -O2. The cache entry is
|
|
// versioned: a tree configured before the -O2 default keeps its cached -O1
|
|
// under the old name, and would otherwise never pick up the new default.
|
|
<< " set(RECOMP_OPT_FLAGS_V2 \"-O2 -foptimize-sibling-calls\" CACHE STRING\n"
|
|
" \"optimisation flags for the generated block bodies\")\n"
|
|
<< " separate_arguments(_recomp_opt NATIVE_COMMAND \"${RECOMP_OPT_FLAGS_V2}\")\n"
|
|
// Same reasoning as /WX- above: a host tree built with -Werror must not
|
|
// fail on a shadowed local inside generated code.
|
|
<< " list(APPEND _recomp_opt \"-Wno-error\")\n"
|
|
<< (g_emit_fpx ? " list(APPEND _recomp_opt \"-ffp-contract=off\" \"-fno-fast-math\")\n" : "")
|
|
<< " set_source_files_properties(${RECOMP_SOURCES} PROPERTIES COMPILE_OPTIONS "
|
|
"\"${_recomp_opt}\")\n"
|
|
<< "endif()\n\n"
|
|
<< (g_emit_standalone_runner ? "if(NOT RECOMP_STATIC_ONLY)\n" : "if(FALSE)\n")
|
|
<< "add_executable(recompiled main.c recomp_runtime.c ${RECOMP_SOURCES})\n"
|
|
<< "if(RECOMP_JOB_POOL)\n"
|
|
<< " set_target_properties(recompiled PROPERTIES JOB_POOL_COMPILE ${RECOMP_JOB_POOL})\n"
|
|
<< "endif()\n"
|
|
<< "if(EXISTS \"${CMAKE_CURRENT_SOURCE_DIR}/data\")\n"
|
|
<< " add_custom_command(TARGET recompiled POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_directory"
|
|
" \"${CMAKE_CURRENT_SOURCE_DIR}/data\" \"$<TARGET_FILE_DIR:recompiled>/data\")\n"
|
|
<< "endif()\n"
|
|
<< "if(SDL2_FOUND)\n"
|
|
<< " target_compile_definitions(recompiled PRIVATE HAVE_SDL2)\n"
|
|
<< " target_include_directories(recompiled PRIVATE ${SDL2_INCLUDE_DIRS})\n"
|
|
<< " target_link_libraries(recompiled ${SDL2_LIBRARIES})\n"
|
|
<< " message(STATUS \"SDL2 found — recompiled will open a game window\")\n"
|
|
<< "else()\n"
|
|
<< " message(STATUS \"SDL2 not found — running headless (no window)\")\n"
|
|
<< "endif()\n"
|
|
<< "# Portable C11: Windows->.exe, Linux/FreeBSD/OpenBSD->ELF, macOS->Mach-O\n"
|
|
<< "endif()\n\n";
|
|
|
|
// A second target builds the same code as a shared library exporting the
|
|
// block lookup. That is what suyu loads to run this image on
|
|
// Core::ArmRecomp: the emulator supplies memory and services through the
|
|
// host bridge, so main.c - which owns a flat buffer and a stub SVC handler
|
|
// - is deliberately left out of this target.
|
|
//
|
|
// Target/output name is per-module so every NSO's DLL builds side by side
|
|
// without colliding: suyu-cmd's loader (src/suyu_cmd/suyu.cpp) looks for
|
|
// recompiled_rtld.dll / recompiled_image.dll (main) / recompiled_sdk.dll /
|
|
// recompiled_subsdkN.dll next to the exe, in NSO load order.
|
|
const std::string dll_target = (mod == "main") ? "recompiled_image" : ("recompiled_" + mod);
|
|
cm << "if(NOT RECOMP_STATIC_ONLY)\n"
|
|
<< "add_library(" << dll_target << " SHARED recomp_export.c recomp_runtime.c "
|
|
"${RECOMP_SOURCES})\n"
|
|
<< "set_target_properties(" << dll_target << " PROPERTIES C_VISIBILITY_PRESET hidden "
|
|
"OUTPUT_NAME \"" << dll_target << "\")\n"
|
|
<< "if(RECOMP_JOB_POOL)\n"
|
|
<< " set_target_properties(" << dll_target
|
|
<< " PROPERTIES JOB_POOL_COMPILE ${RECOMP_JOB_POOL})\n"
|
|
<< "endif()\n"
|
|
<< "target_compile_definitions(" << dll_target
|
|
<< " PRIVATE SUYU_HOSTED_RECOMP=1 RECOMP_SHARED_MODULE=1)\n"
|
|
<< "endif()\n\n";
|
|
|
|
// Static-library variant. Several of these get linked into ONE host
|
|
// executable (the per-game suyu-cmd build), so every symbol a module owns
|
|
// has to be unique. The generated C is written once and renamed at compile
|
|
// time through -D, which keeps the sources identical between the shared and
|
|
// the static shape.
|
|
//
|
|
// recomp_runtime.c is deliberately NOT part of this target: its contents
|
|
// (recomp_svc, the load/store helpers, recomp_cond, ...) are generic and
|
|
// must exist exactly once in the final link. It is built separately, once,
|
|
// as recomp_runtime_shared.
|
|
const std::string static_target = "recomp_static_" + mod;
|
|
cm << "if(RECOMP_BUILD_STATIC_LIB)\n"
|
|
<< "if(NOT TARGET recomp_runtime_shared)\n"
|
|
<< " add_library(recomp_runtime_shared STATIC recomp_runtime.c)\n"
|
|
<< " target_compile_definitions(recomp_runtime_shared PRIVATE SUYU_HOSTED_RECOMP=1 "
|
|
"RECOMP_STATIC_HOST=1)\n"
|
|
<< " target_include_directories(recomp_runtime_shared PUBLIC "
|
|
"${CMAKE_CURRENT_SOURCE_DIR})\n"
|
|
<< "endif()\n"
|
|
<< "add_library(" << static_target << " STATIC recomp_export.c ${RECOMP_SOURCES})\n"
|
|
<< "if(RECOMP_JOB_POOL)\n"
|
|
<< " set_target_properties(" << static_target
|
|
<< " PROPERTIES JOB_POOL_COMPILE ${RECOMP_JOB_POOL})\n"
|
|
<< "endif()\n"
|
|
<< "target_include_directories(" << static_target
|
|
<< " PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})\n"
|
|
<< "target_link_libraries(" << static_target << " PUBLIC recomp_runtime_shared)\n"
|
|
<< "target_compile_definitions(" << static_target
|
|
<< " PRIVATE SUYU_HOSTED_RECOMP=1 RECOMP_STATIC_MODULE=1"
|
|
<< " g_module_base=g_module_base_" << mod
|
|
<< " g_recomp_guard_host_v2=g_recomp_guard_host_v2_" << mod
|
|
<< " recomp_image_abi=recomp_image_abi_" << mod
|
|
<< " recomp_image_guard_v2=recomp_image_guard_v2_" << mod;
|
|
if (g_emit_fastmem) {
|
|
cm << " recomp_image_features=recomp_image_features_" << mod
|
|
<< " recomp_image_fastmem_v1=recomp_image_fastmem_v1_" << mod;
|
|
} else if (g_emit_fpx) {
|
|
cm << " recomp_image_features=recomp_image_features_" << mod;
|
|
}
|
|
if (g_emit_fpx) {
|
|
cm << " recomp_image_fpx_v1=recomp_image_fpx_v1_" << mod;
|
|
}
|
|
if (EmitGuardGen()) {
|
|
cm << " g_recomp_gg_word=g_recomp_gg_word_" << mod
|
|
<< " recomp_image_guard_gen_v1=recomp_image_guard_gen_v1_" << mod;
|
|
}
|
|
cm << " recomp_lookup=recomp_lookup_" << mod
|
|
<< " recomp_build_index=recomp_build_index_" << mod
|
|
<< " _recomp_index_view=_recomp_index_view_" << mod
|
|
<< " recomp_image_index=recomp_image_index_" << mod
|
|
<< " recomp_image_lookup=recomp_image_lookup_" << mod
|
|
<< " recomp_image_run_slice=recomp_image_run_slice_" << mod
|
|
<< " recomp_image_set_base=recomp_image_set_base_" << mod
|
|
<< " recomp_image_entry=recomp_image_entry_" << mod << ")\n"
|
|
<< "endif()\n";
|
|
|
|
std::ostringstream ex;
|
|
ex << "/* auto-generated by suyu static recompiler - DO NOT EDIT */\n"
|
|
"#include \"recomp_runtime.h\"\n\n"
|
|
"/* The one symbol suyu resolves out of a recompiled image. Named\n"
|
|
" distinctly from the internal recomp_lookup so the export is\n"
|
|
" unambiguous, and so the internal one can stay hidden. */\n"
|
|
"#ifdef RECOMP_STATIC_MODULE\n"
|
|
"/* Linked straight into the host executable: static-library symbols are\n"
|
|
" visible to the linker on their own, and the names are already made\n"
|
|
" unique per module by the -D renames the build applies. */\n"
|
|
"#define RECOMP_API\n"
|
|
"/* The shared runtime is compiled once for the whole link and therefore\n"
|
|
" cannot own this - each module needs its own load base. */\n"
|
|
"uint64_t g_module_base = 0;\n"
|
|
"#elif defined(_WIN32)\n"
|
|
"#define RECOMP_API __declspec(dllexport)\n"
|
|
"#else\n"
|
|
"#define RECOMP_API __attribute__((visibility(\"default\")))\n"
|
|
"#endif\n\n"
|
|
"/* Version of the generated-image contract used by automatic bundle\n"
|
|
" selection. ABI 4 adds bounded, nonrecursive module-local slices to\n"
|
|
" guarded instruction side entries. ABI 5 restores per-entry checks\n"
|
|
" and exact guest floating-point status. */\n"
|
|
"RECOMP_API unsigned recomp_image_abi(void){ return RECOMP_IMAGE_ABI; }\n"
|
|
"RECOMP_API unsigned recomp_image_guard_v2(unsigned host_version){\n"
|
|
" g_recomp_guard_host_v2=(host_version==2)?2:0; return 2;\n}\n";
|
|
if (EmitGuardGen()) {
|
|
u64 code_lo = 0, code_end = 0;
|
|
if (!blocks.empty()) {
|
|
code_lo = blocks.front().vaddr;
|
|
for (const auto& b : blocks) {
|
|
code_end = std::max<u64>(code_end, b.vaddr + (u64)b.count * 4);
|
|
}
|
|
}
|
|
char span[160];
|
|
snprintf(span, sizeof span,
|
|
" *code_lo=0x%llxULL; *code_end=0x%llxULL; *base=&g_module_base;\n",
|
|
(unsigned long long)code_lo, (unsigned long long)code_end);
|
|
ex << "/* ABI 6 feature GG1. The generation word starts at VERIFY_ALWAYS, so\n"
|
|
" until a host that knows GG1 takes it over every block verifies on\n"
|
|
" every entry. The host passes its GG1 version and learns the word,\n"
|
|
" the module-relative span the blocks cover, and where the load base\n"
|
|
" lives. The FM1 handshake below refuses to complete before this one,\n"
|
|
" so a host that knows FM1 but not GG1 refuses the whole bundle. */\n"
|
|
"uint32_t g_recomp_gg_word = RECOMP_GG_VERIFY_ALWAYS;\n"
|
|
"static int g_recomp_gg_host = 0;\n"
|
|
"RECOMP_API uint32_t* recomp_image_guard_gen_v1(uint32_t host_version, uint64_t* code_lo,\n"
|
|
" uint64_t* code_end, const uint64_t** base){\n"
|
|
" if(host_version!=2u) return 0;\n"
|
|
" RECOMP_GG_STORE_RELEASE(g_recomp_gg_word,RECOMP_GG_VERIFY_ALWAYS);\n"
|
|
<< span
|
|
<< " g_recomp_gg_host=1;\n"
|
|
" return &g_recomp_gg_word;\n"
|
|
"}\n";
|
|
}
|
|
if (g_emit_fastmem) {
|
|
ex << "/* ABI 6 feature handshake. The host passes its own view of the page\n"
|
|
" table layout and of the context offsets; 1 means this image was\n"
|
|
" compiled against exactly those, so the host may enable fm_limit. */\n"
|
|
"RECOMP_API unsigned recomp_image_features(void){ return RECOMP_FEATURE_FASTMEM_PT1"
|
|
<< (EmitGuardGen() ? " | RECOMP_FEATURE_GUARD_GEN1" : "")
|
|
<< (g_emit_fpx ? " | RECOMP_FEATURE_FPX1" : "") << "; }\n"
|
|
"RECOMP_API unsigned recomp_image_fastmem_v1(uint32_t page_bits, uint32_t stride_log2,\n"
|
|
" uint64_t ptr_mask, uint32_t off_table,\n"
|
|
" uint32_t off_limit){\n"
|
|
<< (EmitGuardGen() ? " if(!g_recomp_gg_host) return 0;\n" : "")
|
|
<< " return page_bits==RECOMP_FM_PAGE_BITS && stride_log2==RECOMP_FM_STRIDE_LOG2 &&\n"
|
|
" ptr_mask==(uint64_t)RECOMP_FM_PTR_MASK &&\n"
|
|
" off_table==offsetof(GuestContext,fm_table) &&\n"
|
|
" off_limit==offsetof(GuestContext,fm_limit);\n"
|
|
"}\n";
|
|
} else if (g_emit_fpx) {
|
|
ex << "/* ABI 6 feature report. */\n"
|
|
"RECOMP_API unsigned recomp_image_features(void){ return RECOMP_FEATURE_FPX1; }\n";
|
|
}
|
|
if (g_emit_fpx) {
|
|
ex << "/* FPX1 handshake. The host passes its own view of the FP control and\n"
|
|
" status fields and of its kill-switch bit; nonzero means this image was\n"
|
|
" compiled against exactly those. The low byte is RECOMP_FPX_HOST, the\n"
|
|
" fast path this image was compiled to (0: compiled out). */\n"
|
|
"RECOMP_API unsigned recomp_image_fpx_v1(uint32_t off_fpcr, uint32_t off_fpsr,\n"
|
|
" uint64_t inhibit_bit){\n"
|
|
" if(off_fpcr!=offsetof(GuestContext,fpcr) || off_fpsr!=offsetof(GuestContext,fpsr) ||\n"
|
|
" inhibit_bit!=RECOMP_FPX_INHIBIT) return 0;\n"
|
|
" return 0x100u | (unsigned)RECOMP_FPX_HOST"
|
|
<< (g_emit_fpx_shadow ? " | 0x200u; /* the shadow instrumentation build */\n" : ";\n")
|
|
<< "}\n";
|
|
}
|
|
ex << "RECOMP_API BlockFn recomp_image_lookup(uint64_t pc){ return recomp_lookup(pc - g_module_base); }\n\n"
|
|
"/* Run a bounded sequence while control flow stays inside this module.\n"
|
|
" Compact conditional branches return to this C loop instead of the\n"
|
|
" cross-library host dispatcher. Every additional block consumes the\n"
|
|
" same execution budget, bounding the interval between interrupt checks\n"
|
|
" and the host's executed-block accounting. */\n"
|
|
"RECOMP_API void recomp_image_run_slice(GuestContext* c){\n"
|
|
" int first=1;\n"
|
|
" while(!c->halted && c->pending_svc==~UINT64_C(0) && c->chain_budget>0){\n"
|
|
" BlockFn f=recomp_lookup(c->pc-g_module_base);\n"
|
|
" if(!f)return;\n"
|
|
" if(!first && --c->chain_budget<=0)return;\n"
|
|
" first=0; f(c);\n"
|
|
" }\n"
|
|
"}\n\n"
|
|
/* Hands the block index out so a host dispatcher can do the lookup
|
|
itself. Going through recomp_image_lookup costs three nested calls
|
|
across the shared-object boundary on every block edge; with this it
|
|
is a bounds check and one load. Addresses are absolute so the caller
|
|
needs to know nothing about the module base. Only valid once
|
|
recomp_image_set_base has run, which is when the index is built. */
|
|
"extern int _recomp_index_view(uint64_t*, uint64_t*, BlockFn**);\n"
|
|
"RECOMP_API int recomp_image_index(uint64_t* lo, uint64_t* hi, BlockFn** idx){\n"
|
|
" if(!_recomp_index_view(lo, hi, idx)) return 0;\n"
|
|
" *lo += g_module_base; *hi += g_module_base;\n"
|
|
" return 1;\n"
|
|
"}\n"
|
|
"\n"
|
|
"/* Tells this image where its module actually got loaded, so the\n"
|
|
" addresses it computes are real rather than module-relative. */\n"
|
|
"RECOMP_API void recomp_image_set_base(uint64_t base){ g_module_base = base;\n"
|
|
" /* Single-threaded here, which is what makes the index safe to\n"
|
|
" build without locking. */\n"
|
|
" recomp_build_index(); }\n\n"
|
|
"/* Reports the entry PC so the loader does not have to be told it\n"
|
|
" separately or parse the image again. */\n"
|
|
<< "RECOMP_API uint64_t recomp_image_entry(void){ return 0x"
|
|
<< std::hex << (entry_pc ? entry_pc : base) << std::dec << "ULL; }\n";
|
|
|
|
auto write = [&](const std::string& name, const std::string& data) {
|
|
std::ofstream o(Utf8Path(out_dir + "/" + name), std::ios::binary);
|
|
o.write(data.data(), (std::streamsize)data.size());
|
|
};
|
|
write("recomp_runtime.h", RuntimeH());
|
|
write("recomp_runtime.c", RuntimeC());
|
|
// The dispatch table is generated code like the block bodies, so it belongs
|
|
// with them rather than at the top level.
|
|
write("src/recompiled_" + mod + ".c", rc);
|
|
// finish_unit closed each block translation unit before reaching here.
|
|
if (g_emit_standalone_runner) {
|
|
write("main.c", mc.str());
|
|
}
|
|
std::string export_text = ex.str();
|
|
if (EmitGuardGen()) {
|
|
// A module moved to another base must be re-verified there: drop back
|
|
// to VERIFY_ALWAYS until the host moves the generation again.
|
|
static constexpr std::string_view set_base =
|
|
"RECOMP_API void recomp_image_set_base(uint64_t base){ g_module_base = base;\n";
|
|
export_text.replace(export_text.find(set_base), set_base.size(),
|
|
"RECOMP_API void recomp_image_set_base(uint64_t base){\n"
|
|
" if(base!=g_module_base) RECOMP_GG_STORE_RELEASE(g_recomp_gg_word,"
|
|
"RECOMP_GG_VERIFY_ALWAYS);\n"
|
|
" g_module_base = base;\n");
|
|
}
|
|
write("recomp_export.c", export_text);
|
|
write("CMakeLists.txt", cm.str());
|
|
|
|
// Coverage report. Without this the exporter cannot describe its own output:
|
|
// block and instruction counts say how much code was walked, not how much of
|
|
// it was actually translated, and an export made entirely of fallbacks looks
|
|
// identical to a complete one. The signature histogram is the actionable
|
|
// part - it ranks which opcode family to implement next by how often it is
|
|
// actually hit, rather than by which gaps look important.
|
|
{
|
|
std::ostringstream cov;
|
|
cov << "{\n";
|
|
cov << " \"module\": \"" << mod << "\",\n";
|
|
cov << " \"blocks\": " << stats.blocks << ",\n";
|
|
cov << " \"text_words\": " << stats.instructions << ",\n";
|
|
cov << " \"instructions_emitted\": " << stats.emitted << ",\n";
|
|
cov << " \"instructions_unhandled\": " << stats.unhandled << ",\n";
|
|
cov << " \"unhandled_fraction\": " << std::fixed << std::setprecision(6)
|
|
<< stats.UnhandledFraction() << ",\n";
|
|
cov.unsetf(std::ios::floatfield);
|
|
|
|
cov << " \"unhandled_by_group\": {";
|
|
bool first_group = true;
|
|
for (const auto& [group, count] : stats.unhandled_by_group) {
|
|
if (!first_group) cov << ",";
|
|
first_group = false;
|
|
cov << "\n \"" << EncodingGroupName(group) << "\": " << count;
|
|
}
|
|
cov << (first_group ? "}" : "\n }") << ",\n";
|
|
|
|
// Ranked, because the only question this file exists to answer is
|
|
// "what should I implement next".
|
|
std::vector<std::pair<u32, UnhandledSite>> ranked(stats.unhandled_by_signature.begin(),
|
|
stats.unhandled_by_signature.end());
|
|
std::sort(ranked.begin(), ranked.end(),
|
|
[](const auto& a, const auto& b) { return a.second.count > b.second.count; });
|
|
|
|
cov << " \"unhandled_by_signature\": [";
|
|
bool first_sig = true;
|
|
size_t shown = 0;
|
|
for (const auto& [sig, site] : ranked) {
|
|
if (shown++ >= 64) break;
|
|
if (!first_sig) cov << ",";
|
|
first_sig = false;
|
|
char line[256];
|
|
snprintf(line, sizeof line,
|
|
"\n { \"signature\": \"0x%08X\", \"count\": %zu, "
|
|
"\"share\": %.4f, \"example_insn\": \"0x%08X\", "
|
|
"\"example_pc\": \"0x%llX\", \"group\": \"%s\" }",
|
|
sig, site.count,
|
|
stats.unhandled ? double(site.count) / double(stats.unhandled) : 0.0,
|
|
site.example_insn, (unsigned long long)site.example_pc,
|
|
EncodingGroupName((site.example_insn >> 25) & 0xF));
|
|
cov << line;
|
|
}
|
|
cov << (first_sig ? "]" : "\n ]") << ",\n";
|
|
cov << " \"distinct_signatures\": " << stats.unhandled_by_signature.size() << "\n";
|
|
cov << "}\n";
|
|
write("recomp_coverage.json", cov.str());
|
|
}
|
|
|
|
// Bundle text/rodata/data as binary blobs so the exe can load them at startup.
|
|
// Only for the standalone runner: nothing else reads them.
|
|
if (g_emit_standalone_runner) {
|
|
std::string data_subdir = out_dir + "/data";
|
|
make_dir(data_subdir);
|
|
// Always write text.bin
|
|
{
|
|
std::ofstream o(Utf8Path(data_subdir + "/text.bin"), std::ios::binary);
|
|
o.write(reinterpret_cast<const char*>(text), (std::streamsize)n_bytes);
|
|
}
|
|
if (rodata && rodata_size > 0) {
|
|
std::ofstream o(Utf8Path(data_subdir + "/rodata.bin"), std::ios::binary);
|
|
o.write(reinterpret_cast<const char*>(rodata), (std::streamsize)rodata_size);
|
|
}
|
|
if (data_seg && data_size > 0) {
|
|
std::ofstream o(Utf8Path(data_subdir + "/data.bin"), std::ios::binary);
|
|
o.write(reinterpret_cast<const char*>(data_seg), (std::streamsize)data_size);
|
|
}
|
|
}
|
|
|
|
return stats;
|
|
}
|
|
|
|
/* Exact-FP accumulator scans, shared by the runtime header below and by the
|
|
standalone test harnesses, which build a self-contained C file rather than
|
|
including recomp_runtime.h. One definition so the two cannot drift. */
|
|
inline const char* FPScanHelpers() {
|
|
return R"FS(
|
|
/* Standalone test harnesses paste this in without recomp_runtime.h. */
|
|
#ifndef RECOMP_INLINE
|
|
#if defined(_MSC_VER)
|
|
#define RECOMP_INLINE __forceinline
|
|
#elif defined(__GNUC__) || defined(__clang__)
|
|
#define RECOMP_INLINE inline __attribute__((always_inline))
|
|
#else
|
|
#define RECOMP_INLINE inline
|
|
#endif
|
|
#endif
|
|
|
|
/* The multiply-add and step emitters keep a 9- (single) or 67-word (double)
|
|
integer accumulator so rounding, subnormals and FPSR are exact. Finding the
|
|
highest set bit and OR-ing the discarded bits one bit at a time made those
|
|
two loops the dominant cost of the generated code: sampled at 99 Hz they
|
|
were ~87-92% of the exclusive cycles of the three hottest attract functions.
|
|
These do the identical operation a word at a time. Results are unchanged. */
|
|
#if defined(_MSC_VER)
|
|
#include <intrin.h>
|
|
#endif
|
|
|
|
/* Index of the highest set bit of `w`. `w` must be nonzero. */
|
|
RECOMP_INLINE unsigned recomp_bit_index64(uint64_t w) {
|
|
#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_ARM64))
|
|
unsigned long i;
|
|
_BitScanReverse64(&i, w);
|
|
return (unsigned)i;
|
|
#elif defined(_MSC_VER)
|
|
/* 32-bit MSVC targets have no _BitScanReverse64. */
|
|
unsigned long i;
|
|
uint32_t hi = (uint32_t)(w >> 32);
|
|
if (hi) {
|
|
_BitScanReverse(&i, (unsigned long)hi);
|
|
return (unsigned)i + 32u;
|
|
}
|
|
_BitScanReverse(&i, (unsigned long)(uint32_t)w);
|
|
return (unsigned)i;
|
|
#elif defined(__GNUC__) || defined(__clang__)
|
|
return 63u - (unsigned)__builtin_clzll(w);
|
|
#else
|
|
/* Portable fallback: no intrinsic and no new CPU feature requirement. */
|
|
unsigned n = 0;
|
|
if (w >> 32) { w >>= 32; n += 32; }
|
|
if (w >> 16) { w >>= 16; n += 16; }
|
|
if (w >> 8) { w >>= 8; n += 8; }
|
|
if (w >> 4) { w >>= 4; n += 4; }
|
|
if (w >> 2) { w >>= 2; n += 2; }
|
|
if (w >> 1) { n += 1; }
|
|
return n;
|
|
#endif
|
|
}
|
|
|
|
/* Highest set bit of the little-endian word array `p`, or -1 if it is zero.
|
|
Equivalent to scanning bit (count*64-1) downwards for the first set bit. */
|
|
RECOMP_INLINE int recomp_fp_top_bit(const uint64_t* p, unsigned count) {
|
|
unsigned k = count;
|
|
while (k--) {
|
|
if (p[k]) {
|
|
return (int)(k * 64u) + (int)recomp_bit_index64(p[k]);
|
|
}
|
|
}
|
|
return -1;
|
|
}
|
|
|
|
/* Sticky bit for a rounding position `cut`: the OR of bits [0, cut-1) of `p`,
|
|
i.e. everything strictly below the round bit, which is bit cut-1 and is
|
|
deliberately excluded. Equivalent to the per-bit OR loop it replaces. */
|
|
RECOMP_INLINE unsigned recomp_fp_sticky(const uint64_t* p, unsigned count, int cut) {
|
|
int n = cut - 1;
|
|
uint64_t agg = 0;
|
|
unsigned whole, rest, i;
|
|
if (n <= 0) {
|
|
return 0u;
|
|
}
|
|
whole = (unsigned)n / 64u;
|
|
if (whole > count) {
|
|
whole = count;
|
|
}
|
|
for (i = 0; i < whole; ++i) {
|
|
agg |= p[i];
|
|
}
|
|
rest = (unsigned)n % 64u;
|
|
/* A zero remainder must not reach a shift by 64. */
|
|
if (rest && whole < count) {
|
|
agg |= p[whole] & ((1ULL << rest) - 1ULL);
|
|
}
|
|
return agg != 0;
|
|
}
|
|
)FS";
|
|
}
|
|
|
|
// Scalar fixed-point helpers, returned as C for the generated runtime.
|
|
// This implements the S/D, W/X non-trapping FPCR/FPSR contract used by the
|
|
// existing runtime; FP16 and alternative-FP extensions are not added here.
|
|
inline const char* FPFixedHelpers() {
|
|
return R"FX(
|
|
/* Convert a binary32/64 significand directly to signed/unsigned fixed-point.
|
|
Round toward zero first, then saturate. IOC takes precedence over IXC. */
|
|
static RECOMP_INLINE uint64_t recomp_fp_to_fixed(uint64_t bits, unsigned fp_bits,
|
|
unsigned int_bits, unsigned fbits, unsigned is_signed,
|
|
uint64_t fpcr, uint64_t* fpsr) {
|
|
const unsigned fraction_bits = fp_bits == 64 ? 52u : 23u;
|
|
const unsigned bias = fp_bits == 64 ? 1023u : 127u;
|
|
const uint64_t fraction_mask = (UINT64_C(1) << fraction_bits) - 1;
|
|
const uint64_t exponent_mask = fp_bits == 64 ? 2047u : 255u;
|
|
const unsigned negative = (unsigned)((bits >> (fp_bits - 1)) & 1);
|
|
const unsigned exponent = (unsigned)((bits >> fraction_bits) & exponent_mask);
|
|
uint64_t mantissa = bits & fraction_mask;
|
|
const uint64_t integer_mask = int_bits == 64 ? UINT64_MAX : UINT64_C(0xffffffff);
|
|
const uint64_t sign_bit = UINT64_C(1) << (int_bits - 1);
|
|
const uint64_t limit = is_signed ? sign_bit - (negative ? 0u : 1u) : integer_mask;
|
|
const uint64_t saturated = negative ? (is_signed ? sign_bit : 0) : limit;
|
|
uint64_t magnitude = 0;
|
|
unsigned inexact = 0, invalid = 0;
|
|
int shift;
|
|
if (exponent == exponent_mask) {
|
|
*fpsr |= UINT64_C(1); /* NaNs and infinities: invalid operation. */
|
|
return mantissa ? 0 : saturated;
|
|
}
|
|
if (exponent == 0) {
|
|
if (!mantissa) return 0; /* Both signed zeros. */
|
|
if (fpcr & (UINT64_C(1) << 24)) {
|
|
*fpsr |= UINT64_C(128); /* FZ: input-denormal, not inexact. */
|
|
return 0;
|
|
}
|
|
} else {
|
|
mantissa |= UINT64_C(1) << fraction_bits;
|
|
}
|
|
shift = (int)(exponent ? exponent : 1u) - (int)bias -
|
|
(int)fraction_bits + (int)fbits;
|
|
if (shift >= 0) {
|
|
if (shift >= 64 || mantissa > (limit >> (unsigned)shift)) invalid = 1;
|
|
else magnitude = mantissa << (unsigned)shift;
|
|
} else {
|
|
const unsigned discarded = (unsigned)(-shift);
|
|
if (discarded >= 64) {
|
|
inexact = mantissa != 0;
|
|
} else {
|
|
magnitude = mantissa >> discarded;
|
|
inexact = (mantissa & ((UINT64_C(1) << discarded) - 1)) != 0;
|
|
}
|
|
if (magnitude > limit) invalid = 1;
|
|
}
|
|
if (negative && !is_signed && magnitude != 0) invalid = 1;
|
|
if (invalid) {
|
|
*fpsr |= UINT64_C(1);
|
|
return saturated;
|
|
}
|
|
if (inexact) *fpsr |= UINT64_C(16);
|
|
return (negative ? UINT64_C(0) - magnitude : magnitude) & integer_mask;
|
|
}
|
|
|
|
/* Convert a binary32/64 value to a signed/unsigned integer in the given
|
|
rounding mode (0 nearest-even, 1 toward +inf, 2 toward -inf, 3 toward zero,
|
|
4 nearest-away), as FCVT{N,P,M,Z,A}{S,U}: round, then saturate. IOC takes
|
|
precedence over IXC; FZ flushes a subnormal input to zero with IDC. */
|
|
static RECOMP_INLINE uint64_t recomp_fp_to_int(uint64_t bits, unsigned fp_bits,
|
|
unsigned int_bits, unsigned is_signed, unsigned rmode,
|
|
uint64_t fpcr, uint64_t* fpsr) {
|
|
const unsigned fraction_bits = fp_bits == 64 ? 52u : 23u;
|
|
const unsigned bias = fp_bits == 64 ? 1023u : 127u;
|
|
const uint64_t fraction_mask = (UINT64_C(1) << fraction_bits) - 1;
|
|
const uint64_t exponent_mask = fp_bits == 64 ? 2047u : 255u;
|
|
const unsigned negative = (unsigned)((bits >> (fp_bits - 1)) & 1);
|
|
const unsigned exponent = (unsigned)((bits >> fraction_bits) & exponent_mask);
|
|
uint64_t mantissa = bits & fraction_mask;
|
|
const uint64_t integer_mask = int_bits == 64 ? UINT64_MAX : UINT64_C(0xffffffff);
|
|
const uint64_t sign_bit = UINT64_C(1) << (int_bits - 1);
|
|
const uint64_t limit = is_signed ? sign_bit - (negative ? 0u : 1u) : integer_mask;
|
|
const uint64_t saturated = negative ? (is_signed ? sign_bit : 0) : limit;
|
|
uint64_t magnitude = 0;
|
|
unsigned inexact = 0, invalid = 0;
|
|
int shift;
|
|
if (exponent == exponent_mask) {
|
|
*fpsr |= UINT64_C(1); /* NaNs and infinities: invalid operation. */
|
|
return mantissa ? 0 : saturated;
|
|
}
|
|
if (exponent == 0) {
|
|
if (!mantissa) return 0; /* Both signed zeros. */
|
|
if (fpcr & (UINT64_C(1) << 24)) {
|
|
*fpsr |= UINT64_C(128); /* FZ: input-denormal, not inexact. */
|
|
return 0;
|
|
}
|
|
} else {
|
|
mantissa |= UINT64_C(1) << fraction_bits;
|
|
}
|
|
shift = (int)(exponent ? exponent : 1u) - (int)bias - (int)fraction_bits;
|
|
if (shift >= 0) {
|
|
if (shift >= 64 || mantissa > (limit >> (unsigned)shift)) invalid = 1;
|
|
else magnitude = mantissa << (unsigned)shift;
|
|
} else {
|
|
const unsigned discarded = (unsigned)(-shift);
|
|
uint64_t remainder = mantissa, half = 0; /* half 0: below one half */
|
|
if (discarded < 64) {
|
|
magnitude = mantissa >> discarded;
|
|
remainder = mantissa & ((UINT64_C(1) << discarded) - 1);
|
|
half = UINT64_C(1) << (discarded - 1);
|
|
}
|
|
inexact = remainder != 0;
|
|
if (inexact &&
|
|
((rmode == 0 && half && (remainder > half || (remainder == half && (magnitude & 1)))) ||
|
|
(rmode == 4 && half && remainder >= half) || (rmode == 1 && !negative) ||
|
|
(rmode == 2 && negative))) {
|
|
++magnitude;
|
|
}
|
|
if (magnitude > limit) invalid = 1;
|
|
}
|
|
if (negative && !is_signed && magnitude != 0) invalid = 1;
|
|
if (invalid) {
|
|
*fpsr |= UINT64_C(1);
|
|
return saturated;
|
|
}
|
|
if (inexact) *fpsr |= UINT64_C(16);
|
|
return (negative ? UINT64_C(0) - magnitude : magnitude) & integer_mask;
|
|
}
|
|
|
|
/* Convert the integer magnitude once, using the guest FPCR rounding mode.
|
|
All legal S/D fixed-point inputs produce zero or a finite normal result:
|
|
exponent range is at least -64 and at most 63. No host FP cast is used. */
|
|
static RECOMP_INLINE uint64_t recomp_fixed_to_fp(uint64_t bits, unsigned fp_bits,
|
|
unsigned int_bits, unsigned fbits, unsigned is_signed,
|
|
uint64_t fpcr, uint64_t* fpsr) {
|
|
const unsigned fraction_bits = fp_bits == 64 ? 52u : 23u;
|
|
const unsigned bias = fp_bits == 64 ? 1023u : 127u;
|
|
const uint64_t integer_mask = int_bits == 64 ? UINT64_MAX : UINT64_C(0xffffffff);
|
|
const uint64_t fraction_mask = (UINT64_C(1) << fraction_bits) - 1;
|
|
const unsigned negative = is_signed && ((bits >> (int_bits - 1)) & 1);
|
|
uint64_t magnitude, mantissa;
|
|
unsigned top;
|
|
int exponent;
|
|
bits &= integer_mask;
|
|
magnitude = negative ? (UINT64_C(0) - bits) & integer_mask : bits;
|
|
if (!magnitude) return 0;
|
|
top = recomp_bit_index64(magnitude);
|
|
exponent = (int)top - (int)fbits;
|
|
if (top <= fraction_bits) {
|
|
mantissa = magnitude << (fraction_bits - top);
|
|
} else {
|
|
const unsigned discarded = top - fraction_bits; /* 1..40 for S/D. */
|
|
const uint64_t remainder = magnitude & ((UINT64_C(1) << discarded) - 1);
|
|
const uint64_t halfway = UINT64_C(1) << (discarded - 1);
|
|
const unsigned mode = (unsigned)((fpcr >> 22) & 3);
|
|
mantissa = magnitude >> discarded;
|
|
if (remainder) {
|
|
*fpsr |= UINT64_C(16);
|
|
if ((mode == 0 && (remainder > halfway ||
|
|
(remainder == halfway && (mantissa & 1)))) ||
|
|
(mode == 1 && !negative) || (mode == 2 && negative)) {
|
|
++mantissa;
|
|
}
|
|
}
|
|
if (mantissa >= (UINT64_C(1) << (fraction_bits + 1))) {
|
|
mantissa >>= 1;
|
|
++exponent;
|
|
}
|
|
}
|
|
return ((uint64_t)negative << (fp_bits - 1)) |
|
|
((uint64_t)(exponent + (int)bias) << fraction_bits) |
|
|
(mantissa & fraction_mask);
|
|
}
|
|
)FX";
|
|
}
|
|
|
|
// ABI 6 (FM1) header pieces. Each is empty in the ABI 5 text.
|
|
inline const char* FastmemAbiH() {
|
|
return R"RT(6
|
|
/* ABI 6 adds FM1, direct guest memory access through the host page table.
|
|
The memory helpers read Common::PageTable's entry array themselves, with its
|
|
layout folded in as the constants below, and serve an access only when it
|
|
lies inside one page whose entry holds a real backing pointer. Anything else
|
|
goes to the unchanged ABI 5 helper. The host enables this per context through
|
|
fm_table/fm_limit, and only after recomp_image_fastmem_v1 has confirmed these
|
|
constants and the field offsets for every loaded module. */
|
|
#define RECOMP_FEATURE_FASTMEM_PT1 1u
|
|
#define RECOMP_FM_PAGE_BITS 12
|
|
#define RECOMP_FM_STRIDE_LOG2 5
|
|
#define RECOMP_FM_PTR_MASK (~(uintptr_t)3)
|
|
#if defined(_MSC_VER) && !defined(__clang__)
|
|
#define RECOMP_NOINLINE __declspec(noinline)
|
|
#define RECOMP_LIKELY(x) (x)
|
|
#elif defined(__GNUC__) || defined(__clang__)
|
|
#define RECOMP_NOINLINE __attribute__((noinline))
|
|
#define RECOMP_LIKELY(x) __builtin_expect(!!(x), 1)
|
|
#else
|
|
#define RECOMP_NOINLINE
|
|
#define RECOMP_LIKELY(x) (x)
|
|
#endif
|
|
)RT";
|
|
}
|
|
|
|
inline const char* FastmemContextFieldsH() {
|
|
return R"RT( /* ABI 6 (FM1). Written only by the host thread that owns this context,
|
|
before it enters generated code. fm_limit 0 turns the fast path off.
|
|
Otherwise fm_table is host_mem->page_entries, and fm_limit is page
|
|
aligned, at most 2^39, and no higher than host_mem->address_space_max. */
|
|
uint32_t fm_reserved;
|
|
const unsigned char* fm_table;
|
|
uint64_t fm_limit;
|
|
)RT";
|
|
}
|
|
|
|
inline const char* FastmemLayoutPinsH() {
|
|
return R"RT(typedef char recomp_layout_fm_table[offsetof(GuestContext, fm_table) == 872 ? 1 : -1];
|
|
typedef char recomp_layout_fm_limit[offsetof(GuestContext, fm_limit) == 880 ? 1 : -1];
|
|
)RT";
|
|
}
|
|
|
|
// ABI 6 feature GG1 (generation code guard). Declared after recomp_code_guard.
|
|
inline const char* GuardGenH() {
|
|
return R"RT(/* ABI 6 feature GG1: the generation code guard.
|
|
A block runs the full recomp_code_guard only when its seen word differs from
|
|
its module's generation word, then records the generation it verified
|
|
against. The host moves the generation after every event that can change a
|
|
guarded instruction word or its mapping, and holds it at
|
|
RECOMP_GG_VERIFY_ALWAYS for anything it cannot prove stable; that value is
|
|
never stored as a seen word, and seen words start at 0, which the host never
|
|
uses, so an unnegotiated module verifies on every entry exactly as ABI 5.
|
|
Ordering: the host stores the word with release; blocks load both words
|
|
relaxed; recomp_code_guard_gen fences with acquire before reading code and
|
|
publishes the seen word with release after the check has passed. */
|
|
#define RECOMP_FEATURE_GUARD_GEN1 2u
|
|
#define RECOMP_GG_VERIFY_ALWAYS 0xFFFFFFFFu
|
|
/* Byte offset of the watch word in a host page-table entry. Nonzero marks a
|
|
code page the host keeps stable; stores to it must go to the host. */
|
|
#define RECOMP_GG_WATCH_OFFSET 24
|
|
#if defined(_MSC_VER) && !defined(__clang__)
|
|
#include <intrin.h>
|
|
#define RECOMP_GG_LOAD(x) (*(const volatile uint32_t*)&(x))
|
|
#if defined(_M_ARM64)
|
|
#define RECOMP_GG_ACQUIRE_FENCE() __dmb(_ARM64_BARRIER_ISHLD)
|
|
#define RECOMP_GG_STORE_RELEASE(x,v) __stlr32((volatile unsigned __int32*)&(x),(unsigned __int32)(v))
|
|
#else
|
|
#define RECOMP_GG_ACQUIRE_FENCE() _ReadWriteBarrier()
|
|
#define RECOMP_GG_STORE_RELEASE(x,v) do{ _ReadWriteBarrier(); *(volatile uint32_t*)&(x)=(v); }while(0)
|
|
#endif
|
|
#elif defined(__GNUC__) || defined(__clang__)
|
|
#define RECOMP_GG_LOAD(x) __atomic_load_n(&(x),__ATOMIC_RELAXED)
|
|
#define RECOMP_GG_ACQUIRE_FENCE() __atomic_thread_fence(__ATOMIC_ACQUIRE)
|
|
#define RECOMP_GG_STORE_RELEASE(x,v) __atomic_store_n(&(x),(v),__ATOMIC_RELEASE)
|
|
#else
|
|
#error "GG1 needs 32-bit atomic loads, stores and fences"
|
|
#endif
|
|
/* Hidden where that exists: the word is only ever reached from inside its own
|
|
image (the host gets its address from the handshake), so blocks address it
|
|
directly instead of through the GOT. Not __attribute__((cold)) on the miss
|
|
path: Apple clang then splits a cold fragment out of every block, one
|
|
symbol and unwind entry each; an unlikely branch keeps the call at the end
|
|
of the block instead. */
|
|
#if defined(__GNUC__) || defined(__clang__)
|
|
#define RECOMP_GG_STATIC static inline
|
|
#define RECOMP_GG_UNLIKELY(x) __builtin_expect(!!(x),0)
|
|
#if defined(_WIN32)
|
|
#define RECOMP_GG_HIDDEN
|
|
#else
|
|
#define RECOMP_GG_HIDDEN __attribute__((visibility("hidden")))
|
|
#endif
|
|
#else
|
|
#define RECOMP_GG_STATIC static __inline
|
|
#define RECOMP_GG_UNLIKELY(x) (x)
|
|
#define RECOMP_GG_HIDDEN
|
|
#endif
|
|
extern uint32_t g_recomp_gg_word RECOMP_GG_HIDDEN;
|
|
void recomp_code_guard_gen(GuestContext*,uint64_t,const uint32_t*,uint32_t,int,uint32_t*,uint32_t);
|
|
/* Each unit keeps its blocks' seen words in its own zero-initialised static
|
|
array, recomp_gg_seen (no file size), so a block reaches its slot at a
|
|
link-time constant address. Each block's expected words are preceded by a
|
|
three-word header: word count, then the module-relative PC, low and high.
|
|
The miss path is one private copy per unit with three register arguments.
|
|
The generation is loaded there, before recomp_code_guard_gen's acquire
|
|
fence; a value newer than the one the block compared is equally valid to
|
|
record. */
|
|
RECOMP_GG_STATIC RECOMP_NOINLINE void recomp_gg_miss(GuestContext* c,uint32_t* seen,const uint32_t* h){
|
|
recomp_code_guard_gen(c,g_module_base+((uint64_t)h[1]|((uint64_t)h[2]<<32)),h+3,h[0],
|
|
g_recomp_guard_host_v2,seen,RECOMP_GG_LOAD(g_recomp_gg_word));
|
|
}
|
|
#define RECOMP_GG_GUARD(hdr,idx) \
|
|
if(RECOMP_GG_UNLIKELY(RECOMP_GG_LOAD(recomp_gg_seen[idx])!=RECOMP_GG_LOAD(g_recomp_gg_word))) \
|
|
recomp_gg_miss(c,&recomp_gg_seen[idx],hdr);
|
|
)RT";
|
|
}
|
|
|
|
// ABI 6 feature FPX1 pieces of the runtime header. Empty unless the option is on.
|
|
// Kept under MSVC's 16380-byte limit per string literal.
|
|
inline const char* FpxAbiH() {
|
|
return R"RT(6
|
|
/* ABI 6 without FM1 carries only FPX1 below. */
|
|
#if defined(_MSC_VER) && !defined(__clang__)
|
|
#define RECOMP_NOINLINE __declspec(noinline)
|
|
#define RECOMP_LIKELY(x) (x)
|
|
#elif defined(__GNUC__) || defined(__clang__)
|
|
#define RECOMP_NOINLINE __attribute__((noinline))
|
|
#define RECOMP_LIKELY(x) __builtin_expect(!!(x), 1)
|
|
#else
|
|
#define RECOMP_NOINLINE
|
|
#define RECOMP_LIKELY(x) (x)
|
|
#endif
|
|
)RT";
|
|
}
|
|
|
|
inline const char* FpxH() {
|
|
return R"RT(
|
|
/* ABI 6 feature FPX1: exact native floating point.
|
|
|
|
A covered op (FADD/FSUB/FMUL/FNMUL/FDIV/FMLA-family/FRECPS/FRSQRTS/FSQRT,
|
|
S and D, scalar and per lane) first computes natively, and keeps that result
|
|
only when it provably equals the architectural one and adds no FPSR bit:
|
|
- the gate RECOMP_FPX_OPEN: guest FPCR at its default apart from FZ (RMode
|
|
RN, no DN, AHP or FZ16, no trap enables, no FEAT_AFP bits) and FPSR.IXC
|
|
already set;
|
|
- the result is finite and larger in magnitude than the smallest normal, so
|
|
it is neither NaN (IOC), infinite (DZC, OFC) nor tiny (UFC); or it is
|
|
exact: any finite sum, or a finite product or quotient of a zero. What is
|
|
left is IXC, which is already set;
|
|
- with FZ clear no input is flushed, so IDC cannot arise. With FZ set
|
|
(RECOMP_FPX_FZ, passed to every helper) the ARM op flushes a subnormal
|
|
input to zero and sets IDC, and flushes a result whose unrounded magnitude
|
|
is below the smallest normal to zero and sets UFC. So under FZ a result is
|
|
kept only when no input is subnormal (RECOMP_FPX_SUBN) and the result
|
|
needs no flush: larger in magnitude than the smallest normal (rounding is
|
|
monotonic, so the unrounded value is at least the smallest normal), or an
|
|
exact zero or exact zero-operand product or quotient as above. A finite
|
|
sum is no longer kept unconditionally: one in the subnormal range, or one
|
|
that rounds to exactly the smallest normal, falls back.
|
|
IEEE add, subtract, multiply, divide, square root and fused multiply-add are
|
|
correctly rounded, so under the host FP mode below every such result equals
|
|
the ARM result bit for bit. Anything else takes the unchanged exact body,
|
|
which FPX1 sites call out of line (recomp_fpe_*, recomp_runtime.c).
|
|
|
|
Lifted code keeps more than the above: a64_lift.py outlines each guard
|
|
into a wrapper (a64_fpx_*, a64_fpx.h) that, past these fast paths and
|
|
inside the same gate, also keeps the native FADD/FSUB S result that
|
|
overflowed with OFC already set or, under FZ, is flushed with UFC already
|
|
set (a64_fpx_fz_add32/sub32), and the native
|
|
FMUL/FNMUL/FMLA-family S result, tiny, infinite or of flushed inputs
|
|
included, when every flag the op could raise (IDC, UFC, OFC) is already
|
|
set (a64_fpx_sticky32; checked game-free by Hookshot's
|
|
tests/test_fpx_sticky.py). The shadow build keeps the inline form above
|
|
and never reaches those wrappers, so it does not validate them.
|
|
|
|
Host contract, enforced by the host once it has negotiated FPX1 through
|
|
recomp_image_fpx_v1: on x86-64 MXCSR round-to-nearest with DAZ and FTZ clear
|
|
and every exception masked; on AArch64 FPCR 0. Bit 32 of fpcr is the host's
|
|
kill switch (RECOMP_FPX_INHIBIT): the guest register is 32 bits, so MRS/MSR
|
|
never expose or clear it, and with it set every op takes the exact body.
|
|
|
|
x86-64 has no FMA in its baseline, so a single-precision FMA is computed in
|
|
binary64: the product of two binary32 values is exact there, one rounding
|
|
follows, and rounding that again to binary32 equals rounding the exact value
|
|
once unless the binary64 value is itself a binary32 midpoint, which falls
|
|
back (RECOMP_FPX_MIDPOINT). Because the product is exact, contracting the
|
|
expression gives the same value. Double-precision FMA needs a hardware FMA
|
|
(__FMA__); without one it stays exact-body only. */
|
|
#include <float.h>
|
|
#define RECOMP_FEATURE_FPX1 4u
|
|
#define RECOMP_FPX_INHIBIT (UINT64_C(1) << 32)
|
|
/* Every FPCR control bit but FZ (bit 24), which the helpers handle. */
|
|
#define RECOMP_FPX_FPCR_MASK (UINT64_C(0x06C8FF07) | RECOMP_FPX_INHIBIT)
|
|
#define RECOMP_FPX_FZ(c) ((uint32_t)(c)->fpcr & 0x01000000u)
|
|
#if defined(RECOMP_NO_NATIVE_FP) || defined(__FAST_MATH__) || defined(_M_FP_FAST) || \
|
|
(defined(FLT_EVAL_METHOD) && FLT_EVAL_METHOD != 0)
|
|
#define RECOMP_FPX_HOST 0 /* compiled out: every op takes the exact body */
|
|
#elif defined(__aarch64__) && (defined(__GNUC__) || defined(__clang__))
|
|
#define RECOMP_FPX_HOST 1 /* the host FPU is the guest's */
|
|
#elif ((defined(__x86_64__) || defined(_M_X64)) && !defined(_M_ARM64EC)) || (defined(__wasm__) && defined(__SSE2__))
|
|
#define RECOMP_FPX_HOST 2 /* SSE2 (wasm: Emscripten's SSE2 on wasm SIMD; its float ops are IEEE round-to-nearest too) */
|
|
#include <emmintrin.h>
|
|
#else
|
|
#define RECOMP_FPX_HOST 0
|
|
#endif
|
|
#if RECOMP_FPX_HOST == 1 || \
|
|
(RECOMP_FPX_HOST == 2 && defined(__FMA__) && (defined(__GNUC__) || defined(__clang__)))
|
|
#define RECOMP_FPX_FMA64 1 /* a correctly rounded binary64 FMA instruction */
|
|
#else
|
|
#define RECOMP_FPX_FMA64 0
|
|
#endif
|
|
#ifndef RECOMP_FPX_PROBE
|
|
#define RECOMP_FPX_PROBE(k) ((void)0) /* test and shadow builds count here */
|
|
#endif
|
|
#define RECOMP_FPX_OPEN(c) (((c)->fpsr & 16u) && !((c)->fpcr & RECOMP_FPX_FPCR_MASK))
|
|
#define RECOMP_FPX_FIN32(t) (((t) & 0x7f800000u) != 0x7f800000u)
|
|
#define RECOMP_FPX_BIG32(t) ((uint32_t)(((t) & 0x7fffffffu) - 0x00800001u) < 0x7effffffu)
|
|
#define RECOMP_FPX_ZERO32(x) (((x) & 0x7fffffffu) == 0)
|
|
/* A nonzero subnormal, which FZ flushes (and sets IDC for). */
|
|
#define RECOMP_FPX_SUBN32(x) ((uint32_t)(((uint32_t)(x) << 1) - 1u) < 0x00ffffffu)
|
|
#define RECOMP_FPX_FIN64(t) (((t) & UINT64_C(0x7ff0000000000000)) != UINT64_C(0x7ff0000000000000))
|
|
#define RECOMP_FPX_BIG64(t) \
|
|
(((t) & UINT64_C(0x7fffffffffffffff)) - UINT64_C(0x0010000000000001) < UINT64_C(0x7fdfffffffffffff))
|
|
#define RECOMP_FPX_ZERO64(x) (((x) & UINT64_C(0x7fffffffffffffff)) == 0)
|
|
#define RECOMP_FPX_SUBN64(x) ((((uint64_t)(x) << 1) - 1u) < UINT64_C(0x001fffffffffffff))
|
|
/* Products: large, or a finite product of a zero (exact, the zero's sign rule
|
|
is IEEE's on both). Quotients: large, or a finite quotient of a zero. */
|
|
#define RECOMP_FPX_KEEP32(t, a, b) \
|
|
(RECOMP_FPX_BIG32(t) || ((RECOMP_FPX_ZERO32(a) || RECOMP_FPX_ZERO32(b)) && RECOMP_FPX_FIN32(t)))
|
|
#define RECOMP_FPX_KEEP64(t, a, b) \
|
|
(RECOMP_FPX_BIG64(t) || ((RECOMP_FPX_ZERO64(a) || RECOMP_FPX_ZERO64(b)) && RECOMP_FPX_FIN64(t)))
|
|
#define RECOMP_FPX_KEEPDIV32(t, a) (RECOMP_FPX_BIG32(t) || (RECOMP_FPX_ZERO32(a) && RECOMP_FPX_FIN32(t)))
|
|
#define RECOMP_FPX_KEEPDIV64(t, a) (RECOMP_FPX_BIG64(t) || (RECOMP_FPX_ZERO64(a) && RECOMP_FPX_FIN64(t)))
|
|
/* Under FZ: no subnormal operand. A sum is then exact whenever it is below the
|
|
smallest normal, so zero is the only such sum it keeps. */
|
|
#define RECOMP_FPX_FZOK32(a, b) (!RECOMP_FPX_SUBN32(a) && !RECOMP_FPX_SUBN32(b))
|
|
#define RECOMP_FPX_FZOK64(a, b) (!RECOMP_FPX_SUBN64(a) && !RECOMP_FPX_SUBN64(b))
|
|
#define RECOMP_FPX_FZSUM32(t, a, b) \
|
|
(RECOMP_FPX_FZOK32(a, b) && (RECOMP_FPX_BIG32(t) || RECOMP_FPX_ZERO32(t)))
|
|
#define RECOMP_FPX_FZSUM64(t, a, b) \
|
|
(RECOMP_FPX_FZOK64(a, b) && (RECOMP_FPX_BIG64(t) || RECOMP_FPX_ZERO64(t)))
|
|
/* A binary64 value in the binary32 normal range that lies exactly halfway
|
|
between two binary32 values. */
|
|
#define RECOMP_FPX_MIDPOINT(s) ((recomp_fpx_db(s) & UINT64_C(0x1fffffff)) == UINT64_C(0x10000000))
|
|
|
|
static RECOMP_INLINE float recomp_fpx_f(uint64_t x) { uint32_t t = (uint32_t)x; float f; memcpy(&f, &t, 4); return f; }
|
|
static RECOMP_INLINE uint64_t recomp_fpx_fb(float f) { uint32_t t; memcpy(&t, &f, 4); return t; }
|
|
static RECOMP_INLINE double recomp_fpx_d(uint64_t x) { double f; memcpy(&f, &x, 8); return f; }
|
|
static RECOMP_INLINE uint64_t recomp_fpx_db(double f) { uint64_t t; memcpy(&t, &f, 8); return t; }
|
|
|
|
/* Each returns 1 and the result bits in *v when the native result may be kept. */
|
|
static RECOMP_INLINE int recomp_fpx_add32(uint64_t a, uint64_t b, uint32_t fz, uint64_t* v) {
|
|
#if RECOMP_FPX_HOST
|
|
const uint64_t t = recomp_fpx_fb(recomp_fpx_f(a) + recomp_fpx_f(b));
|
|
if (RECOMP_FPX_FIN32(t) && (!fz || RECOMP_FPX_FZSUM32(t, a, b))) { *v = t; return 1; }
|
|
#endif
|
|
(void)a; (void)b; (void)fz; (void)v; return 0;
|
|
}
|
|
static RECOMP_INLINE int recomp_fpx_sub32(uint64_t a, uint64_t b, uint32_t fz, uint64_t* v) {
|
|
#if RECOMP_FPX_HOST
|
|
const uint64_t t = recomp_fpx_fb(recomp_fpx_f(a) - recomp_fpx_f(b));
|
|
if (RECOMP_FPX_FIN32(t) && (!fz || RECOMP_FPX_FZSUM32(t, a, b))) { *v = t; return 1; }
|
|
#endif
|
|
(void)a; (void)b; (void)fz; (void)v; return 0;
|
|
}
|
|
static RECOMP_INLINE int recomp_fpx_mul32(uint64_t a, uint64_t b, uint32_t fz, uint64_t* v) {
|
|
#if RECOMP_FPX_HOST
|
|
const uint64_t t = recomp_fpx_fb(recomp_fpx_f(a) * recomp_fpx_f(b));
|
|
if (RECOMP_FPX_KEEP32(t, a, b) && (!fz || RECOMP_FPX_FZOK32(a, b))) { *v = t; return 1; }
|
|
#endif
|
|
(void)a; (void)b; (void)fz; (void)v; return 0;
|
|
}
|
|
static RECOMP_INLINE int recomp_fpx_div32(uint64_t a, uint64_t b, uint32_t fz, uint64_t* v) {
|
|
#if RECOMP_FPX_HOST
|
|
const uint64_t t = recomp_fpx_fb(recomp_fpx_f(a) / recomp_fpx_f(b));
|
|
if (RECOMP_FPX_KEEPDIV32(t, a) && (!fz || RECOMP_FPX_FZOK32(a, b))) { *v = t; return 1; }
|
|
#endif
|
|
(void)a; (void)b; (void)fz; (void)v; return 0;
|
|
}
|
|
static RECOMP_INLINE int recomp_fpx_fma32(uint64_t a, uint64_t b, uint64_t z, uint32_t fz, uint64_t* v) {
|
|
#if RECOMP_FPX_HOST == 1
|
|
const uint64_t t = recomp_fpx_fb(__builtin_fmaf(recomp_fpx_f(a), recomp_fpx_f(b), recomp_fpx_f(z)));
|
|
#elif RECOMP_FPX_HOST == 2
|
|
const double s = (double)recomp_fpx_f(a) * (double)recomp_fpx_f(b) + (double)recomp_fpx_f(z);
|
|
uint64_t t;
|
|
if (RECOMP_FPX_MIDPOINT(s)) return 0;
|
|
t = recomp_fpx_fb((float)s);
|
|
#endif
|
|
#if RECOMP_FPX_HOST
|
|
if (RECOMP_FPX_KEEP32(t, a, b) && (!fz || (RECOMP_FPX_FZOK32(a, b) && !RECOMP_FPX_SUBN32(z)))) {
|
|
*v = t;
|
|
return 1;
|
|
}
|
|
#endif
|
|
(void)a; (void)b; (void)z; (void)fz; (void)v; return 0;
|
|
}
|
|
/* FRECPS: 2 - a*b; FRSQRTS: (3 - a*b)/2, each rounded once. inf*0 gives NaN
|
|
here, which is never kept; the exact body returns 2.0 or 1.5. */
|
|
static RECOMP_INLINE int recomp_fpx_recps32(uint64_t a, uint64_t b, uint32_t fz, uint64_t* v) {
|
|
#if RECOMP_FPX_HOST == 1
|
|
const uint64_t t = recomp_fpx_fb(__builtin_fmaf(-recomp_fpx_f(a), recomp_fpx_f(b), 2.0f));
|
|
#elif RECOMP_FPX_HOST == 2
|
|
const double s = 2.0 - (double)recomp_fpx_f(a) * (double)recomp_fpx_f(b);
|
|
uint64_t t;
|
|
if (RECOMP_FPX_MIDPOINT(s)) return 0;
|
|
t = recomp_fpx_fb((float)s);
|
|
#endif
|
|
#if RECOMP_FPX_HOST
|
|
if (RECOMP_FPX_BIG32(t) && (!fz || RECOMP_FPX_FZOK32(a, b))) { *v = t; return 1; }
|
|
#endif
|
|
(void)a; (void)b; (void)fz; (void)v; return 0;
|
|
}
|
|
static RECOMP_INLINE int recomp_fpx_rsqrts32(uint64_t a, uint64_t b, uint32_t fz, uint64_t* v) {
|
|
#if RECOMP_FPX_HOST == 1
|
|
/* Halving a normal result is exact, and a normal result is all that is kept. */
|
|
const uint64_t t = recomp_fpx_fb(__builtin_fmaf(-recomp_fpx_f(a), recomp_fpx_f(b), 3.0f) * 0.5f);
|
|
#elif RECOMP_FPX_HOST == 2
|
|
const double s = (3.0 - (double)recomp_fpx_f(a) * (double)recomp_fpx_f(b)) * 0.5;
|
|
uint64_t t;
|
|
if (RECOMP_FPX_MIDPOINT(s)) return 0;
|
|
t = recomp_fpx_fb((float)s);
|
|
#endif
|
|
#if RECOMP_FPX_HOST
|
|
if (RECOMP_FPX_BIG32(t) && (!fz || RECOMP_FPX_FZOK32(a, b))) { *v = t; return 1; }
|
|
#endif
|
|
(void)a; (void)b; (void)fz; (void)v; return 0;
|
|
}
|
|
/* Square root of a positive finite input (subnormal included) is normal and
|
|
finite; of +-0 it is the input. Negative inputs, infinities and NaNs fall
|
|
back, and under FZ so do subnormal inputs (flushed, IDC). */
|
|
static RECOMP_INLINE int recomp_fpx_sqrt32(uint64_t a, uint32_t fz, uint64_t* v) {
|
|
#if RECOMP_FPX_HOST
|
|
if (RECOMP_FPX_ZERO32(a)) { *v = a; return 1; }
|
|
if ((uint32_t)a - 1u >= 0x7f7fffffu || (fz && RECOMP_FPX_SUBN32(a))) return 0;
|
|
#if RECOMP_FPX_HOST == 1
|
|
*v = recomp_fpx_fb(__builtin_sqrtf(recomp_fpx_f(a)));
|
|
#else
|
|
*v = recomp_fpx_fb(_mm_cvtss_f32(_mm_sqrt_ss(_mm_set_ss(recomp_fpx_f(a)))));
|
|
#endif
|
|
return 1;
|
|
#else
|
|
(void)a; (void)fz; (void)v; return 0;
|
|
#endif
|
|
}
|
|
)RT";
|
|
}
|
|
|
|
inline const char* FpxH64() {
|
|
return R"RT(
|
|
static RECOMP_INLINE int recomp_fpx_add64(uint64_t a, uint64_t b, uint32_t fz, uint64_t* v) {
|
|
#if RECOMP_FPX_HOST
|
|
const uint64_t t = recomp_fpx_db(recomp_fpx_d(a) + recomp_fpx_d(b));
|
|
if (RECOMP_FPX_FIN64(t) && (!fz || RECOMP_FPX_FZSUM64(t, a, b))) { *v = t; return 1; }
|
|
#endif
|
|
(void)a; (void)b; (void)fz; (void)v; return 0;
|
|
}
|
|
static RECOMP_INLINE int recomp_fpx_sub64(uint64_t a, uint64_t b, uint32_t fz, uint64_t* v) {
|
|
#if RECOMP_FPX_HOST
|
|
const uint64_t t = recomp_fpx_db(recomp_fpx_d(a) - recomp_fpx_d(b));
|
|
if (RECOMP_FPX_FIN64(t) && (!fz || RECOMP_FPX_FZSUM64(t, a, b))) { *v = t; return 1; }
|
|
#endif
|
|
(void)a; (void)b; (void)fz; (void)v; return 0;
|
|
}
|
|
static RECOMP_INLINE int recomp_fpx_mul64(uint64_t a, uint64_t b, uint32_t fz, uint64_t* v) {
|
|
#if RECOMP_FPX_HOST
|
|
const uint64_t t = recomp_fpx_db(recomp_fpx_d(a) * recomp_fpx_d(b));
|
|
if (RECOMP_FPX_KEEP64(t, a, b) && (!fz || RECOMP_FPX_FZOK64(a, b))) { *v = t; return 1; }
|
|
#endif
|
|
(void)a; (void)b; (void)fz; (void)v; return 0;
|
|
}
|
|
static RECOMP_INLINE int recomp_fpx_div64(uint64_t a, uint64_t b, uint32_t fz, uint64_t* v) {
|
|
#if RECOMP_FPX_HOST
|
|
const uint64_t t = recomp_fpx_db(recomp_fpx_d(a) / recomp_fpx_d(b));
|
|
if (RECOMP_FPX_KEEPDIV64(t, a) && (!fz || RECOMP_FPX_FZOK64(a, b))) { *v = t; return 1; }
|
|
#endif
|
|
(void)a; (void)b; (void)fz; (void)v; return 0;
|
|
}
|
|
static RECOMP_INLINE int recomp_fpx_fma64(uint64_t a, uint64_t b, uint64_t z, uint32_t fz, uint64_t* v) {
|
|
#if RECOMP_FPX_FMA64
|
|
const uint64_t t = recomp_fpx_db(__builtin_fma(recomp_fpx_d(a), recomp_fpx_d(b), recomp_fpx_d(z)));
|
|
if (RECOMP_FPX_KEEP64(t, a, b) && (!fz || (RECOMP_FPX_FZOK64(a, b) && !RECOMP_FPX_SUBN64(z)))) {
|
|
*v = t;
|
|
return 1;
|
|
}
|
|
#endif
|
|
(void)a; (void)b; (void)z; (void)fz; (void)v; return 0;
|
|
}
|
|
static RECOMP_INLINE int recomp_fpx_recps64(uint64_t a, uint64_t b, uint32_t fz, uint64_t* v) {
|
|
#if RECOMP_FPX_FMA64
|
|
const uint64_t t = recomp_fpx_db(__builtin_fma(-recomp_fpx_d(a), recomp_fpx_d(b), 2.0));
|
|
if (RECOMP_FPX_BIG64(t) && (!fz || RECOMP_FPX_FZOK64(a, b))) { *v = t; return 1; }
|
|
#endif
|
|
(void)a; (void)b; (void)fz; (void)v; return 0;
|
|
}
|
|
static RECOMP_INLINE int recomp_fpx_rsqrts64(uint64_t a, uint64_t b, uint32_t fz, uint64_t* v) {
|
|
#if RECOMP_FPX_FMA64
|
|
const uint64_t t = recomp_fpx_db(__builtin_fma(-recomp_fpx_d(a), recomp_fpx_d(b), 3.0) * 0.5);
|
|
if (RECOMP_FPX_BIG64(t) && (!fz || RECOMP_FPX_FZOK64(a, b))) { *v = t; return 1; }
|
|
#endif
|
|
(void)a; (void)b; (void)fz; (void)v; return 0;
|
|
}
|
|
static RECOMP_INLINE int recomp_fpx_sqrt64(uint64_t a, uint32_t fz, uint64_t* v) {
|
|
#if RECOMP_FPX_HOST
|
|
if (RECOMP_FPX_ZERO64(a)) { *v = a; return 1; }
|
|
if (a - 1u >= UINT64_C(0x7fefffffffffffff) || (fz && RECOMP_FPX_SUBN64(a))) return 0;
|
|
#if RECOMP_FPX_HOST == 1
|
|
*v = recomp_fpx_db(__builtin_sqrt(recomp_fpx_d(a)));
|
|
#else
|
|
*v = recomp_fpx_db(_mm_cvtsd_f64(_mm_sqrt_sd(_mm_setzero_pd(), _mm_set_sd(recomp_fpx_d(a)))));
|
|
#endif
|
|
return 1;
|
|
#else
|
|
(void)a; (void)fz; (void)v; return 0;
|
|
#endif
|
|
}
|
|
)RT";
|
|
}
|
|
|
|
// Instrumentation only (g_emit_fpx_shadow): the runtime half of the shadow
|
|
// build. Per kind (op*2 + double) it counts calls, open gates, kept fast
|
|
// results and mismatches against the exact body, which runs anyway; the first
|
|
// 100 mismatches are logged with their operands. Counts are per thread and
|
|
// folded into the totals every 2^20 calls; mismatches are counted at once.
|
|
// SUYU_RECOMP_FPX_SHADOW_LOG names the log (stderr otherwise); the totals are
|
|
// rewritten to it + ".sum" as they grow and at exit.
|
|
inline const char* FpxShadowC() {
|
|
return R"RT(
|
|
/* FPX1 shadow instrumentation (never timed). */
|
|
#include <stdio.h>
|
|
#if defined(_MSC_VER)
|
|
#include <intrin.h>
|
|
#define RECOMP_FPX_TLS __declspec(thread)
|
|
#define RECOMP_FPX_ADD(p, v) _InterlockedExchangeAdd64((volatile long long*)(p), (long long)(v))
|
|
#define RECOMP_FPX_OR(p, v) _InterlockedOr64((volatile long long*)(p), (long long)(v))
|
|
#else
|
|
#define RECOMP_FPX_TLS _Thread_local
|
|
#define RECOMP_FPX_ADD(p, v) __atomic_fetch_add((p), (v), __ATOMIC_RELAXED)
|
|
#define RECOMP_FPX_OR(p, v) __atomic_fetch_or((p), (v), __ATOMIC_RELAXED)
|
|
#endif
|
|
static uint64_t g_recomp_fpx_sh[16][4]; /* calls, gate open, kept, mismatches */
|
|
static uint64_t g_recomp_fpx_sh_fpsr, g_recomp_fpx_sh_folds;
|
|
static RECOMP_FPX_TLS uint64_t t_recomp_fpx_sh[16][3];
|
|
static RECOMP_FPX_TLS uint64_t t_recomp_fpx_sh_n, t_recomp_fpx_sh_or;
|
|
static void recomp_fpx_shadow_dump(void) {
|
|
static const char* const names[8] = {"add","sub","mul","div","fma","recps","rsqrts","sqrt"};
|
|
const char* path = getenv("SUYU_RECOMP_FPX_SHADOW_LOG");
|
|
char sum[1024];
|
|
FILE* f = stderr;
|
|
if (path && *path) {
|
|
snprintf(sum, sizeof sum, "%s.sum", path);
|
|
f = fopen(sum, "w");
|
|
if (!f) return;
|
|
}
|
|
fprintf(f, "kind calls gate_open kept mismatches\n");
|
|
for (unsigned k = 0; k < 16; ++k)
|
|
fprintf(f, "%s%u %llu %llu %llu %llu\n", names[k / 2], k & 1 ? 64u : 32u,
|
|
(unsigned long long)g_recomp_fpx_sh[k][0], (unsigned long long)g_recomp_fpx_sh[k][1],
|
|
(unsigned long long)g_recomp_fpx_sh[k][2], (unsigned long long)g_recomp_fpx_sh[k][3]);
|
|
fprintf(f, "fpsr_seen 0x%llx\n", (unsigned long long)g_recomp_fpx_sh_fpsr);
|
|
if (f != stderr) fclose(f);
|
|
}
|
|
static void recomp_fpx_shadow_fold_counts(void) {
|
|
for (unsigned k = 0; k < 16; ++k)
|
|
for (unsigned j = 0; j < 3; ++j) {
|
|
RECOMP_FPX_ADD(&g_recomp_fpx_sh[k][j], t_recomp_fpx_sh[k][j]);
|
|
t_recomp_fpx_sh[k][j] = 0;
|
|
}
|
|
RECOMP_FPX_OR(&g_recomp_fpx_sh_fpsr, t_recomp_fpx_sh_or);
|
|
}
|
|
/* At exit, on the exiting thread: its own unfolded counts, then the totals. */
|
|
static void recomp_fpx_shadow_exit(void) {
|
|
recomp_fpx_shadow_fold_counts();
|
|
recomp_fpx_shadow_dump();
|
|
}
|
|
static void recomp_fpx_shadow_fold(void) {
|
|
static int registered;
|
|
recomp_fpx_shadow_fold_counts();
|
|
if (!registered) { registered = 1; atexit(recomp_fpx_shadow_exit); }
|
|
if ((RECOMP_FPX_ADD(&g_recomp_fpx_sh_folds, 1) & 63) == 0) recomp_fpx_shadow_dump();
|
|
}
|
|
void recomp_fpx_shadow(unsigned kind, int gate, int kept, uint64_t fast, uint64_t exact,
|
|
uint64_t fpsr_before, uint64_t fpsr_after, uint64_t a, uint64_t b) {
|
|
kind &= 15;
|
|
++t_recomp_fpx_sh[kind][0];
|
|
t_recomp_fpx_sh[kind][1] += gate != 0;
|
|
t_recomp_fpx_sh[kind][2] += kept != 0;
|
|
if (kept && (fast != exact || fpsr_before != fpsr_after)) {
|
|
const uint64_t n = RECOMP_FPX_ADD(&g_recomp_fpx_sh[kind][3], 1);
|
|
if (n < 100) {
|
|
const char* path = getenv("SUYU_RECOMP_FPX_SHADOW_LOG");
|
|
FILE* f = path && *path ? fopen(path, "a") : stderr;
|
|
if (f) {
|
|
fprintf(f, "MISMATCH kind=%u a=%016llx b=%016llx fast=%016llx exact=%016llx fpsr %llx->%llx\n",
|
|
kind, (unsigned long long)a, (unsigned long long)b, (unsigned long long)fast,
|
|
(unsigned long long)exact, (unsigned long long)fpsr_before,
|
|
(unsigned long long)fpsr_after);
|
|
if (f != stderr) fclose(f);
|
|
}
|
|
}
|
|
recomp_fpx_shadow_dump();
|
|
}
|
|
t_recomp_fpx_sh_or |= fpsr_after & 0x9f;
|
|
/* The first call registers the exit report; then every 2^20th folds. */
|
|
if (!(t_recomp_fpx_sh_n++ & 0xfffff)) recomp_fpx_shadow_fold();
|
|
}
|
|
)RT";
|
|
}
|
|
|
|
// `fpx`: 0 off, 1 FPX1, 2 FPX1 with the shadow instrumentation.
|
|
inline std::string BuildRuntimeH(bool fastmem, bool guard_gen = false, int fpx = 0) {
|
|
std::string text = std::string(R"RT(#ifndef SUYU_RECOMP_RUNTIME_H
|
|
#define SUYU_RECOMP_RUNTIME_H
|
|
#include <stdint.h>
|
|
#include <stddef.h> /* offsetof, for the layout assertions below */
|
|
#include <string.h> /* memcpy, for the memory accessors */
|
|
#include <stdlib.h> /* calloc, for the block index
|
|
|
|
Included here rather than in the generated module source because the module
|
|
source only pulls in this header and <stdint.h>. Without it calloc is an
|
|
implicit declaration returning int, the returned pointer is truncated to 32
|
|
bits, and the first lookup through the index dereferences garbage - which is
|
|
a crash roughly 20 seconds into boot with nothing in the log to explain it. */
|
|
|
|
/* The memory accessors below are the hottest code in a generated module, and
|
|
leaving them to the compiler's discretion is not worth the risk: without a
|
|
forced inline MSVC declines them at /O2 in the larger translation units,
|
|
which puts a call back on the path this exists to remove. */
|
|
#if defined(_MSC_VER)
|
|
#define RECOMP_INLINE __forceinline
|
|
#elif defined(__GNUC__) || defined(__clang__)
|
|
#define RECOMP_INLINE inline __attribute__((always_inline))
|
|
#else
|
|
#define RECOMP_INLINE inline
|
|
#endif
|
|
)RT") + FPScanHelpers() + FPFixedHelpers() + R"RT(
|
|
/* Supplied by the host when the recompiled image is driven by an emulator
|
|
rather than run standalone. `size` is 1, 2, 4 or 8 bytes. */
|
|
#define RECOMP_IMAGE_ABI 5
|
|
|
|
typedef struct RecompHostMem {
|
|
void* user;
|
|
uint64_t (*load)(void* user, uint64_t va, uint32_t size);
|
|
void (*store)(void* user, uint64_t va, uint32_t size, uint64_t value);
|
|
/* Exclusive access. Appended rather than inserted so the existing two stay
|
|
where they are, but note both sides of this struct must still be rebuilt
|
|
together: a DLL built before these existed would leave them uninitialised
|
|
and the first LDXR would call through a garbage pointer.
|
|
|
|
These have to reach the *emulator's* monitor rather than any private one.
|
|
The recompiled code and the fallback JIT run guest threads that contend
|
|
for the same locks, so two monitors means both engines can win the same
|
|
exclusive and the lock stops being a lock. */
|
|
uint64_t (*excl_load)(void* user, uint64_t va, uint32_t size);
|
|
uint32_t (*excl_store)(void* user, uint64_t va, uint32_t size, uint64_t value);
|
|
void (*clear_excl)(void* user);
|
|
/* The physical counter. Unlike FPCR this cannot live in the context: it has
|
|
to advance, and it has to agree with what the HLE kernel and the fallback
|
|
JIT report, or the guest sees time jump backwards whenever execution
|
|
crosses between engines. */
|
|
uint64_t (*read_cntpct)(void* user);
|
|
/* Exclusive pair forms. `size` is the width of ONE register (4 or 8), so
|
|
the reservation covers 2*size bytes. Split out rather than widening
|
|
excl_load/excl_store because the 64-bit pair needs a 128-bit
|
|
reservation, which no scalar return can carry. */
|
|
void (*excl_load_pair)(void* user, uint64_t va, uint32_t size, uint64_t* lo,
|
|
uint64_t* hi);
|
|
uint32_t (*excl_store_pair)(void* user, uint64_t va, uint32_t size, uint64_t lo,
|
|
uint64_t hi);
|
|
/* The host page table, so a mapped access resolves inline rather than
|
|
through `load`/`store`. Mirrors Memory::GetPointerImpl's fast path: mask
|
|
the address, bounds check, read one entry, extract the backing pointer.
|
|
|
|
An entry whose pointer is null means unmapped, debug, or GPU-tracked
|
|
memory - all of which need the callback, because the rasterizer has to be
|
|
told about the access. Only a non-null pointer is handled inline, so
|
|
nothing is bypassed that suyu would have done itself.
|
|
|
|
page_entries is null until the emulator hands over a real page table;
|
|
standalone builds leave it null and take the callback path always. */
|
|
const void* page_entries;
|
|
uint64_t page_entry_stride;
|
|
uint64_t page_bits;
|
|
uint64_t pointer_mask;
|
|
uint64_t address_space_max;
|
|
/* Reserved layout slot. ABI 5 verifies code on every entry and
|
|
never dereferences this pointer. Hosts must set it to null. */
|
|
const uint64_t* guard_generation;
|
|
} RecompHostMem;
|
|
|
|
typedef struct GuestContext {
|
|
uint64_t x[32]; uint64_t pc; uint8_t n,z,c,v;
|
|
uint8_t* mem; uint64_t mem_size; uint64_t mem_base_vaddr; int halted;
|
|
/* SVC signalling. The emitted code sets this to the instruction's imm
|
|
before calling recomp_svc(). The standalone runtime services the call
|
|
and clears it; when the recompiled code is instead driven by suyu's
|
|
Core::ArmRecomp backend, recomp_svc leaves it set so the emulator can
|
|
see the pending call and dispatch it through the real HLE kernel.
|
|
~0ULL means "no SVC pending". */
|
|
uint64_t pending_svc;
|
|
/* SIMD/FP register file, 128 bits each held as two 64-bit halves.
|
|
Deliberately placed after pending_svc: Core::ArmRecomp mirrors only the
|
|
prefix of this struct up to that field, so appending here leaves that
|
|
view valid. */
|
|
uint64_t vreg[32][2];
|
|
/* Thread pointer (TPIDR_EL0). Read/written by MRS/MSR; user code uses it
|
|
for thread-local storage. */
|
|
uint64_t tpidr_el0;
|
|
/* Host memory bridge. NULL for the standalone runtime, which owns the flat
|
|
buffer above. When suyu drives this code through Core::ArmRecomp the
|
|
guest's address space belongs to the emulator, not to us, so every
|
|
access has to go back out to Core::Memory instead of into `mem` -
|
|
otherwise the recompiled code and the HLE kernel would be looking at two
|
|
different memories. */
|
|
const struct RecompHostMem* host_mem;
|
|
/* Read-only thread pointer (TPIDRRO_EL0). Distinct from tpidr_el0 above:
|
|
on Horizon the kernel publishes the calling thread's thread-local
|
|
region here, and that region's first 0x100 bytes are the IPC message
|
|
buffer the kernel reads a SendSyncRequest out of. tpidr_el0 in contrast
|
|
belongs to the guest, which points it at its own thread structure.
|
|
Aliasing the two makes the guest write its IPC header at its thread
|
|
struct instead of the TLS region - the kernel then parses an all-zero
|
|
buffer (magic 0 instead of 'SFCI') - and makes the kernel's per-switch
|
|
publish of the TLS address clobber the guest's thread pointer, which
|
|
turns every C++ thread-local access into a near-null read.
|
|
Appended after host_mem so every pinned offset above stays put. */
|
|
uint64_t tpidrro_el0;
|
|
/* Floating-point control and status (FPCR/FPSR).
|
|
Their absence was not a missing feature but a silent wrong answer: the
|
|
guest sets a rounding mode or reads the exception flags, the value had
|
|
nowhere to live, and every FP result afterwards used the host default.
|
|
Worse, the state was dropped again on each transition to the fallback
|
|
JIT, so the two engines disagreed about rounding mid-computation.
|
|
Appended after tpidrro_el0 so every pinned offset above is unchanged. */
|
|
uint64_t fpcr;
|
|
uint64_t fpsr;
|
|
/* Blocks a chain of direct calls may still run before returning to the
|
|
dispatcher. It has to sit immediately after fpsr: Core::ArmRecomp mirrors
|
|
this struct by hand and its view ends here, so a field placed after
|
|
save_dir below lands 512 bytes further along on this side than on that
|
|
one. The layout assertions below are what catch that. */
|
|
int chain_budget;
|
|
)RT" + (fastmem ? FastmemContextFieldsH() : "") + R"RT( /* Save-data filesystem state */
|
|
char save_dir[512];
|
|
/* Heap break for SVC memory allocation */
|
|
uint64_t heap_base; uint64_t heap_end; uint64_t heap_cur;
|
|
/* IPC command buffer (simplified HLE) */
|
|
uint32_t ipc_cmd[64];
|
|
/* Open file handles for save data (simplified) */
|
|
void* save_handles[16]; int save_handle_count;
|
|
} GuestContext;
|
|
|
|
/* Core::ArmRecomp mirrors the prefix of this struct with its own declaration
|
|
so the emulator can read and write guest state without including this
|
|
header. Nothing enforces that across the two builds, so the offsets are
|
|
pinned here: if a field is inserted rather than appended, this fails loudly
|
|
at generation time instead of silently handing the emulator the wrong
|
|
register file. Keep in sync with GuestContextView in
|
|
core/arm/recomp/arm_recomp.cpp. */
|
|
/* A negative array size rather than _Static_assert: the generated project is
|
|
plain C compiled by whatever the user has, and MSVC's /TC mode rejects the
|
|
C11 form outright. This construct is valid in every C dialect. */
|
|
typedef char recomp_layout_pc[offsetof(GuestContext, pc) == 256 ? 1 : -1];
|
|
typedef char recomp_layout_svc[offsetof(GuestContext, pending_svc) == 304 ? 1 : -1];
|
|
typedef char recomp_layout_vreg[offsetof(GuestContext, vreg) == 312 ? 1 : -1];
|
|
typedef char recomp_layout_tpidr[offsetof(GuestContext, tpidr_el0) == 824 ? 1 : -1];
|
|
typedef char recomp_layout_fpcr[offsetof(GuestContext, fpcr) == 848 ? 1 : -1];
|
|
typedef char recomp_layout_fpsr[offsetof(GuestContext, fpsr) == 856 ? 1 : -1];
|
|
typedef char recomp_layout_host[offsetof(GuestContext, host_mem) == 832 ? 1 : -1];
|
|
typedef char recomp_layout_chain[offsetof(GuestContext, chain_budget) == 864 ? 1 : -1];
|
|
typedef char recomp_layout_tpidrro[offsetof(GuestContext, tpidrro_el0) == 840 ? 1 : -1];
|
|
)RT" + (fastmem ? FastmemLayoutPinsH() : "") + R"RT(
|
|
/* Where this module is actually loaded in the guest's address space.
|
|
Every address the static pass bakes in - ADR/ADRP results, branch targets,
|
|
return addresses - is relative to the module, because that is all it can
|
|
know: the loader picks the real base at run time and it differs per run.
|
|
Leaving it zero gives the module-relative behaviour the standalone runtime
|
|
wants; the emulator sets it once at load so computed data pointers land in
|
|
real memory rather than near null. One global per image, since each module
|
|
is its own shared library. */
|
|
extern uint64_t g_module_base;
|
|
extern int g_recomp_guard_host_v2;
|
|
void recomp_code_guard(GuestContext*,uint64_t,const uint32_t*,uint32_t,int);
|
|
void recomp_ic_ivau(GuestContext*,uint64_t,int);
|
|
|
|
typedef void (*BlockFn)(GuestContext*);
|
|
BlockFn recomp_lookup(uint64_t pc); void recomp_run(GuestContext* c);
|
|
/* Builds the direct block index. Called once at load, before any
|
|
guest thread runs. */
|
|
void recomp_build_index(void);
|
|
void recomp_set_flags(GuestContext*,int,uint64_t,uint64_t,uint64_t,int);
|
|
/* High 64 bits of a 64x64 multiply, for SMULH/UMULH. */
|
|
uint64_t recomp_smulh(uint64_t,uint64_t);
|
|
uint64_t recomp_umulh(uint64_t,uint64_t);
|
|
uint64_t recomp_fp_estimate(GuestContext*,uint64_t,unsigned,int);
|
|
int recomp_cond(GuestContext*,unsigned);
|
|
/* Guest memory access.
|
|
|
|
Defined once per module in recomp_runtime.c rather than inlined here. They
|
|
resolve the address through the host page table the same way
|
|
Memory::GetPointerImpl does, falling back to the emulator callback for
|
|
unmapped, debug or GPU-tracked pages.
|
|
|
|
Forcing them inline at every access site was measured and was worse: the
|
|
generated main image went from 100 MB to 222 MB and the race-phase frame rate
|
|
dropped 14%. Whatever the call costs, the instruction cache costs more. */
|
|
uint64_t recomp_load8(GuestContext*,uint64_t); uint64_t recomp_load16(GuestContext*,uint64_t);
|
|
uint64_t recomp_load32(GuestContext*,uint64_t); uint64_t recomp_load64(GuestContext*,uint64_t);
|
|
void recomp_store8(GuestContext*,uint64_t,uint64_t); void recomp_store16(GuestContext*,uint64_t,uint64_t);
|
|
void recomp_store32(GuestContext*,uint64_t,uint64_t); void recomp_store64(GuestContext*,uint64_t,uint64_t);
|
|
/* Pair forms, so LDP and STP resolve one address instead of two. */
|
|
void recomp_ldp32(GuestContext*,uint64_t,uint64_t*,uint64_t*);
|
|
void recomp_ldp64(GuestContext*,uint64_t,uint64_t*,uint64_t*);
|
|
void recomp_stp32(GuestContext*,uint64_t,uint64_t,uint64_t);
|
|
void recomp_stp64(GuestContext*,uint64_t,uint64_t,uint64_t);
|
|
void recomp_svc(GuestContext*,unsigned); void recomp_unhandled(GuestContext*,uint32_t,uint64_t);
|
|
void recomp_barrier(void);
|
|
/* AES S-box, forward or inverse. Built on first call. */
|
|
const uint8_t* recomp_aes_sbox(int inverse);
|
|
/* Load-exclusive: marks the address and returns its contents.
|
|
Store-exclusive: returns 0 on success and 1 if the mark was lost, which is
|
|
the sense of the status register STXR writes (0 = stored). */
|
|
uint64_t recomp_ldxr(GuestContext* c, uint64_t addr, uint32_t size);
|
|
uint32_t recomp_stxr(GuestContext* c, uint64_t addr, uint32_t size, uint64_t value);
|
|
void recomp_clrex(GuestContext* c);
|
|
/* CNTPCT_EL0: the 19.2 MHz physical counter the guest reads for timing. */
|
|
uint64_t recomp_cntpct(GuestContext* c);
|
|
/* Exclusive pair forms. `size` is the width of one register, 4 or 8. */
|
|
void recomp_ldxp(GuestContext* c, uint64_t addr, uint32_t size, uint64_t* lo, uint64_t* hi);
|
|
uint32_t recomp_stxp(GuestContext* c, uint64_t addr, uint32_t size, uint64_t lo, uint64_t hi);
|
|
/* halted values. 1 is an ordinary stop; 2 asks the host to re-execute the
|
|
current PC on its interpreter fallback because the decoder had no
|
|
translation for the instruction there. */
|
|
#define RECOMP_HALT_UNHANDLED 2
|
|
/* A guest breakpoint, with PC parked on BRK. Never invokes fallback. */
|
|
#define RECOMP_HALT_BREAKPOINT 3
|
|
#define RECOMP_HALT_IC_IVAU 4
|
|
/* Save-data API callable from recompiled code and runtime */
|
|
int recomp_save_init(GuestContext* c, const char* exe_path);
|
|
int recomp_save_write(GuestContext* c, const char* name, const void* data, uint64_t size);
|
|
int recomp_save_read(GuestContext* c, const char* name, void* buf, uint64_t buf_size, uint64_t* out_size);
|
|
int recomp_save_delete(GuestContext* c, const char* name);
|
|
int recomp_save_exists(GuestContext* c, const char* name);
|
|
/* Load bundled data segments into guest memory */
|
|
int recomp_load_segments(GuestContext* c, const char* data_dir);
|
|
#endif
|
|
)RT";
|
|
if (fastmem || fpx) {
|
|
// The ABI line stays literal above; ABI 6 replaces it with its own block.
|
|
static constexpr std::string_view abi5_line = "#define RECOMP_IMAGE_ABI 5\n";
|
|
text.replace(text.find(abi5_line), abi5_line.size(),
|
|
std::string("#define RECOMP_IMAGE_ABI ") + (fastmem ? FastmemAbiH() : FpxAbiH()));
|
|
}
|
|
if (fpx) {
|
|
// After GuestContext, before the include guard closes.
|
|
static constexpr std::string_view guard_end = "#endif\n";
|
|
const size_t at = text.rfind(guard_end);
|
|
text.insert(at, std::string(FpxH()) + FpxH64() + (fpx == 1 ? FpxExactH() : "") +
|
|
(fpx == 2 ? "/* FPX1 shadow instrumentation build (never timed). */\n"
|
|
"void recomp_fpx_shadow(unsigned kind, int gate, int kept, uint64_t fast,\n"
|
|
" uint64_t exact, uint64_t fpsr_before, uint64_t fpsr_after,\n"
|
|
" uint64_t a, uint64_t b);\n"
|
|
: ""));
|
|
}
|
|
if (fastmem && guard_gen) {
|
|
static constexpr std::string_view anchor = "void recomp_ic_ivau(GuestContext*,uint64_t,int);\n";
|
|
text.insert(text.find(anchor) + anchor.size(), GuardGenH());
|
|
}
|
|
return text;
|
|
}
|
|
|
|
// One cached text per variant: the first caller must not fix the variant for
|
|
// the whole process.
|
|
inline int FpxVariant() {
|
|
return g_emit_fpx ? (g_emit_fpx_shadow ? 2 : 1) : 0;
|
|
}
|
|
inline const char* RuntimeH() {
|
|
// [fastmem][guard_gen][fpx variant: 0 off, 1 FPX1, 2 FPX1+shadow].
|
|
static const std::string texts[2][2][3] = {
|
|
{{BuildRuntimeH(false, false, 0), BuildRuntimeH(false, false, 1), BuildRuntimeH(false, false, 2)},
|
|
{BuildRuntimeH(false, true, 0), BuildRuntimeH(false, true, 1), BuildRuntimeH(false, true, 2)}},
|
|
{{BuildRuntimeH(true, false, 0), BuildRuntimeH(true, false, 1), BuildRuntimeH(true, false, 2)},
|
|
{BuildRuntimeH(true, true, 0), BuildRuntimeH(true, true, 1), BuildRuntimeH(true, true, 2)}}};
|
|
return texts[g_emit_fastmem ? 1 : 0][EmitGuardGen() ? 1 : 0][FpxVariant()].c_str();
|
|
}
|
|
|
|
inline const char* EstimateRuntimeC() {
|
|
return R"EST(/* Isolated Armv8 S/D reciprocal-estimate prototype. Integer-only lane core.
|
|
* Exception flags accumulate in *fpsr. Exception traps are not delivered here.
|
|
* Interfaces deliberately do not depend on GuestContext or shared emitter code.
|
|
*/
|
|
|
|
/* the 8-bit estimates by their table index (recip: the 9-bit scaled significand, rsqrt: the 10-bit a below), each
|
|
computed once by the formula and kept as estimate+1 (0: not yet; threads racing write the same value) */
|
|
static uint16_t recomp_est_tab[2][1024];
|
|
uint64_t recomp_fp_estimate(GuestContext* c,uint64_t bits,unsigned width,int rsqrt){
|
|
uint32_t fpcr=(uint32_t)c->fpcr;
|
|
uint64_t *fpsr=&c->fpsr;
|
|
const unsigned fracbits=width==32?23:52;
|
|
const unsigned expbits=width==32?8:11;
|
|
const int bias=width==32?127:1023,emin=1-bias;
|
|
const uint64_t hidden=UINT64_C(1)<<fracbits;
|
|
const uint64_t fracmask=hidden-1,expmask=((UINT64_C(1)<<expbits)-1)<<fracbits;
|
|
const uint64_t sign=bits&(UINT64_C(1)<<(width-1));
|
|
const uint64_t quiet=hidden>>1,defnan=expmask|quiet;
|
|
uint64_t frac=bits&fracmask;
|
|
unsigned rawexp=(unsigned)((bits&expmask)>>fracbits);
|
|
if(rawexp==((1u<<expbits)-1)&&frac){
|
|
if(!(frac&quiet))*fpsr|=1;
|
|
return(fpcr&(1u<<25))?defnan:(bits|quiet);
|
|
}
|
|
if(!rawexp&&frac&&(fpcr&(1u<<24))){*fpsr|=128;frac=0;}
|
|
if(!rawexp&&!frac){*fpsr|=2;return sign|expmask;}
|
|
if(rsqrt&&sign){*fpsr|=1;return defnan;}
|
|
if(rawexp==((1u<<expbits)-1))return sign;
|
|
int exponent=rawexp?(int)rawexp-bias:emin;
|
|
uint64_t mant=rawexp?(hidden|frac):frac;
|
|
while(mant<hidden){mant<<=1;exponent--;}
|
|
if(!rsqrt){
|
|
if(exponent<emin-2){
|
|
unsigned rm=(fpcr>>22)&3;
|
|
int inf=rm==0||(rm==1&&!sign)||(rm==2&&sign);
|
|
*fpsr|=4|16;return sign|(inf?expmask:(expmask-1));
|
|
}
|
|
if((fpcr&(1u<<24))&&exponent>=-emin){*fpsr|=8;return sign;}
|
|
uint64_t scaled=mant>>(fracbits-8);
|
|
uint64_t estimate;
|
|
if(recomp_est_tab[0][scaled&1023])estimate=recomp_est_tab[0][scaled&1023]-1u;
|
|
else{estimate=(((UINT64_C(1)<<19)/(2*scaled+1)+1)/2)&255;recomp_est_tab[0][scaled&1023]=(uint16_t)(estimate+1);}
|
|
int out_exp=-exponent-1;
|
|
uint64_t sig=(256+estimate)<<(fracbits-8);
|
|
if(out_exp<emin)return sign|(sig>>(emin-out_exp));
|
|
return sign|((uint64_t)(out_exp+bias)<<fracbits)|(estimate<<(fracbits-8));
|
|
}
|
|
uint64_t a=mant>>(fracbits-((exponent%2==0)?7:8));
|
|
a=a<256?2*a+1:2*(a|1);
|
|
uint64_t estimate;
|
|
if(recomp_est_tab[1][a&1023])estimate=recomp_est_tab[1][a&1023]-1u;
|
|
else{
|
|
unsigned lo=512,hi=1024;
|
|
while(lo+1<hi){unsigned mid=(lo+hi)/2;if(a*mid*mid<(UINT64_C(1)<<28))lo=mid;else hi=mid;}
|
|
estimate=((lo+1)/2)&255;
|
|
recomp_est_tab[1][a&1023]=(uint16_t)(estimate+1);
|
|
}
|
|
int neg=-exponent-1;
|
|
int out_exp=neg>=0?neg/2:-((-neg+1)/2);
|
|
return ((uint64_t)(out_exp+bias)<<fracbits)|(estimate<<(fracbits-8));
|
|
}
|
|
|
|
)EST";
|
|
}
|
|
|
|
// ABI 6 (FM1) memory helpers. `abi5` is the ABI 5 helper text. It is kept
|
|
// verbatim as the slow path, with only the twelve public helpers renamed to
|
|
// recomp_*_slow (their calls to one another included) and made static and
|
|
// noinline. Deriving it from the ABI 5 text rather than restating it means the
|
|
// slow path cannot drift from what ABI 5 does.
|
|
inline std::string FastmemHelpersC(std::string abi5, bool guard_gen = false) {
|
|
static constexpr const char* kHelpers[] = {
|
|
"load8", "load16", "load32", "load64", "store8", "store16",
|
|
"store32", "store64", "ldp64", "stp64", "ldp32", "stp32",
|
|
};
|
|
const auto replace_all = [&abi5](const std::string& from, const std::string& to) {
|
|
for (size_t at = abi5.find(from); at != std::string::npos;
|
|
at = abi5.find(from, at + to.size())) {
|
|
abi5.replace(at, from.size(), to);
|
|
}
|
|
};
|
|
for (const char* name : kHelpers) {
|
|
const std::string base = std::string("recomp_") + name;
|
|
replace_all(base + " (", base + "_slow(");
|
|
replace_all(base + "(", base + "_slow(");
|
|
}
|
|
replace_all("\nuint64_t recomp_", "\nstatic RECOMP_NOINLINE uint64_t recomp_");
|
|
replace_all("\nvoid recomp_", "\nstatic RECOMP_NOINLINE void recomp_");
|
|
std::string text = abi5 + R"RT(
|
|
/* ABI 6 (FM1) fast path. One read of the same page-table entry the ABI 5 walk
|
|
would read, for exactly the accesses ABI 5 would serve from that entry:
|
|
in range, inside one page, and backed by a real pointer. It returns the
|
|
entry's pointer part (host page - guest page), or 0 to decline. Declining is
|
|
always safe, because the caller then runs the unchanged ABI 5 helper above.
|
|
|
|
fm_limit is page aligned, no higher than address_space_max and at most 2^39,
|
|
so one compare covers the disabled state (0), every bound ABI 5 checks, and
|
|
any tagged or out-of-range address, which ABI 5 then masks and serves as
|
|
before. Unmapped, debug and GPU-tracked pages have a null pointer part, so
|
|
they decline too and still reach the host callback, in the same order and
|
|
with the same arguments. */
|
|
static RECOMP_INLINE uintptr_t recomp_fm_entry(const GuestContext* c, uint64_t va, unsigned bytes){
|
|
if(va >= c->fm_limit) return 0;
|
|
if((va & 0xfffu) > 0x1000u - bytes) return 0;
|
|
return *(const uintptr_t*)(c->fm_table + ((uintptr_t)(va >> RECOMP_FM_PAGE_BITS)
|
|
<< RECOMP_FM_STRIDE_LOG2)) & RECOMP_FM_PTR_MASK;
|
|
}
|
|
#define RECOMP_FM_PTR(e,a) ((unsigned char*)((e) + (uintptr_t)(a)))
|
|
|
|
uint64_t recomp_load8 (GuestContext* c,uint64_t a){
|
|
uintptr_t e=recomp_fm_entry(c,a,1);
|
|
if(RECOMP_LIKELY(e)) return (uint64_t)*RECOMP_FM_PTR(e,a);
|
|
return recomp_load8_slow(c,a);}
|
|
uint64_t recomp_load16(GuestContext* c,uint64_t a){
|
|
uintptr_t e=recomp_fm_entry(c,a,2);
|
|
if(RECOMP_LIKELY(e)){uint16_t v;memcpy(&v,RECOMP_FM_PTR(e,a),2);return (uint64_t)v;}
|
|
return recomp_load16_slow(c,a);}
|
|
uint64_t recomp_load32(GuestContext* c,uint64_t a){
|
|
uintptr_t e=recomp_fm_entry(c,a,4);
|
|
if(RECOMP_LIKELY(e)){uint32_t v;memcpy(&v,RECOMP_FM_PTR(e,a),4);return (uint64_t)v;}
|
|
return recomp_load32_slow(c,a);}
|
|
uint64_t recomp_load64(GuestContext* c,uint64_t a){
|
|
uintptr_t e=recomp_fm_entry(c,a,8);
|
|
if(RECOMP_LIKELY(e)){uint64_t v;memcpy(&v,RECOMP_FM_PTR(e,a),8);return v;}
|
|
return recomp_load64_slow(c,a);}
|
|
void recomp_store8 (GuestContext* c,uint64_t a,uint64_t v){
|
|
uintptr_t e=recomp_fm_entry(c,a,1);
|
|
if(RECOMP_LIKELY(e)){*RECOMP_FM_PTR(e,a)=(unsigned char)v;return;}
|
|
recomp_store8_slow(c,a,v);}
|
|
void recomp_store16(GuestContext* c,uint64_t a,uint64_t v){
|
|
uintptr_t e=recomp_fm_entry(c,a,2);
|
|
if(RECOMP_LIKELY(e)){uint16_t t=(uint16_t)v;memcpy(RECOMP_FM_PTR(e,a),&t,2);return;}
|
|
recomp_store16_slow(c,a,v);}
|
|
void recomp_store32(GuestContext* c,uint64_t a,uint64_t v){
|
|
uintptr_t e=recomp_fm_entry(c,a,4);
|
|
if(RECOMP_LIKELY(e)){uint32_t t=(uint32_t)v;memcpy(RECOMP_FM_PTR(e,a),&t,4);return;}
|
|
recomp_store32_slow(c,a,v);}
|
|
void recomp_store64(GuestContext* c,uint64_t a,uint64_t v){
|
|
uintptr_t e=recomp_fm_entry(c,a,8);
|
|
if(RECOMP_LIKELY(e)){memcpy(RECOMP_FM_PTR(e,a),&v,8);return;}
|
|
recomp_store64_slow(c,a,v);}
|
|
void recomp_ldp64(GuestContext* c,uint64_t a,uint64_t* lo,uint64_t* hi){
|
|
uintptr_t e=recomp_fm_entry(c,a,16);
|
|
if(RECOMP_LIKELY(e)){const unsigned char* p=RECOMP_FM_PTR(e,a);
|
|
memcpy(lo,p,8); memcpy(hi,p+8,8); return;}
|
|
recomp_ldp64_slow(c,a,lo,hi);}
|
|
void recomp_stp64(GuestContext* c,uint64_t a,uint64_t v0,uint64_t v1){
|
|
uintptr_t e=recomp_fm_entry(c,a,16);
|
|
if(RECOMP_LIKELY(e)){unsigned char* p=RECOMP_FM_PTR(e,a);
|
|
memcpy(p,&v0,8); memcpy(p+8,&v1,8); return;}
|
|
recomp_stp64_slow(c,a,v0,v1);}
|
|
void recomp_ldp32(GuestContext* c,uint64_t a,uint64_t* lo,uint64_t* hi){
|
|
uintptr_t e=recomp_fm_entry(c,a,8);
|
|
if(RECOMP_LIKELY(e)){const unsigned char* p=RECOMP_FM_PTR(e,a); uint32_t x,y;
|
|
memcpy(&x,p,4); memcpy(&y,p+4,4); *lo=(uint64_t)x; *hi=(uint64_t)y; return;}
|
|
recomp_ldp32_slow(c,a,lo,hi);}
|
|
void recomp_stp32(GuestContext* c,uint64_t a,uint64_t v0,uint64_t v1){
|
|
uintptr_t e=recomp_fm_entry(c,a,8);
|
|
if(RECOMP_LIKELY(e)){unsigned char* p=RECOMP_FM_PTR(e,a); uint32_t x=(uint32_t)v0,y=(uint32_t)v1;
|
|
memcpy(p,&x,4); memcpy(p+4,&y,4); return;}
|
|
recomp_stp32_slow(c,a,v0,v1);}
|
|
)RT";
|
|
if (!guard_gen) {
|
|
return text;
|
|
}
|
|
// GG1: stores must not write a watched code page behind the host's back.
|
|
// The fast path also reads the entry's watch word (same cache line) and
|
|
// declines when it is set; the store then goes to the host callback, which
|
|
// writes through Core::Memory and so moves the generation. Stores the fast
|
|
// path declines for other reasons check the pages they touch the same way
|
|
// before taking the unchanged ABI 5 walk, which would write directly.
|
|
// Loads, and the code guard's own reads, are unchanged.
|
|
const auto replace_once = [&text](const std::string& from, const std::string& to) {
|
|
const size_t at = text.find(from);
|
|
text.replace(at, from.size(), to);
|
|
};
|
|
replace_once("#define RECOMP_FM_PTR(e,a) ((unsigned char*)((e) + (uintptr_t)(a)))\n",
|
|
R"RT(#define RECOMP_FM_PTR(e,a) ((unsigned char*)((e) + (uintptr_t)(a)))
|
|
static RECOMP_INLINE uintptr_t recomp_fm_entry_st(const GuestContext* c, uint64_t va, unsigned bytes){
|
|
const unsigned char* ent;
|
|
if(va >= c->fm_limit) return 0;
|
|
if((va & 0xfffu) > 0x1000u - bytes) return 0;
|
|
ent = c->fm_table + ((uintptr_t)(va >> RECOMP_FM_PAGE_BITS) << RECOMP_FM_STRIDE_LOG2);
|
|
if(*(const volatile uint64_t*)(ent + RECOMP_GG_WATCH_OFFSET)) return 0;
|
|
return *(const uintptr_t*)ent & RECOMP_FM_PTR_MASK;
|
|
}
|
|
/* Whether any page of [va, va + bytes) is watched. Addresses the ABI 5 walk
|
|
would not serve from the table (outside the space, or wrapping) reach the
|
|
host callback anyway, so they need no answer here. */
|
|
static RECOMP_NOINLINE int recomp_gg_store_watched(GuestContext* c, uint64_t va, uint64_t bytes){
|
|
const RecompHostMem* hm = c->host_mem;
|
|
uint64_t page, last;
|
|
if(!hm || !hm->page_entries || hm->page_bits >= 64) return 0;
|
|
va &= 0xffffffffffffULL;
|
|
if(va >= hm->address_space_max) return 0;
|
|
last = bytes - 1 > hm->address_space_max - 1 - va ? hm->address_space_max - 1 : va + bytes - 1;
|
|
for(page = va >> hm->page_bits; page <= last >> hm->page_bits; ++page){
|
|
if(*(const volatile uint64_t*)((const unsigned char*)hm->page_entries +
|
|
page * hm->page_entry_stride + RECOMP_GG_WATCH_OFFSET)) return 1;
|
|
}
|
|
return 0;
|
|
}
|
|
)RT");
|
|
static constexpr struct {
|
|
const char* head;
|
|
const char* slow;
|
|
const char* watched;
|
|
} kStores[] = {
|
|
{"void recomp_store8 (GuestContext* c,uint64_t a,uint64_t v){\n", " recomp_store8_slow(c,a,v);}",
|
|
" if(recomp_gg_store_watched(c,a,1)){memstore(c,a,1,v);return;}\n"},
|
|
{"void recomp_store16(GuestContext* c,uint64_t a,uint64_t v){\n", " recomp_store16_slow(c,a,v);}",
|
|
" if(recomp_gg_store_watched(c,a,2)){memstore(c,a,2,v);return;}\n"},
|
|
{"void recomp_store32(GuestContext* c,uint64_t a,uint64_t v){\n", " recomp_store32_slow(c,a,v);}",
|
|
" if(recomp_gg_store_watched(c,a,4)){memstore(c,a,4,v);return;}\n"},
|
|
{"void recomp_store64(GuestContext* c,uint64_t a,uint64_t v){\n", " recomp_store64_slow(c,a,v);}",
|
|
" if(recomp_gg_store_watched(c,a,8)){memstore(c,a,8,v);return;}\n"},
|
|
{"void recomp_stp64(GuestContext* c,uint64_t a,uint64_t v0,uint64_t v1){\n",
|
|
" recomp_stp64_slow(c,a,v0,v1);}",
|
|
" if(recomp_gg_store_watched(c,a,16)){memstore(c,a,8,v0);memstore(c,a+8,8,v1);return;}\n"},
|
|
{"void recomp_stp32(GuestContext* c,uint64_t a,uint64_t v0,uint64_t v1){\n",
|
|
" recomp_stp32_slow(c,a,v0,v1);}",
|
|
" if(recomp_gg_store_watched(c,a,8)){memstore(c,a,4,v0);memstore(c,a+4,4,v1);return;}\n"},
|
|
};
|
|
for (const auto& store : kStores) {
|
|
const size_t at = text.find(store.head);
|
|
const size_t e = text.find("recomp_fm_entry(", at);
|
|
text.replace(e, std::strlen("recomp_fm_entry("), "recomp_fm_entry_st(");
|
|
const size_t slow = text.find(store.slow, at);
|
|
text.insert(slow, store.watched);
|
|
}
|
|
return text;
|
|
}
|
|
|
|
// ABI 6 feature GG1: the verification a block runs when its generation moved.
|
|
inline const char* GuardGenC() {
|
|
return R"RT(/* GG1: the block saw its seen word differ from the generation `gen` it
|
|
loaded. The acquire fence pairs with the host's release store of `gen`, so
|
|
the check below reads code and mappings no older than the event that
|
|
produced it. The check itself is the unchanged per-entry guard, with the
|
|
same strict rejection or hybrid JIT handoff. Only after it passes does the
|
|
seen word take `gen`, with release so another core that skips on it cannot get ahead of
|
|
this check's reads. VERIFY_ALWAYS is never recorded. */
|
|
void recomp_code_guard_gen(GuestContext* c,uint64_t pc,const uint32_t* expected,uint32_t count,
|
|
int host_guard_version,uint32_t* seen,uint32_t gen){
|
|
RECOMP_GG_ACQUIRE_FENCE();
|
|
recomp_code_guard(c,pc,expected,count,host_guard_version);
|
|
if(c->halted) return;
|
|
if(gen!=RECOMP_GG_VERIFY_ALWAYS) RECOMP_GG_STORE_RELEASE(*seen,gen);
|
|
}
|
|
)RT";
|
|
}
|
|
|
|
inline std::string BuildRuntimeC(bool fastmem, bool guard_gen = false) {
|
|
// MSVC caps one string literal at 16380 bytes (C2026) and this runtime is
|
|
// past that, so it is assembled from several pieces at first use rather
|
|
// than being a single literal. Adjacent-literal concatenation would not
|
|
// help: the limit applies to the result as well. The memory helpers are a
|
|
// piece of their own because ABI 6 reuses their exact text as its slow path.
|
|
const std::string head = std::string(R"RT(#include "recomp_runtime.h"
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
/* clock()/CLOCKS_PER_SEC for the standalone recomp_cntpct fallback. */
|
|
#include <time.h>
|
|
#ifdef _WIN32
|
|
#include <windows.h>
|
|
#include <direct.h>
|
|
#include <io.h>
|
|
#define MKDIR(p) _mkdir(p)
|
|
#define PATH_SEP '\\'
|
|
#else
|
|
#include <sys/stat.h>
|
|
#include <unistd.h>
|
|
#include <time.h>
|
|
#define MKDIR(p) mkdir(p,0755)
|
|
#define PATH_SEP '/'
|
|
#endif
|
|
|
|
/* AES S-box, built once on first use.
|
|
S(x) = affine(x^-1), and x^-1 is x^254 because x^255 == 1 for non-zero x.
|
|
Generated rather than tabulated: two 256-byte tables written out as source
|
|
would be 512 bytes of literal in a runtime already split to stay under
|
|
MSVC's 16380-byte cap. Checked against FIPS 197 and for round-trip. */
|
|
static uint8_t recomp_gmul(uint8_t a, uint8_t b){
|
|
uint8_t p=0;
|
|
while(b){ if(b&1) p^=a; a=(uint8_t)((a<<1)^((a>>7)*0x1B)); b=(uint8_t)(b>>1); }
|
|
return p;
|
|
}
|
|
static uint8_t recomp_ginv(uint8_t x){
|
|
uint8_t p=1; int b;
|
|
if(!x) return 0;
|
|
for(b=7;b>=0;b--){ p=recomp_gmul(p,p); if((254>>b)&1) p=recomp_gmul(p,x); }
|
|
return p;
|
|
}
|
|
static uint8_t recomp_rotl8(uint8_t x,int n){ return (uint8_t)((x<<n)|(x>>(8-n))); }
|
|
const uint8_t* recomp_aes_sbox(int inverse){
|
|
static uint8_t fwd[256], inv[256];
|
|
static int built = 0;
|
|
if(!built){
|
|
int i;
|
|
for(i=0;i<256;i++){
|
|
uint8_t b = recomp_ginv((uint8_t)i);
|
|
fwd[i] = (uint8_t)(b ^ recomp_rotl8(b,1) ^ recomp_rotl8(b,2) ^ recomp_rotl8(b,3)
|
|
^ recomp_rotl8(b,4) ^ 0x63);
|
|
}
|
|
for(i=0;i<256;i++){
|
|
uint8_t b = (uint8_t)(recomp_rotl8((uint8_t)i,1) ^ recomp_rotl8((uint8_t)i,3)
|
|
^ recomp_rotl8((uint8_t)i,6) ^ 0x05);
|
|
inv[i] = recomp_ginv(b);
|
|
}
|
|
built = 1;
|
|
}
|
|
return inverse ? inv : fwd;
|
|
}
|
|
static uint8_t* memptr(GuestContext* c, uint64_t va, uint64_t sz){
|
|
uint64_t off=va-c->mem_base_vaddr;
|
|
if(off+sz>c->mem_size) return 0;
|
|
return c->mem+off;
|
|
}
|
|
|
|
/* Every guest access funnels through these two, so the host bridge only has to
|
|
be checked in one place per direction. */
|
|
static uint64_t memload(GuestContext* c, uint64_t a, uint32_t sz){
|
|
if(c->host_mem) return c->host_mem->load(c->host_mem->user,a,sz);
|
|
{ uint8_t* p=memptr(c,a,sz); uint64_t v=0; if(p)memcpy(&v,p,sz); return v; }
|
|
}
|
|
static void memstore(GuestContext* c, uint64_t a, uint32_t sz, uint64_t v){
|
|
if(c->host_mem){ c->host_mem->store(c->host_mem->user,a,sz,v); return; }
|
|
{ uint8_t* p=memptr(c,a,sz); if(p)memcpy(p,&v,sz); }
|
|
}
|
|
|
|
static unsigned char* recomp_host_ptr(GuestContext* c, uint64_t va);
|
|
static unsigned char* recomp_host_ptr_n(GuestContext* c, uint64_t va, uint64_t bytes);
|
|
|
|
/* Every generated block checks its compilation input before any guest effect.
|
|
Normal mapped code is compared directly through the host page table. The
|
|
older checked callback remains the fallback for a page crossing or any
|
|
special mapping, and also pinpoints a mismatching word for the diagnostic.
|
|
This guards synchronized instruction fetch, not unsynchronized concurrent
|
|
mutation of an already executing block. Failure must never enter a JIT.
|
|
|
|
This runs on entry to every block, so the per-word loop below costs one
|
|
indirect host callback per guest instruction before any guest work happens -
|
|
the largest single cost in the profile. Where the block's words lie in mapped
|
|
pages with real backing pointers, the identical comparison is one memcmp per
|
|
page, so take that. A block that crosses a page boundary is compared as its
|
|
in-page pieces; sending it to the loop instead cost one callback per word on
|
|
every entry. It is a faster spelling of the same check, not a weaker one: the
|
|
same words are compared, and every case the fast path cannot prove safe
|
|
(unmapped, debug or GPU-tracked memory, an address outside the space, a
|
|
mismatch) falls through to the loop, which is also what produces the precise
|
|
diagnostic naming the offending word. */
|
|
void recomp_code_guard(GuestContext* c,uint64_t pc,const uint32_t* expected,uint32_t count,int host_guard_version){
|
|
uint32_t k;
|
|
if(c->host_mem && host_guard_version!=2){
|
|
c->pc=pc;
|
|
fprintf(stderr,"[recomp] code guard requires a guard-v2 host and guarded modules at 0x%llx\n",(unsigned long long)pc);
|
|
fflush(stderr); abort();
|
|
}
|
|
if(c->host_mem && c->host_mem->page_entries && c->host_mem->page_bits<63 && count){
|
|
uint64_t page_size=UINT64_C(1)<<c->host_mem->page_bits;
|
|
uint64_t bytes=(uint64_t)count*4, done=0;
|
|
if(pc<=UINT64_C(0xffffffffffff) && bytes<=UINT64_C(0xffffffffffff)-pc){
|
|
while(done<bytes){
|
|
uint64_t va=pc+done, room=page_size-(va&(page_size-1));
|
|
uint64_t n=bytes-done<room?bytes-done:room;
|
|
const unsigned char* p=recomp_host_ptr_n(c,va,n);
|
|
if(!p || memcmp(p,(const unsigned char*)expected+done,(size_t)n)!=0)break;
|
|
done+=n;
|
|
}
|
|
if(done==bytes)return;
|
|
}
|
|
}
|
|
for(k=0;k<count;++k){
|
|
uint64_t va=pc+(uint64_t)k*4, actual=0;
|
|
int valid=va>=pc;
|
|
if(c->host_mem){
|
|
/* Guard-v2 reserved load size 0: bit32 is explicit mapping validity,
|
|
independently of the low32 instruction word (which may be zero). */
|
|
if(valid){
|
|
actual=c->host_mem->load(c->host_mem->user,va,0);
|
|
valid=(actual&UINT64_C(0x100000000))!=0;
|
|
}
|
|
}else{
|
|
uint64_t off=va-c->mem_base_vaddr;
|
|
valid=valid && c->mem && va>=c->mem_base_vaddr && off<=c->mem_size && c->mem_size-off>=4;
|
|
if(valid) memcpy(&actual,c->mem+off,4);
|
|
}
|
|
if(!valid || (uint32_t)actual!=expected[k]){
|
|
#ifdef SUYU_HOSTED_RECOMP
|
|
/* Hybrid hosts already understand halted=2 as a request to execute the
|
|
current PC on the JIT. Keep the original entry PC: no instruction of
|
|
this block has run, including when only a later word changed. */
|
|
const char* strict=getenv("SUYU_RECOMP_STRICT");
|
|
if(strict && *strict=='0'){
|
|
c->halted=RECOMP_HALT_UNHANDLED;
|
|
c->pending_svc=~UINT64_C(0);
|
|
return;
|
|
}
|
|
#endif
|
|
c->pc=va;
|
|
fprintf(stderr,"[recomp] unsupported code change or unavailable code at 0x%llx (expected %08x, read %08x)\n",
|
|
(unsigned long long)va,expected[k],(unsigned)(uint32_t)actual);
|
|
fflush(stderr); abort();
|
|
}
|
|
}
|
|
}
|
|
void recomp_ic_ivau(GuestContext* c,uint64_t address,int host_guard_version){
|
|
recomp_barrier();
|
|
if(c->host_mem){
|
|
if(host_guard_version!=2){
|
|
fprintf(stderr,"[recomp] IC IVAU requires a guard-v2 host and guarded modules; refusing at 0x%llx\n",(unsigned long long)c->pc);
|
|
fflush(stderr); abort();
|
|
}
|
|
c->pending_svc=address&~UINT64_C(63); /* payload tagged by halted, never an SVC */
|
|
c->halted=RECOMP_HALT_IC_IVAU;
|
|
}else{
|
|
c->pending_svc=~UINT64_C(0);
|
|
c->pc+=4; /* no fallback cache; the next block still verifies its bytes */
|
|
}
|
|
}
|
|
|
|
/* Exclusive access.
|
|
|
|
Hosted, these go straight to the emulator's exclusive monitor - the same one
|
|
the fallback JIT uses - so a lock taken by recompiled code is visible to a
|
|
thread running on the JIT and vice versa.
|
|
|
|
Standalone there is exactly one guest thread and nothing else can touch the
|
|
address, so a single-slot address latch is not an approximation: it gives the
|
|
same answers a real monitor would. It still tracks the address rather than
|
|
always succeeding, so code that deliberately fails an STXR (the usual
|
|
compare-and-swap retry loop) still behaves. */
|
|
static uint64_t g_excl_addr = 0;
|
|
static int g_excl_valid = 0;
|
|
|
|
uint64_t recomp_ldxr(GuestContext* c, uint64_t addr, uint32_t size){
|
|
if(c->host_mem && c->host_mem->excl_load) return c->host_mem->excl_load(c->host_mem->user,addr,size);
|
|
g_excl_addr = addr; g_excl_valid = 1;
|
|
return memload(c,addr,size);
|
|
}
|
|
|
|
uint32_t recomp_stxr(GuestContext* c, uint64_t addr, uint32_t size, uint64_t value){
|
|
if(c->host_mem && c->host_mem->excl_store) return c->host_mem->excl_store(c->host_mem->user,addr,size,value);
|
|
if(!g_excl_valid || g_excl_addr != addr) return 1;
|
|
g_excl_valid = 0;
|
|
memstore(c,addr,size,value);
|
|
return 0;
|
|
}
|
|
|
|
void recomp_ldxp(GuestContext* c, uint64_t addr, uint32_t size, uint64_t* lo, uint64_t* hi){
|
|
if(c->host_mem && c->host_mem->excl_load_pair){
|
|
c->host_mem->excl_load_pair(c->host_mem->user,addr,size,lo,hi); return;
|
|
}
|
|
g_excl_addr = addr; g_excl_valid = 1;
|
|
*lo = memload(c,addr,size);
|
|
*hi = memload(c,addr+size,size);
|
|
}
|
|
|
|
uint32_t recomp_stxp(GuestContext* c, uint64_t addr, uint32_t size, uint64_t lo, uint64_t hi){
|
|
if(c->host_mem && c->host_mem->excl_store_pair)
|
|
return c->host_mem->excl_store_pair(c->host_mem->user,addr,size,lo,hi);
|
|
if(!g_excl_valid || g_excl_addr != addr) return 1;
|
|
g_excl_valid = 0;
|
|
memstore(c,addr,size,lo);
|
|
memstore(c,addr+size,size,hi);
|
|
return 0;
|
|
}
|
|
|
|
void recomp_clrex(GuestContext* c){
|
|
if(c->host_mem && c->host_mem->clear_excl){ c->host_mem->clear_excl(c->host_mem->user); return; }
|
|
g_excl_valid = 0;
|
|
}
|
|
|
|
uint64_t recomp_cntpct(GuestContext* c){
|
|
if(c->host_mem && c->host_mem->read_cntpct) return c->host_mem->read_cntpct(c->host_mem->user);
|
|
/* Standalone: derive from the host clock at the guest's 19.2 MHz rate. A
|
|
free-running counter would satisfy code that only measures deltas, but
|
|
anything converting ticks to seconds would then be wrong by whatever the
|
|
host happens to run at. */
|
|
{
|
|
static uint64_t base_ns = 0;
|
|
uint64_t now_ns = (uint64_t)(clock() * (1000000000ULL / CLOCKS_PER_SEC));
|
|
if(!base_ns) base_ns = now_ns;
|
|
return ((now_ns - base_ns) / 1000ULL) * 19200ULL / 1000ULL;
|
|
}
|
|
}
|
|
)RT");
|
|
const std::string abi5_helpers = R"RT(
|
|
/* Resolve a guest address to a host pointer the way Memory::GetPointerImpl
|
|
does: mask, bounds check, one page-table entry, extract the backing pointer.
|
|
A null result means unmapped, debug, or GPU-tracked memory, all of which have
|
|
to go through the emulator callback so the rasterizer is told about the
|
|
access. Only a real backing pointer is handled here.
|
|
|
|
Deliberately not inlined into the generated code. Forcing it inline at every
|
|
access site was measured: main.dll went from 100 MB to 222 MB and the race
|
|
phase lost 14%, so whatever the call cost, the instruction cache cost more. */
|
|
static unsigned char* recomp_host_ptr(GuestContext* c, uint64_t va){
|
|
const RecompHostMem* hm = c->host_mem;
|
|
uintptr_t raw, p;
|
|
if(!hm || !hm->page_entries || hm->page_bits >= 64) return 0;
|
|
va &= 0xffffffffffffULL; /* AArch64 ignores the top 16 bits */
|
|
if(va >= hm->address_space_max) return 0;
|
|
raw = *(const uintptr_t*)((const unsigned char*)hm->page_entries
|
|
+ (va >> hm->page_bits) * hm->page_entry_stride);
|
|
p = raw & (uintptr_t)hm->pointer_mask;
|
|
return p ? (unsigned char*)(p + (uintptr_t)va) : 0;
|
|
}
|
|
|
|
/* A multi-byte access whose bytes straddle a page boundary cannot be served
|
|
from one page-table entry. The walk resolves the entry for the first byte and
|
|
returns p + va; the next guest page need not sit next to this one in host
|
|
memory, so the tail bytes would be read from whatever follows this page's
|
|
mapping rather than from the next guest page. The pair helpers below already
|
|
refuse to cross. These did not, and an unaligned stream crosses regularly - a
|
|
32-bit load advancing two bytes at a time through a compressed archive is the
|
|
pattern that finds it. Hand the crossing case to the emulator, which splits
|
|
it correctly.
|
|
|
|
Even aligned accesses must respect an address-space limit inside a page.
|
|
Subtraction-based checks avoid overflowing the end address. */
|
|
static unsigned char* recomp_host_ptr_n(GuestContext* c, uint64_t va, uint64_t bytes){
|
|
const RecompHostMem* hm = c->host_mem;
|
|
uint64_t psz;
|
|
if(!hm || !hm->page_entries || hm->page_bits >= 64 || !bytes) return 0;
|
|
va &= 0xffffffffffffULL;
|
|
/* The fast-path limit can be below a page boundary (diagnostic slow paths).
|
|
Validate the entire span before reading any entry or backing bytes. */
|
|
if(va >= hm->address_space_max || bytes > hm->address_space_max - va ||
|
|
bytes > UINT64_C(0x1000000000000) - va) return 0;
|
|
psz = UINT64_C(1) << hm->page_bits;
|
|
if(bytes > psz - (va & (psz - 1))) return 0;
|
|
return recomp_host_ptr(c, va);
|
|
}
|
|
|
|
uint64_t recomp_load8 (GuestContext* c,uint64_t a){
|
|
unsigned char* p=recomp_host_ptr(c,a); if(p) return (uint64_t)*p; return memload(c,a,1);}
|
|
uint64_t recomp_load16(GuestContext* c,uint64_t a){
|
|
unsigned char* p=recomp_host_ptr_n(c,a,2); if(p){uint16_t v;memcpy(&v,p,2);return (uint64_t)v;} return memload(c,a,2);}
|
|
uint64_t recomp_load32(GuestContext* c,uint64_t a){
|
|
unsigned char* p=recomp_host_ptr_n(c,a,4); if(p){uint32_t v;memcpy(&v,p,4);return (uint64_t)v;} return memload(c,a,4);}
|
|
uint64_t recomp_load64(GuestContext* c,uint64_t a){
|
|
unsigned char* p=recomp_host_ptr_n(c,a,8); if(p){uint64_t v;memcpy(&v,p,8);return v;} return memload(c,a,8);}
|
|
void recomp_store8 (GuestContext* c,uint64_t a,uint64_t v){
|
|
unsigned char* p=recomp_host_ptr(c,a); if(p){*p=(unsigned char)v;return;} memstore(c,a,1,v);}
|
|
void recomp_store16(GuestContext* c,uint64_t a,uint64_t v){
|
|
unsigned char* p=recomp_host_ptr_n(c,a,2); if(p){uint16_t t=(uint16_t)v;memcpy(p,&t,2);return;} memstore(c,a,2,v);}
|
|
void recomp_store32(GuestContext* c,uint64_t a,uint64_t v){
|
|
unsigned char* p=recomp_host_ptr_n(c,a,4); if(p){uint32_t t=(uint32_t)v;memcpy(p,&t,4);return;} memstore(c,a,4,v);}
|
|
void recomp_store64(GuestContext* c,uint64_t a,uint64_t v){
|
|
unsigned char* p=recomp_host_ptr_n(c,a,8); if(p){memcpy(p,&v,8);return;} memstore(c,a,8,v);}
|
|
|
|
/* Pair access. LDP and STP open and close every non-leaf function, which makes
|
|
them the most frequent guest memory operations there are, and as two separate
|
|
calls they walked the page table twice for one address. One walk covers both
|
|
words whenever the second does not cross out of the page; when it does, or
|
|
when the page is not plain backed memory, the single-word helpers answer
|
|
exactly as before - so nothing the emulator would have been told about an
|
|
access is skipped. */
|
|
static int recomp_pair_same_page(const RecompHostMem* hm, uint64_t a, uint64_t bytes){
|
|
uint64_t psz;
|
|
if(!hm || !hm->page_entries || hm->page_bits >= 64 || !bytes) return 0;
|
|
a &= 0xffffffffffffULL;
|
|
if(a >= hm->address_space_max || bytes > hm->address_space_max - a ||
|
|
bytes > UINT64_C(0x1000000000000) - a) return 0;
|
|
psz = UINT64_C(1) << hm->page_bits;
|
|
return bytes <= psz - (a & (psz - 1));
|
|
}
|
|
void recomp_ldp64(GuestContext* c,uint64_t a,uint64_t* lo,uint64_t* hi){
|
|
if(recomp_pair_same_page(c->host_mem,a,16)){
|
|
unsigned char* p=recomp_host_ptr(c,a);
|
|
if(p){ memcpy(lo,p,8); memcpy(hi,p+8,8); return; }
|
|
}
|
|
*lo=recomp_load64(c,a); *hi=recomp_load64(c,a+8);
|
|
}
|
|
void recomp_stp64(GuestContext* c,uint64_t a,uint64_t v0,uint64_t v1){
|
|
if(recomp_pair_same_page(c->host_mem,a,16)){
|
|
unsigned char* p=recomp_host_ptr(c,a);
|
|
if(p){ memcpy(p,&v0,8); memcpy(p+8,&v1,8); return; }
|
|
}
|
|
recomp_store64(c,a,v0); recomp_store64(c,a+8,v1);
|
|
}
|
|
void recomp_ldp32(GuestContext* c,uint64_t a,uint64_t* lo,uint64_t* hi){
|
|
if(recomp_pair_same_page(c->host_mem,a,8)){
|
|
unsigned char* p=recomp_host_ptr(c,a);
|
|
if(p){ uint32_t x,y; memcpy(&x,p,4); memcpy(&y,p+4,4);
|
|
*lo=(uint64_t)x; *hi=(uint64_t)y; return; }
|
|
}
|
|
*lo=recomp_load32(c,a); *hi=recomp_load32(c,a+4);
|
|
}
|
|
void recomp_stp32(GuestContext* c,uint64_t a,uint64_t v0,uint64_t v1){
|
|
if(recomp_pair_same_page(c->host_mem,a,8)){
|
|
unsigned char* p=recomp_host_ptr(c,a);
|
|
if(p){ uint32_t x=(uint32_t)v0,y=(uint32_t)v1;
|
|
memcpy(p,&x,4); memcpy(p+4,&y,4); return; }
|
|
}
|
|
recomp_store32(c,a,v0); recomp_store32(c,a+4,v1);
|
|
}
|
|
)RT";
|
|
const std::string tail = std::string(R"RT(
|
|
#ifndef RECOMP_STATIC_HOST
|
|
/* Owned by the runtime in the single-module shapes (standalone exe, loadable
|
|
shared image). When several modules are linked statically into one host this
|
|
file is compiled once for all of them, so each module defines its own
|
|
(renamed) copy in recomp_export.c instead. */
|
|
uint64_t g_module_base = 0;
|
|
#endif
|
|
|
|
uint64_t recomp_umulh(uint64_t a,uint64_t b){
|
|
/* Portable 64x64->high64: split into 32-bit halves. Avoids depending on
|
|
__int128 or MSVC intrinsics so the generated project stays plain C11. */
|
|
uint64_t al=a&0xFFFFFFFFULL, ah=a>>32, bl=b&0xFFFFFFFFULL, bh=b>>32;
|
|
uint64_t ll=al*bl, lh=al*bh, hl=ah*bl, hh=ah*bh;
|
|
uint64_t mid=(ll>>32)+(lh&0xFFFFFFFFULL)+(hl&0xFFFFFFFFULL);
|
|
return hh+(lh>>32)+(hl>>32)+(mid>>32);
|
|
}
|
|
uint64_t recomp_smulh(uint64_t a,uint64_t b){
|
|
uint64_t hi=recomp_umulh(a,b);
|
|
/* Convert the unsigned high half to the signed one. */
|
|
if((int64_t)a<0) hi-=b;
|
|
if((int64_t)b<0) hi-=a;
|
|
return hi;
|
|
}
|
|
void recomp_set_flags(GuestContext* c,int is_sub,uint64_t a,uint64_t b,uint64_t r,int is64){
|
|
uint64_t m=is64?~0ULL:0xFFFFFFFFULL; r&=m;a&=m;b&=m;
|
|
uint64_t s=is64?0x8000000000000000ULL:0x80000000ULL;
|
|
c->z=(r==0); c->n=(r&s)?1:0;
|
|
if(is_sub){ c->c=(a>=b); c->v=(((a^b)&(a^r))&s)?1:0; }
|
|
else { c->c=(r<a); c->v=((~(a^b)&(a^r))&s)?1:0; }
|
|
}
|
|
|
|
int recomp_cond(GuestContext* c,unsigned cond){
|
|
int n=c->n,z=c->z,cc=c->c,v=c->v,res;
|
|
switch(cond>>1){case 0:res=z;break;case 1:res=cc;break;case 2:res=n;break;case 3:res=v;break;
|
|
case 4:res=cc&&!z;break;case 5:res=(n==v);break;case 6:res=(n==v)&&!z;break;default:res=1;}
|
|
return ((cond&1)&&cond!=15)? !res:res;
|
|
}
|
|
|
|
/* ── Save-data filesystem ── */
|
|
|
|
static void mkpath(const char* path) {
|
|
char tmp[512]; size_t len;
|
|
snprintf(tmp,sizeof tmp,"%s",path); len=strlen(tmp);
|
|
for(size_t i=1;i<len;i++){
|
|
if(tmp[i]==PATH_SEP||tmp[i]=='/'){tmp[i]=0; MKDIR(tmp); tmp[i]=PATH_SEP;}
|
|
}
|
|
MKDIR(tmp);
|
|
}
|
|
|
|
int recomp_save_init(GuestContext* c, const char* exe_path) {
|
|
char dir[512];
|
|
/* Put save_data/ next to the executable */
|
|
snprintf(dir,sizeof dir,"%s",exe_path);
|
|
char* sl=strrchr(dir,PATH_SEP);
|
|
if(!sl) sl=strrchr(dir,'/');
|
|
if(sl) *(sl+1)=0; else dir[0]=0;
|
|
snprintf(c->save_dir,sizeof c->save_dir,"%ssave_data",dir);
|
|
mkpath(c->save_dir);
|
|
printf("[recomp] Save directory: %s\n",c->save_dir);
|
|
return 1;
|
|
}
|
|
|
|
int recomp_save_write(GuestContext* c, const char* name, const void* data, uint64_t size) {
|
|
char path[1024];
|
|
snprintf(path,sizeof path,"%s%c%s",c->save_dir,PATH_SEP,name);
|
|
/* Ensure parent dirs exist */
|
|
char parent[1024]; snprintf(parent,sizeof parent,"%s",path);
|
|
char* sl=strrchr(parent,PATH_SEP); if(!sl) sl=strrchr(parent,'/'); if(sl)*sl=0;
|
|
)RT") + R"RT( mkpath(parent);
|
|
FILE* f=fopen(path,"wb");
|
|
if(!f){fprintf(stderr,"[recomp] save write failed: %s\n",path); return 0;}
|
|
fwrite(data,1,(size_t)size,f); fclose(f);
|
|
printf("[recomp] Saved %llu bytes -> %s\n",(unsigned long long)size,path);
|
|
return 1;
|
|
}
|
|
|
|
int recomp_save_read(GuestContext* c, const char* name, void* buf, uint64_t buf_size, uint64_t* out_size) {
|
|
char path[1024];
|
|
snprintf(path,sizeof path,"%s%c%s",c->save_dir,PATH_SEP,name);
|
|
FILE* f=fopen(path,"rb");
|
|
if(!f){if(out_size)*out_size=0; return 0;}
|
|
fseek(f,0,SEEK_END); long sz=ftell(f); fseek(f,0,SEEK_SET);
|
|
uint64_t to_read=(uint64_t)sz<buf_size?(uint64_t)sz:buf_size;
|
|
fread(buf,1,(size_t)to_read,f); fclose(f);
|
|
if(out_size)*out_size=to_read;
|
|
printf("[recomp] Loaded %llu bytes <- %s\n",(unsigned long long)to_read,path);
|
|
return 1;
|
|
}
|
|
|
|
int recomp_save_delete(GuestContext* c, const char* name) {
|
|
char path[1024];
|
|
snprintf(path,sizeof path,"%s%c%s",c->save_dir,PATH_SEP,name);
|
|
return remove(path)==0;
|
|
}
|
|
|
|
int recomp_save_exists(GuestContext* c, const char* name) {
|
|
char path[1024];
|
|
snprintf(path,sizeof path,"%s%c%s",c->save_dir,PATH_SEP,name);
|
|
FILE* f=fopen(path,"rb");
|
|
if(f){fclose(f); return 1;} return 0;
|
|
}
|
|
|
|
/* ── Segment loader: loads rodata.bin + data.bin from the data dir into guest memory ── */
|
|
|
|
int recomp_load_segments(GuestContext* c, const char* data_dir) {
|
|
const char* names[]={"rodata.bin","data.bin","text.bin"};
|
|
/* Corresponding offsets from segment info embedded in manifest — for now, load
|
|
sequentially after .text in memory. The real offsets come from the blockmap. */
|
|
for(int i=0;i<3;i++){
|
|
char path[1024];
|
|
snprintf(path,sizeof path,"%s%c%s",data_dir,PATH_SEP,names[i]);
|
|
FILE* f=fopen(path,"rb");
|
|
if(!f) continue;
|
|
fseek(f,0,SEEK_END); long sz=ftell(f); fseek(f,0,SEEK_SET);
|
|
if((uint64_t)sz<=c->mem_size){
|
|
/* Load at the appropriate offset — text at base, others after */
|
|
fread(c->mem,1,(size_t)sz,f);
|
|
}
|
|
fclose(f);
|
|
printf("[recomp] Loaded segment %s (%ld bytes)\n",names[i],sz);
|
|
}
|
|
return 1;
|
|
}
|
|
|
|
/* ── SVC handler with HLE filesystem support ── */
|
|
|
|
#ifdef SUYU_HOSTED_RECOMP
|
|
/* In the suyu-hosted build the block already set pending_svc before calling
|
|
this. Just return — arm_recomp.cpp's run loop detects pending_svc != ~0 and
|
|
dispatches to the real HLE kernel. */
|
|
void recomp_svc(GuestContext* c,unsigned imm){ (void)c; (void)imm; }
|
|
#else
|
|
void recomp_svc(GuestContext* c,unsigned imm){
|
|
/* Standalone runtime: this handler services the call itself, so clear the
|
|
pending flag. The suyu-hosted build replaces this translation unit with a
|
|
bridge that leaves pending_svc set and returns, handing the call to the
|
|
real HLE kernel instead. */
|
|
c->pending_svc = ~0ULL;
|
|
switch(imm){
|
|
case 0x1: /* SetHeapSize — x1 = requested size */
|
|
if(c->heap_base==0){
|
|
c->heap_base=c->mem_base_vaddr+c->mem_size/2;
|
|
c->heap_cur=c->heap_base;
|
|
c->heap_end=c->heap_base+c->mem_size/2;
|
|
}
|
|
c->x[0]=0; /* success */
|
|
c->x[1]=c->heap_base;
|
|
break;
|
|
case 0x2: /* SetMemoryPermission — stub success */
|
|
c->x[0]=0;
|
|
break;
|
|
case 0x3: /* SetMemoryAttribute — stub success */
|
|
c->x[0]=0;
|
|
break;
|
|
case 0x6: /* QueryMemory — stub: report all memory as readable/writable */
|
|
c->x[0]=0;
|
|
c->x[1]=0; /* MemoryInfo written to [x0] — simplified */
|
|
break;
|
|
case 0x7: /* ExitProcess */
|
|
printf("[recomp] ExitProcess called\n");
|
|
c->halted=1;
|
|
break;
|
|
case 0x8: /* CreateThread — stub, return handle=1 */
|
|
c->x[0]=0; c->x[1]=1;
|
|
break;
|
|
case 0xB: /* SleepThread — yield CPU to prevent spin-lock lag */
|
|
#ifdef _WIN32
|
|
Sleep((DWORD)(c->x[0] / 1000000ULL)); /* ns to ms */
|
|
#else
|
|
{ struct timespec ts; ts.tv_sec=0; ts.tv_nsec=(long)(c->x[0]>0?c->x[0]:1000000);
|
|
nanosleep(&ts,0); }
|
|
#endif
|
|
c->x[0]=0;
|
|
break;
|
|
case 0x15: /* SendSyncRequest — IPC for fsp-srv / save data */
|
|
/* Simplified HLE: check x[0] for handle, interpret IPC command buffer.
|
|
For now, stub success so game save code paths don't crash. */
|
|
c->x[0]=0;
|
|
break;
|
|
case 0x16: /* SendSyncRequestWithUserBuffer */
|
|
c->x[0]=0;
|
|
break;
|
|
case 0x18: /* CloseHandle — stub */
|
|
c->x[0]=0;
|
|
break;
|
|
case 0x1A: /* WaitSynchronization — stub immediate return */
|
|
c->x[0]=0; c->x[1]=0;
|
|
break;
|
|
case 0x1F: /* ConnectToNamedPort — stub, return handle */
|
|
c->x[0]=0; c->x[1]=0x100;
|
|
break;
|
|
case 0x21: /* SendSyncRequest (sm: variant) */
|
|
c->x[0]=0;
|
|
break;
|
|
case 0x26: /* Break — debug break */
|
|
printf("[recomp] Break SVC x0=%llu\n",(unsigned long long)c->x[0]);
|
|
break;
|
|
case 0x27: /* OutputDebugString */
|
|
{ uint8_t* p=memptr(c,c->x[0],(uint64_t)c->x[1]);
|
|
if(p) printf("[guest] %.*s\n",(int)c->x[1],(char*)p);
|
|
c->x[0]=0;
|
|
}
|
|
break;
|
|
case 0x29: /* GetInfo — return stub values for system info queries */
|
|
{ uint32_t id=(uint32_t)c->x[1];
|
|
switch(id){
|
|
case 0: c->x[1]=0xFFFFFF; break; /* AllowedCPUCoreMask */
|
|
case 1: c->x[1]=0xF; break; /* AllowedThreadPrioMask */
|
|
case 2: c->x[1]=c->mem_base_vaddr; break; /* MapRegionBaseAddr */
|
|
case 3: c->x[1]=c->mem_size; break; /* MapRegionSize */
|
|
case 4: c->x[1]=c->heap_base; break; /* HeapRegionBaseAddr */
|
|
case 5: c->x[1]=c->heap_end-c->heap_base; break; /* HeapRegionSize */
|
|
case 6: c->x[1]=c->mem_size; break; /* TotalMemorySize */
|
|
case 7: c->x[1]=c->mem_size/2; break; /* UsedMemorySize */
|
|
case 12: c->x[1]=c->mem_base_vaddr+c->mem_size; break; /* AslrRegionBaseAddr */
|
|
case 13: c->x[1]=0x1000000; break; /* AslrRegionSize */
|
|
case 14: c->x[1]=c->mem_base_vaddr+c->mem_size; break; /* StackRegionBaseAddr */
|
|
case 15: c->x[1]=0x100000; break; /* StackRegionSize */
|
|
default: c->x[1]=0; break;
|
|
}
|
|
c->x[0]=0;
|
|
}
|
|
break;
|
|
default:
|
|
printf("[recomp] Unhandled SVC #0x%x x0=0x%llx x1=0x%llx\n",imm,
|
|
(unsigned long long)c->x[0],(unsigned long long)c->x[1]);
|
|
c->x[0]=0;
|
|
break;
|
|
}
|
|
}
|
|
#endif /* SUYU_HOSTED_RECOMP */
|
|
|
|
/* An opcode the decoder does not implement. Stubbing it out (the old
|
|
behaviour: zero x0 and carry on) silently corrupts guest state - a
|
|
function whose body is one unimplemented SIMD instruction returns a
|
|
plausible-looking 0, and the caller then dereferences it. That is
|
|
indistinguishable from a real null and shows up much later as a crash
|
|
nowhere near the actual gap.
|
|
|
|
Instead, park the context at this exact PC and halt with a distinct code.
|
|
The host (ArmRecomp::RunThread) treats halted == RECOMP_HALT_UNHANDLED as
|
|
"run this address on the interpreter fallback instead", so the instruction
|
|
executes correctly and control returns to recompiled code as soon as the
|
|
PC is covered again. Correct by construction whatever the decoder does or
|
|
does not cover, and every remaining gap costs speed rather than
|
|
correctness.
|
|
|
|
Standalone builds have no fallback engine to hand off to, so they keep the
|
|
old step-over behaviour - degraded, but still the best available there. */
|
|
/* DMB/DSB/ISB. Under the hosted backend the guest is genuinely multi-threaded,
|
|
so these must be real fences: without one the host compiler is free to sink a
|
|
pointer store past the field stores the barrier was there to publish. The
|
|
standalone runtime drives one guest thread on one host thread and has nothing
|
|
to order against, so it keeps the free no-op. */
|
|
void recomp_barrier(void){
|
|
#ifdef SUYU_HOSTED_RECOMP
|
|
#if defined(__cplusplus)
|
|
atomic_thread_fence(memory_order_seq_cst);
|
|
#elif defined(_MSC_VER)
|
|
MemoryBarrier();
|
|
#else
|
|
__atomic_thread_fence(__ATOMIC_SEQ_CST);
|
|
#endif
|
|
#endif
|
|
}
|
|
|
|
void recomp_unhandled(GuestContext* c,uint32_t insn,uint64_t pc){
|
|
#ifdef SUYU_HOSTED_RECOMP
|
|
(void)insn;
|
|
c->pc=pc;
|
|
c->halted=RECOMP_HALT_UNHANDLED;
|
|
#else
|
|
fprintf(stderr,"[recomp] unhandled insn 0x%08x at 0x%llx\n",insn,(unsigned long long)pc);
|
|
(void)c; /* stepping over is bad enough; do not clobber a register too */
|
|
#endif
|
|
}
|
|
|
|
#ifndef RECOMP_STATIC_HOST
|
|
/* Drives a single module's own dispatch table. Meaningless when the runtime is
|
|
shared between several statically linked modules - the host dispatches
|
|
across them instead - so it is compiled out there, where recomp_lookup has
|
|
been renamed per module and would not resolve. */
|
|
void recomp_run(GuestContext* c){
|
|
uint64_t g=0;
|
|
while(!c->halted){
|
|
BlockFn f=recomp_lookup(c->pc);
|
|
if(!f){ fprintf(stderr,"[recomp] no block at 0x%llx\n",(unsigned long long)c->pc); break;}
|
|
f(c);
|
|
if(++g>100000000ULL){ fprintf(stderr,"[recomp] watchdog (100M iterations)\n"); break; }
|
|
/* Yield every 4096 blocks to prevent 100% CPU spin on tight loops */
|
|
if((g & 0xFFF)==0){
|
|
#ifdef _WIN32
|
|
Sleep(0);
|
|
#else
|
|
{ struct timespec ts={0,0}; nanosleep(&ts,0); }
|
|
#endif
|
|
}
|
|
}
|
|
}
|
|
#endif /* !RECOMP_STATIC_HOST */
|
|
)RT";
|
|
std::string text = head + (fastmem ? FastmemHelpersC(abi5_helpers, guard_gen) : abi5_helpers) + tail +
|
|
EstimateRuntimeC();
|
|
if (fastmem && guard_gen) {
|
|
static constexpr std::string_view anchor =
|
|
"void recomp_ic_ivau(GuestContext* c,uint64_t address,int host_guard_version){\n";
|
|
text.insert(text.find(anchor), GuardGenC());
|
|
}
|
|
return text;
|
|
}
|
|
|
|
inline const char* RuntimeC() {
|
|
static const std::string abi5 = BuildRuntimeC(false);
|
|
static const std::string fastmem = BuildRuntimeC(true);
|
|
static const std::string guard_gen = BuildRuntimeC(true, true);
|
|
static const std::string abi5_shadow = abi5 + FpxShadowC();
|
|
static const std::string fastmem_shadow = fastmem + FpxShadowC();
|
|
static const std::string guard_gen_shadow = guard_gen + FpxShadowC();
|
|
static const std::string abi5_fpx = abi5 + FpxExactC();
|
|
static const std::string fastmem_fpx = fastmem + FpxExactC();
|
|
static const std::string guard_gen_fpx = guard_gen + FpxExactC();
|
|
if (FpxVariant() == 2) {
|
|
return (EmitGuardGen() ? guard_gen_shadow : g_emit_fastmem ? fastmem_shadow : abi5_shadow)
|
|
.c_str();
|
|
}
|
|
if (FpxVariant() == 1) {
|
|
return (EmitGuardGen() ? guard_gen_fpx : g_emit_fastmem ? fastmem_fpx : abi5_fpx).c_str();
|
|
}
|
|
return (EmitGuardGen() ? guard_gen : g_emit_fastmem ? fastmem : abi5).c_str();
|
|
}
|
|
|
|
} // namespace suyu::recomp
|