MacKwnoX/NX2navi
1
0
Fork 0
mirror of https://github.com/tangerineturtle1/NX2navi.git synced 2026-11-15 19:19:58 -04:00
Code Issues Packages Projects Releases Wiki Activity
Files
2dd363b86ee5ae61c3604e5a76b5ee1de523326c
NX2navi/runtime/simd.h
chrg6 c4268779fc Write a V register with one 16-byte store. 
A scalar result (an FP operation, a scalar load, FMOV, a compare, a conversion, a 64-bit vector operation) was assigned
to the register as a zero-extended 128-bit integer, which the compiler writes as two 8-byte stores, and INS and a
single-lane load wrote their lane alone. The next 16-byte load of the register then cannot take its value from the
store buffer and waits until the stores have left it.

Every such write now builds the whole value in a vector register and stores it once (runtime/simd.h): VSET puts a
scalar in the low bits with zero above it, VSET128 writes a 128-bit value built in an integer, and VINS loads the
register, replaces one lane and stores all of it. VINS passes the value through an empty asm statement, without which
the compiler turns the load, the insert and the store back into a store of the lane.

In the 360 guest functions a play profile sampled, stores of a part of a V register fall from 64322 to 4 and whole
stores rise from 10932 to 43665; the four left copy a 16-byte guest load that the compiler itself splits because the
memory was just written in halves. Host instructions fall from 1448075 to 1427163 and stores to the Cpu from 177160 to
145566; its loads rise from 141090 to 143521, as an insert loads the register it changes.

The lifted text of FP and SIMD code changes with this, so those files are compiled again.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-12 21:33:21 -05:00

862 lines
44 KiB
C
Raw Blame History

// Contains code Copyright (c) 2026 Peiikai (MIT; see THIRD_PARTY_NOTICES.md).
// Portable C lowering of AArch64 FP / AdvSIMD / crypto / CRC32 (tools/simd_c.py emits calls with constant
// fields; everything is always_inline so the size/lane/op switches fold at -O1).
// Semantics follow the ARM pseudocode, not the host: NaN propagation (first signalling NaN quietened, then
// first quiet NaN), FPCR.DN default NaN, FPCR.FZ flushing (inputs and pre-rounding-tiny results), FPCR.RMode
// for arithmetic / conversions / FRINTX/I, saturating FP->int conversions, fused multiply-add, ARM estimate
// tables. Not modelled: FPSR cumulative exception bits and QC, FPCR trap enables, FP16 arithmetic (ARMv8.2).
// FPCR.AHP is honoured by the half-precision conversions. Host assumptions: IEEE binary32/64, MXCSR in
// round-to-nearest outside the slow paths, with FTZ|DAZ equal to FPCR.FZ (cpu.h rt_fpcr_sync keeps it so); fma must be
// correctly rounded: -march=x86-64-v3 has the FMA instruction, and the v2 build links runtime/fma.c for it.
#pragma once
#include <fenv.h>
#include <immintrin.h>
#include <math.h>
#include "cpu.h"
typedef __uint128_t u128;
#define SI static inline __attribute__((always_inline))
#define SN static __attribute__((noinline))
#define SIMD_CAT2(a, b) a##_##b
#define SIMD_CAT(a, b) SIMD_CAT2(a, b)
#define SIMD_VT(name, T) typedef T name __attribute__((vector_size(16), may_alias));
SIMD_VT(vu8, uint8_t) SIMD_VT(vs8, int8_t) SIMD_VT(vu16, uint16_t) SIMD_VT(vs16, int16_t)
SIMD_VT(vu32, uint32_t) SIMD_VT(vs32, int32_t) SIMD_VT(vu64, uint64_t) SIMD_VT(vs64, int64_t)
SIMD_VT(vf32, float) SIMD_VT(vf64, double)
typedef uint16_t __attribute__((may_alias)) sa16;
typedef uint32_t __attribute__((may_alias)) sa32;
typedef uint64_t __attribute__((may_alias)) sa64;
typedef float __attribute__((may_alias)) saf32;
typedef double __attribute__((may_alias)) saf64;
typedef u128 __attribute__((aligned(1), may_alias)) u128u;
// ---- registers: VREG = whole V register as a vector type; VQ = source with the upper half cleared when
// Q=0; VW = write, zeroing the upper half when Q=0. Scalar views of lane 0: VS8..VS64, VSF32/VSF64.
// A write is one 16-byte store, also where the instruction sets a scalar (the bits above it become zero) or a single
// lane (the other lanes stay): a V register written as two 8-byte halves or as one lane makes the next 16-byte load
// of it wait until those stores have left the store buffer, because the processor forwards a store to a load only
// when the one store covers the load. Assigning a zero-extended u128 to c->v[d] compiles to the two halves, so the
// writes go through a vector type:
// VSET = a scalar of at most 64 bits in the low bits, zero above; VSET128 = all 128 bits of an integer value;
// VINS = lane i replaced by x, the other lanes kept.
#define VREG(T, n) (*(T *)&c->v[n])
#define VQ(T, Q, n) ((T)((Q) ? c->v[n] : (u128)(uint64_t)c->v[n]))
#define VSET(d, x) (VREG(vu64, d) = (vu64){(uint64_t)(x), 0})
#define VSET128(d, x) (VREG(vu64, d) = (vu64)(u128)(x))
#define VW(T, Q, d, e) do { T _w = (e); VREG(vu64, d) = (Q) ? (vu64)_w : (vu64){((vu64)_w)[0], 0}; } while (0)
#define VINS(T, d, i, x) do { T _l = VREG(T, d); _l[i] = (x); VREG(vu64, d) = simd_whole((vu64)_l); } while (0)
#define VS8(n) (*(uint8_t *)&c->v[n])
#define VS16(n) (*(sa16 *)&c->v[n])
#define VS32(n) (*(sa32 *)&c->v[n])
#define VS64(n) (*(sa64 *)&c->v[n])
#define VSF32(n) (*(saf32 *)&c->v[n])
#define VSF64(n) (*(saf64 *)&c->v[n])
#define VHI64(n) (((sa64 *)&c->v[n])[1])
// v, with where it came from hidden from the compiler: a load of a V register, a change of one lane and a store of
// the 16 bytes would otherwise be compiled as a store of that lane alone.
SI vu64 simd_whole(vu64 v) {
__asm__("" : "+x"(v));
return v;
}
#define LD128(a) (*(const u128u *)(uintptr_t)(a))
#define ST128(a, v) (*(u128u *)(uintptr_t)(a) = (u128)(v))
#define SHUF __builtin_shufflevector
#define CVT __builtin_convertvector
#define FPCR_AHP (1u << 26)
#define FPCR_DN (1u << 25)
#define FPCR_FZ (1u << 24)
#define FPCR_RM (3u << 22)
enum { FOP_ADD, FOP_SUB, FOP_MUL, FOP_MULX, FOP_NMUL, FOP_DIV, FOP_ABD, FOP_SQRT, FOP_MADD, FOP_RECPS, FOP_RSQRTS,
FOP_MAX, FOP_MIN, FOP_MAXNM, FOP_MINNM };
static const uint8_t simd_recip_tab[256] = {
255, 253, 251, 249, 247, 245, 243, 241, 240, 238, 236, 234, 232, 230, 229, 227,
225, 223, 221, 220, 218, 216, 215, 213, 211, 210, 208, 206, 205, 203, 201, 200,
198, 197, 195, 194, 192, 191, 189, 188, 186, 185, 183, 182, 180, 179, 177, 176,
174, 173, 172, 170, 169, 167, 166, 165, 163, 162, 161, 159, 158, 157, 156, 154,
153, 152, 150, 149, 148, 147, 145, 144, 143, 142, 141, 139, 138, 137, 136, 135,
134, 132, 131, 130, 129, 128, 127, 126, 124, 123, 122, 121, 120, 119, 118, 117,
116, 115, 114, 113, 112, 111, 110, 109, 108, 107, 106, 105, 104, 103, 102, 101,
100, 99, 98, 97, 96, 95, 94, 93, 92, 91, 90, 89, 88, 88, 87, 86,
85, 84, 83, 82, 81, 81, 80, 79, 78, 77, 76, 75, 75, 74, 73, 72,
71, 70, 70, 69, 68, 67, 66, 66, 65, 64, 63, 63, 62, 61, 60, 59,
59, 58, 57, 56, 56, 55, 54, 53, 53, 52, 51, 51, 50, 49, 48, 48,
47, 46, 46, 45, 44, 44, 43, 42, 42, 41, 40, 40, 39, 38, 38, 37,
36, 36, 35, 34, 34, 33, 32, 32, 31, 30, 30, 29, 29, 28, 27, 27,
26, 26, 25, 24, 24, 23, 23, 22, 21, 21, 20, 20, 19, 18, 18, 17,
17, 16, 16, 15, 15, 14, 13, 13, 12, 12, 11, 11, 10, 10, 9, 9,
8, 7, 7, 6, 6, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0,
};
static const uint8_t simd_rsqrt_tab[384] = {
255, 253, 251, 249, 247, 245, 243, 242, 240, 238, 236, 234, 233, 231, 229, 228,
226, 224, 223, 221, 219, 218, 216, 215, 213, 212, 210, 209, 207, 206, 204, 203,
201, 200, 198, 197, 196, 194, 193, 192, 190, 189, 188, 186, 185, 184, 183, 181,
180, 179, 178, 176, 175, 174, 173, 172, 170, 169, 168, 167, 166, 165, 164, 163,
162, 160, 159, 158, 157, 156, 155, 154, 153, 152, 151, 150, 149, 148, 147, 146,
145, 144, 143, 142, 141, 140, 140, 139, 138, 137, 136, 135, 134, 133, 132, 131,
131, 130, 129, 128, 127, 126, 126, 125, 124, 123, 122, 121, 121, 120, 119, 118,
118, 117, 116, 115, 114, 114, 113, 112, 111, 111, 110, 109, 109, 108, 107, 106,
105, 105, 104, 104, 103, 103, 101, 101, 100, 100, 99, 99, 97, 97, 96, 96,
95, 95, 93, 93, 92, 92, 91, 91, 90, 90, 88, 88, 87, 87, 86, 86,
85, 85, 84, 84, 82, 82, 81, 81, 80, 80, 79, 79, 78, 78, 77, 77,
76, 76, 75, 75, 74, 74, 72, 72, 71, 71, 70, 70, 69, 69, 68, 68,
67, 67, 66, 66, 65, 65, 64, 64, 63, 63, 62, 62, 61, 61, 60, 60,
60, 60, 59, 59, 58, 58, 57, 57, 56, 56, 55, 55, 54, 54, 53, 53,
52, 52, 51, 51, 51, 51, 50, 50, 49, 49, 48, 48, 47, 47, 46, 46,
46, 46, 45, 45, 44, 44, 43, 43, 42, 42, 42, 42, 41, 41, 40, 40,
39, 39, 38, 38, 38, 38, 37, 37, 36, 36, 35, 35, 35, 35, 34, 34,
33, 33, 33, 33, 32, 32, 31, 31, 30, 30, 30, 30, 29, 29, 28, 28,
28, 28, 27, 27, 26, 26, 26, 26, 25, 25, 24, 24, 24, 24, 23, 23,
22, 22, 22, 22, 21, 21, 20, 20, 20, 20, 19, 19, 19, 19, 18, 18,
17, 17, 17, 17, 16, 16, 16, 16, 15, 15, 14, 14, 14, 14, 13, 13,
13, 13, 12, 12, 11, 11, 11, 11, 10, 10, 10, 10, 9, 9, 9, 9,
8, 8, 8, 8, 7, 7, 6, 6, 6, 6, 5, 5, 5, 5, 4, 4,
4, 4, 3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0,
};
static const uint32_t simd_crc32_tab[256] = {
0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3,
0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91,
0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7,
0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5,
0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b,
0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59,
0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f,
0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d,
0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433,
0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01,
0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457,
0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65,
0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb,
0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9,
0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f,
0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad,
0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683,
0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1,
0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7,
0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5,
0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b,
0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79,
0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f,
0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d,
0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713,
0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21,
0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777,
0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45,
0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db,
0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9,
0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf,
0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d,
};
static const uint32_t simd_crc32c_tab[256] = {
0x00000000, 0xf26b8303, 0xe13b70f7, 0x1350f3f4, 0xc79a971f, 0x35f1141c, 0x26a1e7e8, 0xd4ca64eb,
0x8ad958cf, 0x78b2dbcc, 0x6be22838, 0x9989ab3b, 0x4d43cfd0, 0xbf284cd3, 0xac78bf27, 0x5e133c24,
0x105ec76f, 0xe235446c, 0xf165b798, 0x030e349b, 0xd7c45070, 0x25afd373, 0x36ff2087, 0xc494a384,
0x9a879fa0, 0x68ec1ca3, 0x7bbcef57, 0x89d76c54, 0x5d1d08bf, 0xaf768bbc, 0xbc267848, 0x4e4dfb4b,
0x20bd8ede, 0xd2d60ddd, 0xc186fe29, 0x33ed7d2a, 0xe72719c1, 0x154c9ac2, 0x061c6936, 0xf477ea35,
0xaa64d611, 0x580f5512, 0x4b5fa6e6, 0xb93425e5, 0x6dfe410e, 0x9f95c20d, 0x8cc531f9, 0x7eaeb2fa,
0x30e349b1, 0xc288cab2, 0xd1d83946, 0x23b3ba45, 0xf779deae, 0x05125dad, 0x1642ae59, 0xe4292d5a,
0xba3a117e, 0x4851927d, 0x5b016189, 0xa96ae28a, 0x7da08661, 0x8fcb0562, 0x9c9bf696, 0x6ef07595,
0x417b1dbc, 0xb3109ebf, 0xa0406d4b, 0x522bee48, 0x86e18aa3, 0x748a09a0, 0x67dafa54, 0x95b17957,
0xcba24573, 0x39c9c670, 0x2a993584, 0xd8f2b687, 0x0c38d26c, 0xfe53516f, 0xed03a29b, 0x1f682198,
0x5125dad3, 0xa34e59d0, 0xb01eaa24, 0x42752927, 0x96bf4dcc, 0x64d4cecf, 0x77843d3b, 0x85efbe38,
0xdbfc821c, 0x2997011f, 0x3ac7f2eb, 0xc8ac71e8, 0x1c661503, 0xee0d9600, 0xfd5d65f4, 0x0f36e6f7,
0x61c69362, 0x93ad1061, 0x80fde395, 0x72966096, 0xa65c047d, 0x5437877e, 0x4767748a, 0xb50cf789,
0xeb1fcbad, 0x197448ae, 0x0a24bb5a, 0xf84f3859, 0x2c855cb2, 0xdeeedfb1, 0xcdbe2c45, 0x3fd5af46,
0x7198540d, 0x83f3d70e, 0x90a324fa, 0x62c8a7f9, 0xb602c312, 0x44694011, 0x5739b3e5, 0xa55230e6,
0xfb410cc2, 0x092a8fc1, 0x1a7a7c35, 0xe811ff36, 0x3cdb9bdd, 0xceb018de, 0xdde0eb2a, 0x2f8b6829,
0x82f63b78, 0x709db87b, 0x63cd4b8f, 0x91a6c88c, 0x456cac67, 0xb7072f64, 0xa457dc90, 0x563c5f93,
0x082f63b7, 0xfa44e0b4, 0xe9141340, 0x1b7f9043, 0xcfb5f4a8, 0x3dde77ab, 0x2e8e845f, 0xdce5075c,
0x92a8fc17, 0x60c37f14, 0x73938ce0, 0x81f80fe3, 0x55326b08, 0xa759e80b, 0xb4091bff, 0x466298fc,
0x1871a4d8, 0xea1a27db, 0xf94ad42f, 0x0b21572c, 0xdfeb33c7, 0x2d80b0c4, 0x3ed04330, 0xccbbc033,
0xa24bb5a6, 0x502036a5, 0x4370c551, 0xb11b4652, 0x65d122b9, 0x97baa1ba, 0x84ea524e, 0x7681d14d,
0x2892ed69, 0xdaf96e6a, 0xc9a99d9e, 0x3bc21e9d, 0xef087a76, 0x1d63f975, 0x0e330a81, 0xfc588982,
0xb21572c9, 0x407ef1ca, 0x532e023e, 0xa145813d, 0x758fe5d6, 0x87e466d5, 0x94b49521, 0x66df1622,
0x38cc2a06, 0xcaa7a905, 0xd9f75af1, 0x2b9cd9f2, 0xff56bd19, 0x0d3d3e1a, 0x1e6dcdee, 0xec064eed,
0xc38d26c4, 0x31e6a5c7, 0x22b65633, 0xd0ddd530, 0x0417b1db, 0xf67c32d8, 0xe52cc12c, 0x1747422f,
0x49547e0b, 0xbb3ffd08, 0xa86f0efc, 0x5a048dff, 0x8ecee914, 0x7ca56a17, 0x6ff599e3, 0x9d9e1ae0,
0xd3d3e1ab, 0x21b862a8, 0x32e8915c, 0xc083125f, 0x144976b4, 0xe622f5b7, 0xf5720643, 0x07198540,
0x590ab964, 0xab613a67, 0xb831c993, 0x4a5a4a90, 0x9e902e7b, 0x6cfbad78, 0x7fab5e8c, 0x8dc0dd8f,
0xe330a81a, 0x115b2b19, 0x020bd8ed, 0xf0605bee, 0x24aa3f05, 0xd6c1bc06, 0xc5914ff2, 0x37faccf1,
0x69e9f0d5, 0x9b8273d6, 0x88d28022, 0x7ab90321, 0xae7367ca, 0x5c18e4c9, 0x4f48173d, 0xbd23943e,
0xf36e6f75, 0x0105ec76, 0x12551f82, 0xe03e9c81, 0x34f4f86a, 0xc69f7b69, 0xd5cf889d, 0x27a40b9e,
0x79b737ba, 0x8bdcb4b9, 0x988c474d, 0x6ae7c44e, 0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351,
};
static const uint8_t simd_aes_sbox[256] = {
0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76,
0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0,
0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc, 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15,
0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75,
0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84,
0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b, 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf,
0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8,
0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2,
0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17, 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73,
0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb,
0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c, 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79,
0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9, 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08,
0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a,
0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e,
0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94, 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf,
0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16,
};
static const uint8_t simd_aes_isbox[256] = {
0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb,
0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87, 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb,
0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d, 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e,
0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2, 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25,
0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16, 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92,
0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda, 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84,
0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a, 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06,
0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02, 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b,
0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea, 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73,
0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85, 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e,
0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89, 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b,
0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20, 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4,
0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31, 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f,
0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d, 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef,
0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0, 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61,
0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d,
};
SI int simd_fe_mode(uint64_t fpcr) {
static const int m[4] = {FE_TONEAREST, FE_UPWARD, FE_DOWNWARD, FE_TOWARDZERO};
return m[(fpcr >> 22) & 3];
}
// MXCSR rounding control (bits 13-14) without fesetround's x87 + SSE double write; ldmxcsr only on change.
#define SIMD_RC_MASK 0x6000u
SI unsigned simd_rm_set_rc(unsigned cur, unsigned rc) {
if ((cur & SIMD_RC_MASK) != rc) _mm_setcsr((cur & ~SIMD_RC_MASK) | rc);
return cur;
}
// Set MXCSR RC from FPCR.RMode (RN 00, RP 10, RM 01, RZ 11); returns the previous MXCSR.
SI unsigned simd_rm_enter(uint64_t fpcr) {
static const unsigned rc[4] = {0x0000u, 0x4000u, 0x2000u, 0x6000u};
return simd_rm_set_rc(_mm_getcsr(), rc[(fpcr >> 22) & 3]);
}
// Force toward-zero.
SI void simd_rm_rz(void) { simd_rm_set_rc(_mm_getcsr(), 0x6000u); }
// Restore the MXCSR saved by simd_rm_enter, only if it changed.
SI void simd_rm_leave(unsigned old) {
unsigned cur = _mm_getcsr();
if ((cur & SIMD_RC_MASK) != (old & SIMD_RC_MASK)) _mm_setcsr((cur & ~SIMD_RC_MASK) | (old & SIMD_RC_MASK));
}
// The thread's MXCSR mirrors the guest FPCR.FZ with RC = nearest whenever guest code runs (cpu.h rt_fpcr_sync), so
// plain host FP gives the ARM result unless FPCR.RMode asks for directed rounding, which the slow paths handle.
SI int simd_directed(const Cpu *c) { return __builtin_expect((c->fpcr & FPCR_RM) != 0, 0); }
#define F s
#define T float
#define U uint32_t
#define NB 32
#define EB 8
#define MB 23
#define VT vf32
#define VTU vu32
#define VTS vs32
#define NL 4
#include "simd_fp.h"
#undef F
#undef T
#undef U
#undef NB
#undef EB
#undef MB
#undef VT
#undef VTU
#undef VTS
#undef NL
#define F d
#define T double
#define U uint64_t
#define NB 64
#define EB 11
#define MB 52
#define VT vf64
#define VTU vu64
#define VTS vs64
#define NL 2
#include "simd_fp.h"
#undef F
#undef T
#undef U
#undef NB
#undef EB
#undef MB
#undef VT
#undef VTU
#undef VTS
#undef NL
// ---- precision conversions (FCVT, FCVTL/N, FCVTXN); FZ flushes single/double inputs, never halves
SI double simd_s2d(Cpu *c, float x) {
uint32_t u = fb_s(fz_s(c, x));
if (isnan_s(u)) {
if (c->fpcr & FPCR_DN) return fv_d(0x7ff8000000000000ull);
return fv_d(((uint64_t)(u >> 31) << 63) | 0x7ff8000000000000ull | ((uint64_t)(u & 0x3fffff) << 29));
}
return (double)fv_s(u);
}
SN float simd_d2s_slow(Cpu *c, double x) {
#pragma STDC FENV_ACCESS ON
unsigned old = simd_rm_enter(c->fpcr);
volatile double vx = x;
volatile float r = (float)vx;
simd_rm_leave(old);
return r;
}
SI float simd_d2s(Cpu *c, double x) {
uint64_t u = fb_d(fz_d(c, x));
if (isnan_d(u)) {
if (c->fpcr & FPCR_DN) return fv_s(0x7fc00000u);
return fv_s((uint32_t)(u >> 63) << 31 | 0x7fc00000u | (uint32_t)((u >> 29) & 0x3fffff));
}
x = fv_d(u);
// FZ: tiny before rounding (host FTZ would keep a result rounded up to the smallest normal)
if ((c->fpcr & FPCR_FZ) && __builtin_fabs(x) < 0x1p-126 && x != 0) return fv_s((uint32_t)(u >> 63) << 31);
if (__builtin_expect(c->fpcr & FPCR_RM, 0)) return simd_d2s_slow(c, x);
return (float)x;
}
// round to odd (FCVTXN)
SI float simd_d2s_odd(Cpu *c, double x) {
uint64_t u = fb_d(fz_d(c, x));
if (isnan_d(u)) return simd_d2s(c, x);
x = fv_d(u);
if ((c->fpcr & FPCR_FZ) && __builtin_fabs(x) < 0x1p-126 && x != 0) return fv_s((uint32_t)(u >> 63) << 31);
float f = (float)x;
if ((double)f != x && x == x) {
uint32_t b = fb_s(f);
if (__builtin_fabs((double)f) > __builtin_fabs(x)) b--;
return fv_s(b | 1);
}
return f;
}
SI uint64_t simd_rnd_shift(uint64_t m, int sh, int rm, int neg) {
if (sh >= 64) return (rm == 1 && !neg && m) || (rm == 2 && neg && m);
uint64_t q = m >> sh, rem = m & ((1ull << sh) - 1), half = 1ull << (sh - 1);
if (rm == 0) q += rem > half || (rem == half && (q & 1));
else if (rm == 1) q += rem && !neg;
else if (rm == 2) q += rem && neg;
return q;
}
// double (exact image of the single/double source, NaN payload left-aligned) -> half, FPCR rounding
SI uint16_t simd_d2h(Cpu *c, uint64_t u) {
uint64_t fpcr = c->fpcr;
int ahp = !!(fpcr & FPCR_AHP), rm = (fpcr >> 22) & 3, neg = (int)(u >> 63);
uint16_t s = (uint16_t)(neg << 15);
int e = (int)((u >> 52) & 0x7ff);
uint64_t f = u & ((1ull << 52) - 1);
if (e == 0x7ff) {
if (f) return ahp ? s : (fpcr & FPCR_DN) ? 0x7e00 : (uint16_t)(s | 0x7e00 | (f >> 42));
return ahp ? (uint16_t)(s | 0x7fff) : (uint16_t)(s | 0x7c00);
}
if (!e && !f) return s;
uint64_t m = e ? f | (1ull << 52) : f;
int E = e ? e - 1075 : -1074, ue = e ? e - 1023 : -1023;
uint64_t q;
int bits;
if (ue < -14) {
q = simd_rnd_shift(m, -24 - E, rm, neg);
bits = (int)q;
} else {
q = simd_rnd_shift(m, 42, rm, neg);
bits = ((ue + 14) << 10) + (int)q;
}
if (ahp) return (uint16_t)(s | (bits > 0x7fff ? 0x7fff : bits));
if (bits >= 0x7c00) {
int inf = rm == 0 || (rm == 1 && !neg) || (rm == 2 && neg);
return (uint16_t)(s | (inf ? 0x7c00 : 0x7bff));
}
return (uint16_t)(s | bits);
}
SI uint16_t simd_s2h(Cpu *c, float x) {
uint32_t u = fb_s(fz_s(c, x));
uint64_t d = isnan_s(u) ? ((uint64_t)(u >> 31) << 63) | 0x7ff0000000000000ull | ((uint64_t)(u & 0x7fffff) << 29)
: fb_d((double)fv_s(u));
return simd_d2h(c, d);
}
SI uint16_t simd_dd2h(Cpu *c, double x) { return simd_d2h(c, fb_d(fz_d(c, x))); }
SI uint64_t simd_h2dbits(Cpu *c, uint16_t h) {
uint64_t s = (uint64_t)(h >> 15) << 63;
int e = (h >> 10) & 31, f = h & 0x3ff;
if (e == 31 && !(c->fpcr & FPCR_AHP)) {
if (!f) return s | 0x7ff0000000000000ull;
if (c->fpcr & FPCR_DN) return 0x7ff8000000000000ull;
return s | 0x7ff8000000000000ull | ((uint64_t)f << 42);
}
if (!e) return fb_d((s ? -1.0 : 1.0) * f * 0x1p-24);
return s | ((uint64_t)(e - 15 + 1023) << 52) | ((uint64_t)f << 42);
}
SI double simd_h2d(Cpu *c, uint16_t h) { return fv_d(simd_h2dbits(c, h)); }
SI float simd_h2s(Cpu *c, uint16_t h) {
uint64_t d = simd_h2dbits(c, h);
if (isnan_d(d)) return fv_s((uint32_t)(d >> 63) << 31 | 0x7fc00000u | (uint32_t)((d >> 29) & 0x3fffff));
return (float)fv_d(d);
}
// ---- scalar FP instructions (FP data processing)
#define FP_SETS(d, x) VSET(d, fb_s(x))
#define FP_SETD(d, x) VSET(d, fb_d(x))
SI void fp_op_s(Cpu *c, int op, int d, int n, int m) { FP_SETS(d, op_s(c, op, VSF32(n), VSF32(m), 0)); }
SI void fp_op_d(Cpu *c, int op, int d, int n, int m) { FP_SETD(d, op_d(c, op, VSF64(n), VSF64(m), 0)); }
// FMADD family: neg_a / neg_n negate the addend / first factor before the fused op
SI void fp_fma_s(Cpu *c, int na, int nn, int d, int n, int m, int a) {
float x = VSF32(a), y = VSF32(n);
if (na) x = neg_s(x);
if (nn) y = neg_s(y);
FP_SETS(d, op_s(c, FOP_MADD, x, y, VSF32(m)));
}
SI void fp_fma_d(Cpu *c, int na, int nn, int d, int n, int m, int a) {
double x = VSF64(a), y = VSF64(n);
if (na) x = neg_d(x);
if (nn) y = neg_d(y);
FP_SETD(d, op_d(c, FOP_MADD, x, y, VSF64(m)));
}
// ---- integer lanes (complex ops run lane by lane; constant sizes fold to plain loads/stores)
SI uint64_t lmask(int sz) { return sz == 3 ? ~0ull : (1ull << (8 << sz)) - 1; }
SI int64_t lsx(uint64_t v, int sz) { int s = 64 - (8 << sz); return (int64_t)(v << s) >> s; }
SI uint64_t lget(const u128 *v, int sz, int i) {
const uint8_t *p = (const uint8_t *)v + (i << sz);
switch (sz) {
case 0: return *p;
case 1: return *(const sa16 *)p;
case 2: return *(const sa32 *)p;
default: return *(const sa64 *)p;
}
}
SI void lset(u128 *v, int sz, int i, uint64_t x) {
uint8_t *p = (uint8_t *)v + (i << sz);
switch (sz) {
case 0: *p = (uint8_t)x; break;
case 1: *(sa16 *)p = (uint16_t)x; break;
case 2: *(sa32 *)p = (uint32_t)x; break;
default: *(sa64 *)p = x; break;
}
}
SI uint64_t sat_s(__int128 v, int sz) {
__int128 hi = ((__int128)1 << ((8 << sz) - 1)) - 1, lo = -hi - 1;
return (uint64_t)(v > hi ? hi : v < lo ? lo : v) & lmask(sz);
}
SI uint64_t sat_u(__int128 v, int sz) {
__int128 hi = ((__int128)1 << (8 << sz)) - 1;
return (uint64_t)(v > hi ? hi : v < 0 ? 0 : v);
}
SI uint64_t sat_uu(unsigned __int128 v, int sz) { return v > lmask(sz) ? lmask(sz) : (uint64_t)v; }
SI __int128 shl128(__int128 v, int s) { return (__int128)((unsigned __int128)v << s); }
// right shift by s >= 1 with optional rounding, as an infinite-precision value
SI __int128 lshr_s(int64_t v, int s, int rnd) {
if (s > 100) s = 100;
return rnd ? ((__int128)v + ((__int128)1 << (s - 1))) >> s : (__int128)v >> s;
}
SI unsigned __int128 lshr_u(uint64_t v, int s, int rnd) {
if (s > 100) s = 100;
return rnd ? ((unsigned __int128)v + ((unsigned __int128)1 << (s - 1))) >> s : (unsigned __int128)v >> s;
}
SI uint64_t pmul64(uint64_t a, uint64_t b, uint64_t *hi) { // carry-less
uint64_t lo = 0, h = 0;
for (int i = 0; i < 64; i++)
if ((b >> i) & 1) lo ^= a << i, h ^= i ? a >> (64 - i) : 0;
if (hi) *hi = h;
return lo;
}
enum {
LOP_SQADD, LOP_UQADD, LOP_SQSUB, LOP_UQSUB, LOP_SUQADD, LOP_USQADD,
LOP_SHL, LOP_RSHL, LOP_QSHL, LOP_QRSHL, // register shifts (b = signed byte), U selects unsigned
LOP_SHLI, LOP_QSHLI, LOP_QSHLUI, LOP_SHRI, LOP_RSHRI, LOP_SRAI, LOP_RSRAI, LOP_SRI, LOP_SLI,
LOP_SQDMULH, LOP_SQRDMULH, LOP_PMUL,
LOP_SQABS, LOP_SQNEG, LOP_ABS, LOP_NEG, LOP_CLS, LOP_CLZ, LOP_CNT, LOP_RBIT, LOP_URECPE, LOP_URSQRTE,
LOP_HADD, LOP_RHADD, LOP_HSUB, LOP_ABD, LOP_ABA,
// long: result lanes sz+1
LOP_ADDL, LOP_SUBL, LOP_ABDL, LOP_ABAL, LOP_MLAL, LOP_MLSL, LOP_MULL, LOP_ADDW, LOP_SUBW,
LOP_SQDMULL, LOP_SQDMLAL, LOP_SQDMLSL, LOP_PMULL, LOP_SHLL,
// narrow: sz = destination lane size, a/b are sz+1
LOP_XTN, LOP_SQXTN, LOP_UQXTN, LOP_SQXTUN, LOP_SHRN, LOP_RSHRN, LOP_QSHRN, LOP_QRSHRN, LOP_QSHRUN,
LOP_QRSHRUN, LOP_ADDHN, LOP_RADDHN, LOP_SUBHN, LOP_RSUBHN,
LOP_SHLLI, // SSHLL/USHLL: extend, shift (long)
};
#define LOP_U 256 // unsigned variant flag
SI uint64_t lop(int op, int sz, uint64_t a, uint64_t b, uint64_t d) {
int u = op >> 8, es = 8 << sz;
op &= 255;
int64_t sa = lsx(a, sz), sb = lsx(b, sz);
__int128 xa = u ? (__int128)a : (__int128)sa, xb = u ? (__int128)b : (__int128)sb; // extended operands
int sh = (int8_t)b;
switch (op) {
case LOP_SQADD: return sat_s((__int128)sa + sb, sz);
case LOP_UQADD: return sat_u((__int128)a + b, sz);
case LOP_SQSUB: return sat_s((__int128)sa - sb, sz);
case LOP_UQSUB: return sat_u((__int128)a - b, sz);
case LOP_SUQADD: return sat_s((__int128)lsx(d, sz) + a, sz);
case LOP_USQADD: return sat_u((__int128)d + sa, sz);
case LOP_SHL: case LOP_RSHL: case LOP_QSHL: case LOP_QRSHL: {
int sat = op == LOP_QSHL || op == LOP_QRSHL, rnd = op == LOP_RSHL || op == LOP_QRSHL;
if (sh >= 0) {
if (sat) return u ? sat_uu((unsigned __int128)a << (sh > 64 ? 64 : sh), sz)
: sat_s(shl128(sa, sh > 64 ? 64 : sh), sz);
return sh >= es ? 0 : a << sh;
}
return u ? (uint64_t)lshr_u(a, -sh, rnd) : (uint64_t)lshr_s(sa, -sh, rnd);
}
case LOP_SHLI: return a << b;
case LOP_QSHLI: return u ? sat_uu((unsigned __int128)a << b, sz) : sat_s(shl128(sa, (int)b), sz);
case LOP_QSHLUI: return sat_u(shl128(sa, (int)b), sz);
case LOP_SHRI: return u ? (uint64_t)lshr_u(a, (int)b, 0) : (uint64_t)lshr_s(sa, (int)b, 0);
case LOP_RSHRI: return u ? (uint64_t)lshr_u(a, (int)b, 1) : (uint64_t)lshr_s(sa, (int)b, 1);
case LOP_SRAI: return d + (u ? (uint64_t)lshr_u(a, (int)b, 0) : (uint64_t)lshr_s(sa, (int)b, 0));
case LOP_RSRAI: return d + (u ? (uint64_t)lshr_u(a, (int)b, 1) : (uint64_t)lshr_s(sa, (int)b, 1));
case LOP_SRI: {
uint64_t m = (int)b >= es ? 0 : lmask(sz) >> b;
return (d & ~m) | ((int)b >= es ? 0 : a >> b);
}
case LOP_SLI: { uint64_t m = (lmask(sz) << b) & lmask(sz); return (d & ~m) | ((a << b) & m); }
case LOP_SQDMULH: return sat_s(((__int128)sa * sb * 2) >> es, sz);
case LOP_SQRDMULH: return sat_s(((__int128)sa * sb * 2 + ((__int128)1 << (es - 1))) >> es, sz);
case LOP_PMUL: return pmul64(a, b, 0);
case LOP_SQABS: return sat_s(sa < 0 ? -(__int128)sa : sa, sz);
case LOP_SQNEG: return sat_s(-(__int128)sa, sz);
case LOP_ABS: return sa < 0 ? (uint64_t)-sa : (uint64_t)sa;
case LOP_NEG: return -a;
case LOP_CLS: {
uint64_t x = (uint64_t)(sa ^ (sa >> 1)) & lmask(sz);
return (x ? (uint64_t)__builtin_clzll(x << (64 - es)) : (uint64_t)es) - 1;
}
case LOP_CLZ: return a ? (uint64_t)__builtin_clzll(a << (64 - es)) : (uint64_t)es;
case LOP_CNT: return (uint64_t)__builtin_popcountll(a);
case LOP_RBIT: return __builtin_bitreverse8((uint8_t)a);
case LOP_URECPE: return (a >> 31) ? (uint64_t)(256 | simd_recip_tab[(a >> 23) & 255]) << 23 : 0xffffffffull;
case LOP_URSQRTE: return (a >> 30) ? (uint64_t)(256 | simd_rsqrt_tab[((a >> 23) & 511) - 128]) << 23 : 0xffffffffull;
case LOP_HADD: return (uint64_t)((xa + xb) >> 1);
case LOP_RHADD: return (uint64_t)((xa + xb + 1) >> 1);
case LOP_HSUB: return (uint64_t)((xa - xb) >> 1);
case LOP_ABD: return (uint64_t)(xa > xb ? xa - xb : xb - xa);
case LOP_ABA: return d + (uint64_t)(xa > xb ? xa - xb : xb - xa);
case LOP_ADDL: return (uint64_t)(xa + xb);
case LOP_SUBL: return (uint64_t)(xa - xb);
case LOP_ABDL: return (uint64_t)(xa > xb ? xa - xb : xb - xa);
case LOP_ABAL: return d + (uint64_t)(xa > xb ? xa - xb : xb - xa);
case LOP_MLAL: return d + (uint64_t)(xa * xb);
case LOP_MLSL: return d - (uint64_t)(xa * xb);
case LOP_MULL: return (uint64_t)(xa * xb);
case LOP_ADDW: return a + (uint64_t)xb;
case LOP_SUBW: return a - (uint64_t)xb;
case LOP_SQDMULL: return sat_s((__int128)sa * sb * 2, sz + 1);
case LOP_SQDMLAL: return sat_s((__int128)lsx(d, sz + 1) + (__int128)lsx(sat_s((__int128)sa * sb * 2, sz + 1), sz + 1), sz + 1);
case LOP_SQDMLSL: return sat_s((__int128)lsx(d, sz + 1) - (__int128)lsx(sat_s((__int128)sa * sb * 2, sz + 1), sz + 1), sz + 1);
case LOP_PMULL: return pmul64(a, b, 0);
case LOP_SHLL: return a << es;
// narrowing: a (and b for xHN) are 2*es wide
case LOP_XTN: return a;
case LOP_SQXTN: return sat_s(lsx(a, sz + 1), sz);
case LOP_UQXTN: return sat_u(a, sz);
case LOP_SQXTUN: return sat_u(lsx(a, sz + 1), sz);
case LOP_SHRN: return a >> b;
case LOP_RSHRN: return (uint64_t)lshr_u(a, (int)b, 1);
case LOP_QSHRN: case LOP_QRSHRN: {
int rnd = op == LOP_QRSHRN;
return u ? sat_u((__int128)lshr_u(a, (int)b, rnd), sz) : sat_s(lshr_s(lsx(a, sz + 1), (int)b, rnd), sz);
}
case LOP_QSHRUN: return sat_u(lshr_s(lsx(a, sz + 1), (int)b, 0), sz);
case LOP_QRSHRUN: return sat_u(lshr_s(lsx(a, sz + 1), (int)b, 1), sz);
case LOP_ADDHN: return (a + b) >> es;
case LOP_RADDHN: return (a + b + (1ull << (es - 1))) >> es;
case LOP_SUBHN: return (a - b) >> es;
case LOP_RSUBHN: return (a - b + (1ull << (es - 1))) >> es;
case LOP_SHLLI: return (uint64_t)xa << b;
}
return 0;
}
// Generic lane loop: ne lanes; result lane i (size szd) goes to lane offd+i of d (other lanes kept when keep,
// else zero); a = n lane offn+i (size szn), b = m lane offm+i (size szm), or lane bidx of m when bidx >= 0,
// or the constant bimm when m < 0; the accumulator is d's old lane offd+i.
SI void sl_gen(Cpu *c, int op, int sz, int ne, int d, int szd, int offd, int keep, int n, int szn, int offn, int m,
int szm, int offm, int bidx, uint64_t bimm) {
u128 a = c->v[n], b = m >= 0 ? c->v[m] : 0, o = c->v[d], r = keep ? o : 0;
for (int i = 0; i < ne; i++) {
uint64_t y = m < 0 ? bimm : lget(&b, szm, bidx >= 0 ? bidx : offm + i);
lset(&r, szd, offd + i, lop(op, sz, lget(&a, szn, offn + i), y, lget(&o, szd, offd + i)));
}
VSET128(d, r);
}
// SADDLP/UADDLP/SADALP/UADALP
SI void sl_addlp(Cpu *c, int u, int acc, int Q, int sz, int d, int n) {
u128 a = c->v[n], o = c->v[d], r = 0;
for (int i = 0; i < (4 << Q) >> sz; i++) {
uint64_t x = lget(&a, sz, 2 * i), y = lget(&a, sz, 2 * i + 1);
uint64_t s = u ? x + y : (uint64_t)(lsx(x, sz) + lsx(y, sz));
lset(&r, sz + 1, i, s + (acc ? lget(&o, sz + 1, i) : 0));
}
VSET128(d, r);
}
// ADDV, SADDLV/UADDLV, SMAXV/UMAXV, SMINV/UMINV (op 0..3; u = unsigned)
SI void sl_across(Cpu *c, int op, int u, int Q, int sz, int d, int n) {
u128 a = c->v[n];
int ne = (8 << Q) >> sz;
int64_t acc = u ? (int64_t)lget(&a, sz, 0) : lsx(lget(&a, sz, 0), sz);
for (int i = 1; i < ne; i++) {
int64_t x = u ? (int64_t)lget(&a, sz, i) : lsx(lget(&a, sz, i), sz);
if (op <= 1) acc += x;
else if (op == 2) acc = u ? ((uint64_t)x > (uint64_t)acc ? x : acc) : (x > acc ? x : acc);
else acc = u ? ((uint64_t)x < (uint64_t)acc ? x : acc) : (x < acc ? x : acc);
}
VSET(d, (uint64_t)acc & lmask(op == 1 ? sz + 1 : sz));
}
// PMULL2/PMULL 1Q (64x64 -> 128 carry-less)
SI void sl_pmull64(Cpu *c, int part, int d, int n, int m) {
uint64_t hi, lo = pmul64(((sa64 *)&c->v[n])[part], ((sa64 *)&c->v[m])[part], &hi);
VSET128(d, ((u128)hi << 64) | lo);
}
// ---- FP vector helpers: lanes of the 64-bit form are computed on a zero upper half
SI vf32 vsplat_s(vf32 v, int i, int Q) { float x = v[i]; return Q ? (vf32){x, x, x, x} : (vf32){x, x, 0, 0}; }
SI vf64 vsplat_d(vf64 v, int i, int Q) { double x = v[i]; (void)Q; return (vf64){x, x}; }
#define SV_FOP_S(op, Q, d, n, m, a) VW(vf32, Q, d, vop_s(c, op, n, m, a))
#define SV_FOP_D(op, Q, d, n, m, a) VW(vf64, Q, d, vop_d(c, op, n, m, a))
SI vf32 vneg_s(vf32 v) { return (vf32)((vu32)v ^ 0x80000000u); }
SI vf64 vneg_d(vf64 v) { return (vf64)((vu64)v ^ 0x8000000000000000ull); }
SI float pair_s(Cpu *c, int op, float a, float b) { return op_s(c, op, a, b, 0); }
SI double pair_d(Cpu *c, int op, double a, double b) { return op_d(c, op, a, b, 0); }
// FMAXV/FMINV/FMAXNMV/FMINNMV (4S): pairwise tree
SI void sv_fred_s(Cpu *c, int op, int d, int n) {
vf32 v = VREG(vf32, n);
VSET(d, fb_s(pair_s(c, op, pair_s(c, op, v[0], v[1]), pair_s(c, op, v[2], v[3]))));
}
// lane-wise FP conversions
SI vu32 vtoint_s(Cpu *c, vf32 v, int rm, int fbits, int uns) {
vu32 r;
for (int i = 0; i < 4; i++) r[i] = (uint32_t)toint_s(c, v[i], rm, fbits, 32, uns);
return r;
}
SI vu64 vtoint_d(Cpu *c, vf64 v, int rm, int fbits, int uns) {
vu64 r;
for (int i = 0; i < 2; i++) r[i] = toint_d(c, v[i], rm, fbits, 64, uns);
return r;
}
SI vf32 vfromint_s(Cpu *c, vu32 v, int fbits, int uns) {
vf32 r;
if (!(c->fpcr & FPCR_RM)) {
r = uns ? CVT(v, vf32) : CVT((vs32)v, vf32);
if (fbits) r = r / (float)(1ull << (fbits - 1)) / 2.0f;
return r;
}
for (int i = 0; i < 4; i++) r[i] = fromint_s(c, v[i], 32, uns, fbits);
return r;
}
SI vf64 vfromint_d(Cpu *c, vu64 v, int fbits, int uns) {
vf64 r;
for (int i = 0; i < 2; i++) r[i] = fromint_d(c, v[i], 64, uns, fbits);
return r;
}
SI vf32 vrecpe_s(Cpu *c, vf32 v) { for (int i = 0; i < 4; i++) v[i] = recpe_s(c, v[i]); return v; }
SI vf64 vrecpe_d(Cpu *c, vf64 v) { for (int i = 0; i < 2; i++) v[i] = recpe_d(c, v[i]); return v; }
SI vf32 vrsqrte_s(Cpu *c, vf32 v) { for (int i = 0; i < 4; i++) v[i] = rsqrte_s(c, v[i]); return v; }
SI vf64 vrsqrte_d(Cpu *c, vf64 v) { for (int i = 0; i < 2; i++) v[i] = rsqrte_d(c, v[i]); return v; }
// FCVTL/FCVTL2 (h->s, s->d), FCVTN/FCVTN2 (s->h, d->s), FCVTXN/FCVTXN2
SI void sv_fcvtl(Cpu *c, int sz, int part, int d, int n) {
u128 a = c->v[n], r = 0;
if (sz == 0) {
vf32 o;
for (int i = 0; i < 4; i++) o[i] = simd_h2s(c, (uint16_t)lget(&a, 1, 4 * part + i));
r = (u128)o;
} else {
vf64 o;
for (int i = 0; i < 2; i++) o[i] = simd_s2d(c, fv_s((uint32_t)lget(&a, 2, 2 * part + i)));
r = (u128)o;
}
VSET128(d, r);
}
SI void sv_fcvtn(Cpu *c, int sz, int part, int odd, int d, int n) {
u128 a = c->v[n], r = part ? (uint64_t)c->v[d] : 0;
if (sz == 0) {
for (int i = 0; i < 4; i++) lset(&r, 1, 4 * part + i, simd_s2h(c, fv_s((uint32_t)lget(&a, 2, i))));
} else {
for (int i = 0; i < 2; i++) {
double x = fv_d(lget(&a, 3, i));
lset(&r, 2, 2 * part + i, fb_s(odd ? simd_d2s_odd(c, x) : simd_d2s(c, x)));
}
}
VSET128(d, r);
}
// TBL / TBX with len+1 table registers
SI void sv_tbl(Cpu *c, int Q, int len, int tbx, int d, int n, int m) {
uint8_t tab[64];
for (int i = 0; i <= len; i++) memcpy(tab + 16 * i, &c->v[(n + i) & 31], 16);
u128 idx = c->v[m], o = c->v[d], r = 0;
for (int i = 0; i < 8 << Q; i++) {
unsigned k = (unsigned)lget(&idx, 0, i);
lset(&r, 0, i, k < 16u * (len + 1) ? tab[k] : tbx ? lget(&o, 0, i) : 0);
}
VSET128(d, r);
}
// ---- CRC32 / CRC32C (bytes = 1, 2, 4, 8)
SI uint32_t simd_crc(const uint32_t *t, uint32_t acc, uint64_t v, int bytes) {
for (int i = 0; i < bytes; i++) acc = t[(acc ^ (uint8_t)(v >> (8 * i))) & 0xff] ^ (acc >> 8);
return acc;
}
#define CRC32(acc, v, n) simd_crc(simd_crc32_tab, (uint32_t)(acc), (v), n)
#define CRC32C(acc, v, n) simd_crc(simd_crc32c_tab, (uint32_t)(acc), (v), n)
// ---- crypto (AES, SHA1, SHA256)
SI uint8_t aes_xt(uint8_t x) { return (uint8_t)((x << 1) ^ ((x >> 7) * 0x1b)); }
SI uint8_t aes_mul(uint8_t x, int k) { // k in {9, 11, 13, 14}
uint8_t x2 = aes_xt(x), x4 = aes_xt(x2), x8 = aes_xt(x4);
return (uint8_t)(x8 ^ (k & 4 ? x4 : 0) ^ (k & 2 ? x2 : 0) ^ (k & 1 ? x : 0));
}
SI void sv_aes(Cpu *c, int op, int d, int n) { // 0 AESE, 1 AESD, 2 AESMC, 3 AESIMC
uint8_t s[16], r[16];
u128 v = op < 2 ? c->v[d] ^ c->v[n] : c->v[n];
memcpy(s, &v, 16);
if (op == 0) {
for (int i = 0; i < 16; i++) r[i] = simd_aes_sbox[s[(i + 4 * (i & 3)) & 15]];
} else if (op == 1) {
for (int i = 0; i < 16; i++) r[i] = simd_aes_isbox[s[(i - 4 * (i & 3)) & 15]];
} else {
for (int col = 0; col < 4; col++) {
uint8_t *a = s + 4 * col, *o = r + 4 * col;
for (int i = 0; i < 4; i++) {
uint8_t a0 = a[i], a1 = a[(i + 1) & 3], a2 = a[(i + 2) & 3], a3 = a[(i + 3) & 3];
o[i] = op == 2 ? (uint8_t)(aes_xt(a0) ^ aes_xt(a1) ^ a1 ^ a2 ^ a3)
: (uint8_t)(aes_mul(a0, 14) ^ aes_mul(a1, 11) ^ aes_mul(a2, 13) ^ aes_mul(a3, 9));
}
}
}
memcpy(&v, r, 16);
VSET128(d, v);
}
SI uint32_t rol32_(uint32_t x, int s) { return (x << s) | (x >> (32 - s)); }
SI uint32_t ror32_(uint32_t x, int s) { return (x >> s) | (x << (32 - s)); }
SI void sv_sha1hash(Cpu *c, int op, int d, int n, int m) { // 0 SHA1C, 1 SHA1P, 2 SHA1M
uint32_t x[4], w[4], y = (uint32_t)c->v[n];
memcpy(x, &c->v[d], 16);
memcpy(w, &c->v[m], 16);
for (int e = 0; e < 4; e++) {
uint32_t t = op == 0 ? (((x[2] ^ x[3]) & x[1]) ^ x[3]) : op == 1 ? (x[1] ^ x[2] ^ x[3])
: ((x[1] & x[2]) | ((x[1] | x[2]) & x[3]));
y = y + rol32_(x[0], 5) + t + w[e];
x[1] = rol32_(x[1], 30);
uint32_t top = x[3];
x[3] = x[2], x[2] = x[1], x[1] = x[0], x[0] = y, y = top;
}
memcpy(&c->v[d], x, 16);
}
SI void sv_sha256hash(Cpu *c, int part1, int d, int n, int m) { // SHA256H (part1), SHA256H2
uint32_t x[4], y[4], w[4];
memcpy(part1 ? x : y, &c->v[d], 16);
memcpy(part1 ? y : x, &c->v[n], 16);
memcpy(w, &c->v[m], 16);
for (int e = 0; e < 4; e++) {
uint32_t chs = ((y[1] ^ y[2]) & y[0]) ^ y[2];
uint32_t maj = (x[0] & x[1]) | ((x[0] | x[1]) & x[2]);
uint32_t t1 = y[3] + (ror32_(y[0], 6) ^ ror32_(y[0], 11) ^ ror32_(y[0], 25)) + chs + w[e];
x[3] = t1 + x[3];
y[3] = t1 + (ror32_(x[0], 2) ^ ror32_(x[0], 13) ^ ror32_(x[0], 22)) + maj;
uint32_t t = y[3];
y[3] = y[2], y[2] = y[1], y[1] = y[0], y[0] = x[3];
x[3] = x[2], x[2] = x[1], x[1] = x[0], x[0] = t;
}
memcpy(&c->v[d], part1 ? x : y, 16);
}
SI void sv_sha_misc(Cpu *c, int op, int d, int n, int m) { // 0 SHA1SU0, 1 SHA1SU1, 2 SHA256SU0, 3 SHA256SU1, 4 SHA1H
uint32_t a[4], b[4], k[4], r[4], t[4];
memcpy(a, &c->v[d], 16);
memcpy(b, &c->v[n], 16);
memcpy(k, &c->v[m], 16);
switch (op) {
case 0:
t[0] = a[2], t[1] = a[3], t[2] = b[0], t[3] = b[1];
for (int i = 0; i < 4; i++) r[i] = t[i] ^ a[i] ^ k[i];
break;
case 1:
for (int i = 0; i < 4; i++) t[i] = a[i] ^ (i < 3 ? b[i + 1] : 0);
for (int i = 0; i < 4; i++) r[i] = rol32_(t[i], 1);
r[3] ^= rol32_(t[0], 2);
break;
case 2:
t[0] = a[1], t[1] = a[2], t[2] = a[3], t[3] = b[0];
for (int i = 0; i < 4; i++) r[i] = (ror32_(t[i], 7) ^ ror32_(t[i], 18) ^ (t[i] >> 3)) + a[i];
break;
case 3: {
uint32_t t0[4] = {b[1], b[2], b[3], k[0]};
for (int e = 0; e < 4; e++) {
uint32_t x = e < 2 ? k[e + 2] : r[e - 2];
r[e] = (ror32_(x, 17) ^ ror32_(x, 19) ^ (x >> 10)) + a[e] + t0[e];
}
break;
}
default:
VSET(d, rol32_(b[0], 30));
return;
}
memcpy(&c->v[d], r, 16);
}
// ---- loads / stores of lanes
SI uint64_t simd_ld(uint64_t a, int sz) {
switch (sz) {
case 0: return LD8(a);
case 1: return LD16(a);
case 2: return LD32(a);
default: return LD64(a);
}
}
SI void simd_st(uint64_t a, int sz, uint64_t v) {
switch (sz) {
case 0: ST8(a, v); break;
case 1: ST16(a, v); break;
case 2: ST32(a, v); break;
default: ST64(a, v); break;
}
}
// LDn/STn (multiple structures): selem interleaved registers, rpt repeats (LD1 with 2-4 registers)
SI void sl_ldm(Cpu *c, int Q, int sz, int selem, int rpt, int t, uint64_t ea) {
u128 r[4] = {0, 0, 0, 0};
int ne = (8 << Q) >> sz;
for (int k = 0; k < rpt; k++)
for (int e = 0; e < ne; e++)
for (int s = 0; s < selem; s++, ea += 1 << sz) lset(&r[k + s], sz, e, simd_ld(ea, sz));
for (int i = 0; i < rpt + selem - 1; i++) VSET128((t + i) & 31, r[i]);
}
SI void sl_stm(Cpu *c, int Q, int sz, int selem, int rpt, int t, uint64_t ea) {
u128 r[4];
int ne = (8 << Q) >> sz;
for (int i = 0; i < rpt + selem - 1; i++) r[i] = c->v[(t + i) & 31];
for (int k = 0; k < rpt; k++)
for (int e = 0; e < ne; e++)
for (int s = 0; s < selem; s++, ea += 1 << sz) simd_st(ea, sz, lget(&r[k + s], sz, e));
}
// single structure: lane idx of selem registers; LDnR replicates into every lane
SI void sl_lds(Cpu *c, int sz, int selem, int idx, int t, uint64_t ea) {
for (int s = 0; s < selem; s++) {
uint64_t x = simd_ld(ea + (s << sz), sz);
switch (sz) {
case 0: VINS(vu8, (t + s) & 31, idx, (uint8_t)x); break;
case 1: VINS(vu16, (t + s) & 31, idx, (uint16_t)x); break;
case 2: VINS(vu32, (t + s) & 31, idx, (uint32_t)x); break;
default: VINS(vu64, (t + s) & 31, idx, x); break;
}
}
}
SI void sl_sts(Cpu *c, int sz, int selem, int idx, int t, uint64_t ea) {
for (int s = 0; s < selem; s++) simd_st(ea + (s << sz), sz, lget(&c->v[(t + s) & 31], sz, idx));
}
SI void sl_ldr(Cpu *c, int Q, int sz, int selem, int t, uint64_t ea) {
uint64_t x[4];
for (int s = 0; s < selem; s++) x[s] = simd_ld(ea + (s << sz), sz);
for (int s = 0; s < selem; s++) {
u128 v = 0;
for (int e = 0; e < (8 << Q) >> sz; e++) lset(&v, sz, e, x[s]);
VSET128((t + s) & 31, v);
}
}
Reference in New Issue View Git Blame Copy Permalink