From c4268779fc3125438e3ca020ade6680645750ce4 Mon Sep 17 00:00:00 2001 From: chrg6 Date: Thu, 12 Nov 2026 20:50:19 -0500 Subject: [PATCH] Write a V register with one 16-byte store. A scalar result (an FP operation, a scalar load, FMOV, a compare, a conversion, a 64-bit vector operation) was assigned to the register as a zero-extended 128-bit integer, which the compiler writes as two 8-byte stores, and INS and a single-lane load wrote their lane alone. The next 16-byte load of the register then cannot take its value from the store buffer and waits until the stores have left it. Every such write now builds the whole value in a vector register and stores it once (runtime/simd.h): VSET puts a scalar in the low bits with zero above it, VSET128 writes a 128-bit value built in an integer, and VINS loads the register, replaces one lane and stores all of it. VINS passes the value through an empty asm statement, without which the compiler turns the load, the insert and the store back into a store of the lane. In the 360 guest functions a play profile sampled, stores of a part of a V register fall from 64322 to 4 and whole stores rise from 10932 to 43665; the four left copy a 16-byte guest load that the compiler itself splits because the memory was just written in halves. Host instructions fall from 1448075 to 1427163 and stores to the Cpu from 177160 to 145566; its loads rise from 141090 to 143521, as an insert loads the register it changes. The lifted text of FP and SIMD code changes with this, so those files are compiled again. Co-Authored-By: Claude Fable 5.2 --- recomp/simd_c.py | 73 ++++++++++++++++++++++++++---------------------- runtime/simd.h | 56 +++++++++++++++++++++++++------------ 2 files changed, 78 insertions(+), 51 deletions(-) diff --git a/recomp/simd_c.py b/recomp/simd_c.py index 7787a544..0cb74a75 100644 --- a/recomp/simd_c.py +++ b/recomp/simd_c.py @@ -38,6 +38,12 @@ def VWr(T, Q, d, e): return f'VW({T}, {Q}, {d}, {e});' +def vset(d, e, full=False): + """C that writes V register d whole (runtime/simd.h VSET, VSET128): e is a scalar of at most 64 bits, which leaves + the bits above it zero, or with full a 128-bit value.""" + return f'VSET128({d}, {e});' if full else f'VSET({d}, {e});' + + def cond(c): return f'COND_{COND_NAMES[c].upper()}(c)' if c < 14 else '1' @@ -248,7 +254,7 @@ class SimdC: if sz > 3: raise Unsupported('scalar dup size') d, n = bits(w, 0, 5), bits(w, 5, 5) - return f'c->v[{d}] = (({("uint8_t", "sa16", "sa32", "sa64")[sz]} *)&c->v[{n}])[{idx}];' + return f'VSET({d}, (({("uint8_t", "sa16", "sa32", "sa64")[sz]} *)&c->v[{n}])[{idx}]);' def copy(self, w): Q, op, imm5, imm4 = bits(w, 30, 1), bits(w, 29, 1), bits(w, 16, 5), bits(w, 11, 4) @@ -261,7 +267,7 @@ class SimdC: if not Q: raise Unsupported('ins elem Q') idx2 = imm4 >> sz - return f'(({P} *)&c->v[{d}])[{idx}] = (({P} *)&c->v[{n}])[{idx2}];' + return f'VINS({vu(sz)}, {d}, {idx}, (({P} *)&c->v[{n}])[{idx2}]);' if imm4 == 0: # DUP (element) if sz == 3 and not Q: raise Unsupported('dup 1d') @@ -274,7 +280,7 @@ class SimdC: if imm4 == 3: # INS (general) if not Q: raise Unsupported('ins Q') - return f'(({P} *)&c->v[{d}])[{idx}] = ({ut(sz)}){X(n)};' + return f'VINS({vu(sz)}, {d}, {idx}, ({ut(sz)}){X(n)});' if imm4 == 5: # SMOV if sz >= 2 + Q: raise Unsupported('smov size') @@ -424,15 +430,15 @@ class SimdC: sa, sb = f'(int64_t)VS64({n})', f'(int64_t)VS64({m})' x, y = (a, b) if U else (sa, sb) if opc == 6: - return f'c->v[{d}] = ({x} > {y}) ? ~0ull : 0;' + return f'VSET({d}, ({x} > {y}) ? ~0ull : 0);' if opc == 7: - return f'c->v[{d}] = ({x} >= {y}) ? ~0ull : 0;' + return f'VSET({d}, ({x} >= {y}) ? ~0ull : 0);' if opc in (8, 10): return same(lop('SHL' if opc == 8 else 'RSHL', U), 3, 1, d, n, m) if opc == 16: - return f'c->v[{d}] = (uint64_t)({a} {"-" if U else "+"} {b});' + return f'VSET({d}, (uint64_t)({a} {"-" if U else "+"} {b}));' if opc == 17: - return f'c->v[{d}] = ({a} == {b}) ? ~0ull : 0;' if U else f'c->v[{d}] = ({a} & {b}) ? ~0ull : 0;' + return f'VSET({d}, ({a} == {b}) ? ~0ull : 0);' if U else f'VSET({d}, ({a} & {b}) ? ~0ull : 0);' raise Unsupported('scalar 3same opcode') def three_same_fp(self, w, scalar): @@ -445,8 +451,8 @@ class SimdC: if key in FCMP3: cop = FCMP3[key] if scalar: - return (f'c->v[{d}] = ({ut(2 + sz)})vcmp_{F}(c, {cop}, ({VT}){{{SF}({n})}}, ' - f'({VT}){{{SF}({m})}})[0];') + return (f'VSET({d}, ({ut(2 + sz)})vcmp_{F}(c, {cop}, ({VT}){{{SF}({n})}}, ' + f'({VT}){{{SF}({m})}})[0]);') return VWr(VU, Q, d, f'vcmp_{F}(c, {cop}, {VQ(VT, Q, n)}, {VQ(VT, Q, m)})') if scalar: if key not in ((0, 0, 0b11011), (0, 0, 0b11111), (0, 1, 0b11111), (1, 1, 0b11010)): @@ -586,13 +592,13 @@ class SimdC: raise Unsupported('scalar misc size') a, sa = f'VS64({n})', f'(int64_t)VS64({n})' if opc == 8: - return f'c->v[{d}] = ({sa} {">=" if U else ">"} 0) ? ~0ull : 0;' + return f'VSET({d}, ({sa} {">=" if U else ">"} 0) ? ~0ull : 0);' if opc == 9: - return f'c->v[{d}] = ({sa} {"<=" if U else "=="} 0) ? ~0ull : 0;' + return f'VSET({d}, ({sa} {"<=" if U else "=="} 0) ? ~0ull : 0);' if opc == 10 and not U: - return f'c->v[{d}] = ({sa} < 0) ? ~0ull : 0;' + return f'VSET({d}, ({sa} < 0) ? ~0ull : 0);' if opc == 11: - return f'c->v[{d}] = (uint64_t)-{a};' if U else f'c->v[{d}] = ({sa} < 0) ? (uint64_t)-{a} : {a};' + return f'VSET({d}, (uint64_t)-{a});' if U else f'VSET({d}, ({sa} < 0) ? (uint64_t)-{a} : {a});' raise Unsupported('scalar misc opcode') def misc2_fp(self, w, scalar): @@ -634,7 +640,7 @@ class SimdC: a, b = f'({VT}){{{SF}({n})}}', f'({VT}){{}}' if swap: a, b = b, a - return f'c->v[{d}] = ({ut(2 + sz)})vcmp_{F}(c, {op}, {a}, {b})[0];' + return f'VSET({d}, ({ut(2 + sz)})vcmp_{F}(c, {op}, {a}, {b})[0]);' a, b = x, f'({VT}){{}}' if swap: a, b = b, a @@ -660,7 +666,7 @@ class SimdC: if opc in (0b11010, 0b11011) or (opc == 0b11100 and not a_): # FCVT[NPMZA][SU] rm = {(0b11010, 0): 0, (0b11010, 1): 1, (0b11011, 0): 2, (0b11011, 1): 3, (0b11100, 0): 4}[(opc, a_)] if scalar: - return f'c->v[{d}] = ({ut(2 + sz)})toint_{F}(c, {SF}({n}), {rm}, 0, {32 << sz}, {U});' + return f'VSET({d}, ({ut(2 + sz)})toint_{F}(c, {SF}({n}), {rm}, 0, {32 << sz}, {U}));' return VWr(VU, Q, d, f'vtoint_{F}(c, {x}, {rm}, 0, {U})') if opc == 0b11100: # URECPE / URSQRTE if scalar or sz: @@ -697,7 +703,7 @@ class SimdC: U, sz, opc, n, d = bits(w, 29, 1), bits(w, 22, 2), bits(w, 12, 5), bits(w, 5, 5), bits(w, 0, 5) if not U: if opc == 0b11011 and sz == 3: - return f'c->v[{d}] = (uint64_t)(VS64({n}) + VHI64({n}));' + return f'VSET({d}, (uint64_t)(VS64({n}) + VHI64({n})));' raise Unsupported('scalar pairwise') fop = {(0b01100, 0): 'FOP_MAXNM', (0b01100, 1): 'FOP_MINNM', (0b01101, 0): 'FOP_ADD', (0b01111, 0): 'FOP_MAX', (0b01111, 1): 'FOP_MIN'}.get((opc, sz >> 1)) @@ -725,7 +731,7 @@ class SimdC: if op and not (c4 == 14 or c4 == 15): # MVNI imm = ~imm & ((1 << 64) - 1) v = imm | (imm << 64) if Q else imm - return f'c->v[{d}] = {u128(v)};' + return vset(d, u128(v), v >> 64 != 0) # ------------------------------------------------------------ shift by immediate @@ -754,7 +760,7 @@ class SimdC: return f'{SET}({d}, fromint_{F}(c, {SU}({n}), {es}, {U}, {fb}));' return VWr(VT, Q, d, f'vfromint_{F}(c, {VR(VU, n)}, {fb}, {U})') if scalar: - return f'c->v[{d}] = ({ut(sz)})toint_{F}(c, {SF}({n}), 3, {fb}, {es}, {U});' + return f'VSET({d}, ({ut(sz)})toint_{F}(c, {SF}({n}), 3, {fb}, {es}, {U}));' return VWr(VU, Q, d, f'vtoint_{F}(c, {VQ(VT, Q, n)}, 3, {fb}, {U})') if narrow: if sz == 3: @@ -784,7 +790,7 @@ class SimdC: e = f'({T})({VR(S, n)} >> {min(rsh, es - 1)})' if opc == 2: e = f'{VR(T, d)} + {e}' - return VWr(T, Q, d, e) if not scalar else f'c->v[{d}] = (uint64_t)(({e})[0]);' + return VWr(T, Q, d, e) if not scalar else f'VSET({d}, (uint64_t)(({e})[0]));' if opc in (4, 6): return same(lop('RSHRI' if opc == 4 else 'RSRAI', U), sz, ne, d, n, -1, rsh) if opc == 8: @@ -795,7 +801,7 @@ class SimdC: if U: return same(lop('SLI'), sz, ne, d, n, -1, lsh) e = f'{VR(T, n)} << {lsh}' - return VWr(T, Q, d, e) if not scalar else f'c->v[{d}] = (uint64_t)(({e})[0]);' + return VWr(T, Q, d, e) if not scalar else f'VSET({d}, (uint64_t)(({e})[0]));' if opc == 12: if not U: raise Unsupported('sqshlu U') @@ -905,11 +911,11 @@ class SimdC: F, VT, VU, T, SF, SU, SET = FT[t] sign = '0x80000000u' if not t else '0x8000000000000000ull' if opc == 0: - return f'c->v[{d}] = {SU}({n});' + return f'VSET({d}, {SU}({n}));' if opc == 1: - return f'c->v[{d}] = {SU}({n}) & ~{sign};' + return f'VSET({d}, {SU}({n}) & ~{sign});' if opc == 2: - return f'c->v[{d}] = {SU}({n}) ^ {sign};' + return f'VSET({d}, {SU}({n}) ^ {sign});' if opc == 3: return f'fp_op_{F}(c, FOP_SQRT, {d}, {n}, {n});' if opc in (4, 5, 7): @@ -917,7 +923,7 @@ class SimdC: if to == t: raise Unsupported('fcvt same') if to == 3: - return f'c->v[{d}] = {"simd_s2h" if not t else "simd_dd2h"}(c, {SF}({n}));' + return f'VSET({d}, {"simd_s2h" if not t else "simd_dd2h"}(c, {SF}({n})));' return f'FP_SETD({d}, simd_s2d(c, VSF32({n})));' if to else f'FP_SETS({d}, simd_d2s(c, VSF64({n})));' rm = {8: 0, 9: 1, 10: 2, 11: 3, 12: 4, 14: 5, 15: 6}.get(opc) if rm is None: @@ -943,14 +949,14 @@ class SimdC: t = self._ftype(w) SU = FT[t][5] d, n, m, cnd = bits(w, 0, 5), bits(w, 5, 5), bits(w, 16, 5), bits(w, 12, 4) - return f'c->v[{d}] = {cond(cnd)} ? {SU}({n}) : {SU}({m});' + return f'VSET({d}, {cond(cnd)} ? {SU}({n}) : {SU}({m}));' def fp_imm(self, w): t = self._ftype(w) if bits(w, 5, 5): raise Unsupported('fmov imm5') imm8, d = bits(w, 13, 8), bits(w, 0, 5) - return f'c->v[{d}] = {hx(vfp_expand_imm(imm8, 32 << t))};' + return f'VSET({d}, {hx(vfp_expand_imm(imm8, 32 << t))});' def fp_fixed(self, w): sf, t, rmode, opc, scale, n, d = (bits(w, 31, 1), bits(w, 22, 2), bits(w, 19, 2), bits(w, 16, 3), bits(w, 10, 6), @@ -972,12 +978,12 @@ class SimdC: sf, t, rmode, opc, n, d = bits(w, 31, 1), bits(w, 22, 2), bits(w, 19, 2), bits(w, 16, 3), bits(w, 5, 5), bits(w, 0, 5) if opc in (6, 7): # FMOV if t == 2 and sf and rmode == 1: - return set_reg(1, d, f'VHI64({n})') if opc == 6 else f'VHI64({d}) = {X(n)};' + return set_reg(1, d, f'VHI64({n})') if opc == 6 else f'VINS(vu64, {d}, 1, {X(n)});' if rmode or t != sf or t > 1: raise Unsupported('fmov general') if opc == 6: return set_reg(sf, d, f'VS64({n})' if sf else f'VS32({n})') - return f'c->v[{d}] = {X(n) if sf else W(n)};' + return f'VSET({d}, {X(n) if sf else W(n)});' if t > 1: raise Unsupported('fp int type') F, VT, VU, T, SF, SU, SET = FT[t] @@ -1003,7 +1009,7 @@ class SimdC: if opc == 3: raise Unsupported('ldr literal opc') addr = (pc + sext(bits(w, 5, 19), 19) * 4) & ((1 << 64) - 1) - return f'c->v[{rt}] = {("LD32", "LD64", "LD128")[opc]}({hx(addr)});' + return vset(rt, f'{("LD32", "LD64", "LD128")[opc]}({hx(addr)})', opc == 2) if bits(w, 31, 1) == 0 and bits(w, 24, 6) in (0b001100, 0b001101): return self.ldst_struct(w) if bits(w, 27, 3) == 0b101: @@ -1026,8 +1032,8 @@ class SimdC: sz = 1 << scale lines = [f'uint64_t ea = {ea};'] if L: - lines.append(f'u128 a = {ld}(ea), b = {ld}(ea + {sz});') - lines.append(f'c->v[{rt}] = a; c->v[{rt2}] = b;') + lines.append(f'{"u128" if scale == 4 else "uint64_t"} a = {ld}(ea), b = {ld}(ea + {sz});') + lines.append(f'{vset(rt, "a", scale == 4)} {vset(rt2, "b", scale == 4)}') else: cast = {2: 'VS32', 3: 'VS64', 4: 'c->v'}[scale] v1, v2 = (f'c->v[{rt}]', f'c->v[{rt2}]') if scale == 4 else (f'{cast}({rt})', f'{cast}({rt2})') @@ -1077,7 +1083,7 @@ class SimdC: raise Unsupported('ldst simd') parts = [f'uint64_t ea = {ea};'] if L: - parts.append(f'c->v[{rt}] = {("LD8", "LD16", "LD32", "LD64", "LD128")[scale]}(ea);') + parts.append(vset(rt, f'{("LD8", "LD16", "LD32", "LD64", "LD128")[scale]}(ea)', scale == 4)) else: src = (f'VS8({rt})', f'VS16({rt})', f'VS32({rt})', f'VS64({rt})', f'c->v[{rt}]')[scale] parts.append(f'{("ST8", "ST16", "ST32", "ST64", "ST128")[scale]}(ea, {src});') @@ -1106,7 +1112,8 @@ class SimdC: regs = [(rt + i) % 32 for i in range(rpt)] step = 16 if Q else 8 if L: - acc = ' '.join(f'c->v[{r}] = {"LD128" if Q else "LD64"}(ea + {i * step});' for i, r in enumerate(regs)) + acc = ' '.join(vset(r, f'{"LD128" if Q else "LD64"}(ea + {i * step})', Q) + for i, r in enumerate(regs)) else: acc = ' '.join(f'{"ST128" if Q else "ST64"}(ea + {i * step}, {"c->v[%d]" % r if Q else "VS64(%d)" % r});' for i, r in enumerate(regs)) diff --git a/runtime/simd.h b/runtime/simd.h index aaeaa2ff..c3196230 100644 --- a/runtime/simd.h +++ b/runtime/simd.h @@ -34,9 +34,19 @@ typedef u128 __attribute__((aligned(1), may_alias)) u128u; // ---- registers: VREG = whole V register as a vector type; VQ = source with the upper half cleared when // Q=0; VW = write, zeroing the upper half when Q=0. Scalar views of lane 0: VS8..VS64, VSF32/VSF64. +// A write is one 16-byte store, also where the instruction sets a scalar (the bits above it become zero) or a single +// lane (the other lanes stay): a V register written as two 8-byte halves or as one lane makes the next 16-byte load +// of it wait until those stores have left the store buffer, because the processor forwards a store to a load only +// when the one store covers the load. Assigning a zero-extended u128 to c->v[d] compiles to the two halves, so the +// writes go through a vector type: +// VSET = a scalar of at most 64 bits in the low bits, zero above; VSET128 = all 128 bits of an integer value; +// VINS = lane i replaced by x, the other lanes kept. #define VREG(T, n) (*(T *)&c->v[n]) #define VQ(T, Q, n) ((T)((Q) ? c->v[n] : (u128)(uint64_t)c->v[n])) -#define VW(T, Q, d, e) do { T _w = (e); c->v[d] = (Q) ? (u128)_w : (u128)(uint64_t)(u128)_w; } while (0) +#define VSET(d, x) (VREG(vu64, d) = (vu64){(uint64_t)(x), 0}) +#define VSET128(d, x) (VREG(vu64, d) = (vu64)(u128)(x)) +#define VW(T, Q, d, e) do { T _w = (e); VREG(vu64, d) = (Q) ? (vu64)_w : (vu64){((vu64)_w)[0], 0}; } while (0) +#define VINS(T, d, i, x) do { T _l = VREG(T, d); _l[i] = (x); VREG(vu64, d) = simd_whole((vu64)_l); } while (0) #define VS8(n) (*(uint8_t *)&c->v[n]) #define VS16(n) (*(sa16 *)&c->v[n]) #define VS32(n) (*(sa32 *)&c->v[n]) @@ -44,6 +54,12 @@ typedef u128 __attribute__((aligned(1), may_alias)) u128u; #define VSF32(n) (*(saf32 *)&c->v[n]) #define VSF64(n) (*(saf64 *)&c->v[n]) #define VHI64(n) (((sa64 *)&c->v[n])[1]) +// v, with where it came from hidden from the compiler: a load of a V register, a change of one lane and a store of +// the 16 bytes would otherwise be compiled as a store of that lane alone. +SI vu64 simd_whole(vu64 v) { + __asm__("" : "+x"(v)); + return v; +} #define LD128(a) (*(const u128u *)(uintptr_t)(a)) #define ST128(a, v) (*(u128u *)(uintptr_t)(a) = (u128)(v)) #define SHUF __builtin_shufflevector @@ -384,8 +400,8 @@ SI float simd_h2s(Cpu *c, uint16_t h) { } // ---- scalar FP instructions (FP data processing) -#define FP_SETS(d, x) (c->v[d] = fb_s(x)) -#define FP_SETD(d, x) (c->v[d] = fb_d(x)) +#define FP_SETS(d, x) VSET(d, fb_s(x)) +#define FP_SETD(d, x) VSET(d, fb_d(x)) SI void fp_op_s(Cpu *c, int op, int d, int n, int m) { FP_SETS(d, op_s(c, op, VSF32(n), VSF32(m), 0)); } SI void fp_op_d(Cpu *c, int op, int d, int n, int m) { FP_SETD(d, op_d(c, op, VSF64(n), VSF64(m), 0)); } // FMADD family: neg_a / neg_n negate the addend / first factor before the fused op @@ -568,7 +584,7 @@ SI void sl_gen(Cpu *c, int op, int sz, int ne, int d, int szd, int offd, int kee uint64_t y = m < 0 ? bimm : lget(&b, szm, bidx >= 0 ? bidx : offm + i); lset(&r, szd, offd + i, lop(op, sz, lget(&a, szn, offn + i), y, lget(&o, szd, offd + i))); } - c->v[d] = r; + VSET128(d, r); } // SADDLP/UADDLP/SADALP/UADALP SI void sl_addlp(Cpu *c, int u, int acc, int Q, int sz, int d, int n) { @@ -578,7 +594,7 @@ SI void sl_addlp(Cpu *c, int u, int acc, int Q, int sz, int d, int n) { uint64_t s = u ? x + y : (uint64_t)(lsx(x, sz) + lsx(y, sz)); lset(&r, sz + 1, i, s + (acc ? lget(&o, sz + 1, i) : 0)); } - c->v[d] = r; + VSET128(d, r); } // ADDV, SADDLV/UADDLV, SMAXV/UMAXV, SMINV/UMINV (op 0..3; u = unsigned) SI void sl_across(Cpu *c, int op, int u, int Q, int sz, int d, int n) { @@ -591,12 +607,12 @@ SI void sl_across(Cpu *c, int op, int u, int Q, int sz, int d, int n) { else if (op == 2) acc = u ? ((uint64_t)x > (uint64_t)acc ? x : acc) : (x > acc ? x : acc); else acc = u ? ((uint64_t)x < (uint64_t)acc ? x : acc) : (x < acc ? x : acc); } - c->v[d] = (uint64_t)acc & lmask(op == 1 ? sz + 1 : sz); + VSET(d, (uint64_t)acc & lmask(op == 1 ? sz + 1 : sz)); } // PMULL2/PMULL 1Q (64x64 -> 128 carry-less) SI void sl_pmull64(Cpu *c, int part, int d, int n, int m) { uint64_t hi, lo = pmul64(((sa64 *)&c->v[n])[part], ((sa64 *)&c->v[m])[part], &hi); - c->v[d] = ((u128)hi << 64) | lo; + VSET128(d, ((u128)hi << 64) | lo); } // ---- FP vector helpers: lanes of the 64-bit form are computed on a zero upper half @@ -611,7 +627,7 @@ SI double pair_d(Cpu *c, int op, double a, double b) { return op_d(c, op, a, b, // FMAXV/FMINV/FMAXNMV/FMINNMV (4S): pairwise tree SI void sv_fred_s(Cpu *c, int op, int d, int n) { vf32 v = VREG(vf32, n); - c->v[d] = fb_s(pair_s(c, op, pair_s(c, op, v[0], v[1]), pair_s(c, op, v[2], v[3]))); + VSET(d, fb_s(pair_s(c, op, pair_s(c, op, v[0], v[1]), pair_s(c, op, v[2], v[3])))); } // lane-wise FP conversions SI vu32 vtoint_s(Cpu *c, vf32 v, int rm, int fbits, int uns) { @@ -656,7 +672,7 @@ SI void sv_fcvtl(Cpu *c, int sz, int part, int d, int n) { for (int i = 0; i < 2; i++) o[i] = simd_s2d(c, fv_s((uint32_t)lget(&a, 2, 2 * part + i))); r = (u128)o; } - c->v[d] = r; + VSET128(d, r); } SI void sv_fcvtn(Cpu *c, int sz, int part, int odd, int d, int n) { u128 a = c->v[n], r = part ? (uint64_t)c->v[d] : 0; @@ -668,7 +684,7 @@ SI void sv_fcvtn(Cpu *c, int sz, int part, int odd, int d, int n) { lset(&r, 2, 2 * part + i, fb_s(odd ? simd_d2s_odd(c, x) : simd_d2s(c, x))); } } - c->v[d] = r; + VSET128(d, r); } // TBL / TBX with len+1 table registers @@ -680,7 +696,7 @@ SI void sv_tbl(Cpu *c, int Q, int len, int tbx, int d, int n, int m) { unsigned k = (unsigned)lget(&idx, 0, i); lset(&r, 0, i, k < 16u * (len + 1) ? tab[k] : tbx ? lget(&o, 0, i) : 0); } - c->v[d] = r; + VSET128(d, r); } // ---- CRC32 / CRC32C (bytes = 1, 2, 4, 8) @@ -716,7 +732,7 @@ SI void sv_aes(Cpu *c, int op, int d, int n) { // 0 AESE, 1 AESD, 2 AESMC, 3 AES } } memcpy(&v, r, 16); - c->v[d] = v; + VSET128(d, v); } SI uint32_t rol32_(uint32_t x, int s) { return (x << s) | (x >> (32 - s)); } SI uint32_t ror32_(uint32_t x, int s) { return (x >> s) | (x << (32 - s)); } @@ -779,7 +795,7 @@ SI void sv_sha_misc(Cpu *c, int op, int d, int n, int m) { // 0 SHA1SU0, 1 SHA1S break; } default: - c->v[d] = rol32_(b[0], 30); + VSET(d, rol32_(b[0], 30)); return; } memcpy(&c->v[d], r, 16); @@ -809,7 +825,7 @@ SI void sl_ldm(Cpu *c, int Q, int sz, int selem, int rpt, int t, uint64_t ea) { for (int k = 0; k < rpt; k++) for (int e = 0; e < ne; e++) for (int s = 0; s < selem; s++, ea += 1 << sz) lset(&r[k + s], sz, e, simd_ld(ea, sz)); - for (int i = 0; i < rpt + selem - 1; i++) c->v[(t + i) & 31] = r[i]; + for (int i = 0; i < rpt + selem - 1; i++) VSET128((t + i) & 31, r[i]); } SI void sl_stm(Cpu *c, int Q, int sz, int selem, int rpt, int t, uint64_t ea) { u128 r[4]; @@ -822,9 +838,13 @@ SI void sl_stm(Cpu *c, int Q, int sz, int selem, int rpt, int t, uint64_t ea) { // single structure: lane idx of selem registers; LDnR replicates into every lane SI void sl_lds(Cpu *c, int sz, int selem, int idx, int t, uint64_t ea) { for (int s = 0; s < selem; s++) { - u128 v = c->v[(t + s) & 31]; - lset(&v, sz, idx, simd_ld(ea + (s << sz), sz)); - c->v[(t + s) & 31] = v; + uint64_t x = simd_ld(ea + (s << sz), sz); + switch (sz) { + case 0: VINS(vu8, (t + s) & 31, idx, (uint8_t)x); break; + case 1: VINS(vu16, (t + s) & 31, idx, (uint16_t)x); break; + case 2: VINS(vu32, (t + s) & 31, idx, (uint32_t)x); break; + default: VINS(vu64, (t + s) & 31, idx, x); break; + } } } SI void sl_sts(Cpu *c, int sz, int selem, int idx, int t, uint64_t ea) { @@ -836,6 +856,6 @@ SI void sl_ldr(Cpu *c, int Q, int sz, int selem, int t, uint64_t ea) { for (int s = 0; s < selem; s++) { u128 v = 0; for (int e = 0; e < (8 << Q) >> sz; e++) lset(&v, sz, e, x[s]); - c->v[(t + s) & 31] = v; + VSET128((t + s) & 31, v); } }