diff --git a/recomp/simd_c.py b/recomp/simd_c.py index 7787a544..0cb74a75 100644 --- a/recomp/simd_c.py +++ b/recomp/simd_c.py @@ -38,6 +38,12 @@ def VWr(T, Q, d, e): return f'VW({T}, {Q}, {d}, {e});' +def vset(d, e, full=False): + """C that writes V register d whole (runtime/simd.h VSET, VSET128): e is a scalar of at most 64 bits, which leaves + the bits above it zero, or with full a 128-bit value.""" + return f'VSET128({d}, {e});' if full else f'VSET({d}, {e});' + + def cond(c): return f'COND_{COND_NAMES[c].upper()}(c)' if c < 14 else '1' @@ -248,7 +254,7 @@ class SimdC: if sz > 3: raise Unsupported('scalar dup size') d, n = bits(w, 0, 5), bits(w, 5, 5) - return f'c->v[{d}] = (({("uint8_t", "sa16", "sa32", "sa64")[sz]} *)&c->v[{n}])[{idx}];' + return f'VSET({d}, (({("uint8_t", "sa16", "sa32", "sa64")[sz]} *)&c->v[{n}])[{idx}]);' def copy(self, w): Q, op, imm5, imm4 = bits(w, 30, 1), bits(w, 29, 1), bits(w, 16, 5), bits(w, 11, 4) @@ -261,7 +267,7 @@ class SimdC: if not Q: raise Unsupported('ins elem Q') idx2 = imm4 >> sz - return f'(({P} *)&c->v[{d}])[{idx}] = (({P} *)&c->v[{n}])[{idx2}];' + return f'VINS({vu(sz)}, {d}, {idx}, (({P} *)&c->v[{n}])[{idx2}]);' if imm4 == 0: # DUP (element) if sz == 3 and not Q: raise Unsupported('dup 1d') @@ -274,7 +280,7 @@ class SimdC: if imm4 == 3: # INS (general) if not Q: raise Unsupported('ins Q') - return f'(({P} *)&c->v[{d}])[{idx}] = ({ut(sz)}){X(n)};' + return f'VINS({vu(sz)}, {d}, {idx}, ({ut(sz)}){X(n)});' if imm4 == 5: # SMOV if sz >= 2 + Q: raise Unsupported('smov size') @@ -424,15 +430,15 @@ class SimdC: sa, sb = f'(int64_t)VS64({n})', f'(int64_t)VS64({m})' x, y = (a, b) if U else (sa, sb) if opc == 6: - return f'c->v[{d}] = ({x} > {y}) ? ~0ull : 0;' + return f'VSET({d}, ({x} > {y}) ? ~0ull : 0);' if opc == 7: - return f'c->v[{d}] = ({x} >= {y}) ? ~0ull : 0;' + return f'VSET({d}, ({x} >= {y}) ? ~0ull : 0);' if opc in (8, 10): return same(lop('SHL' if opc == 8 else 'RSHL', U), 3, 1, d, n, m) if opc == 16: - return f'c->v[{d}] = (uint64_t)({a} {"-" if U else "+"} {b});' + return f'VSET({d}, (uint64_t)({a} {"-" if U else "+"} {b}));' if opc == 17: - return f'c->v[{d}] = ({a} == {b}) ? ~0ull : 0;' if U else f'c->v[{d}] = ({a} & {b}) ? ~0ull : 0;' + return f'VSET({d}, ({a} == {b}) ? ~0ull : 0);' if U else f'VSET({d}, ({a} & {b}) ? ~0ull : 0);' raise Unsupported('scalar 3same opcode') def three_same_fp(self, w, scalar): @@ -445,8 +451,8 @@ class SimdC: if key in FCMP3: cop = FCMP3[key] if scalar: - return (f'c->v[{d}] = ({ut(2 + sz)})vcmp_{F}(c, {cop}, ({VT}){{{SF}({n})}}, ' - f'({VT}){{{SF}({m})}})[0];') + return (f'VSET({d}, ({ut(2 + sz)})vcmp_{F}(c, {cop}, ({VT}){{{SF}({n})}}, ' + f'({VT}){{{SF}({m})}})[0]);') return VWr(VU, Q, d, f'vcmp_{F}(c, {cop}, {VQ(VT, Q, n)}, {VQ(VT, Q, m)})') if scalar: if key not in ((0, 0, 0b11011), (0, 0, 0b11111), (0, 1, 0b11111), (1, 1, 0b11010)): @@ -586,13 +592,13 @@ class SimdC: raise Unsupported('scalar misc size') a, sa = f'VS64({n})', f'(int64_t)VS64({n})' if opc == 8: - return f'c->v[{d}] = ({sa} {">=" if U else ">"} 0) ? ~0ull : 0;' + return f'VSET({d}, ({sa} {">=" if U else ">"} 0) ? ~0ull : 0);' if opc == 9: - return f'c->v[{d}] = ({sa} {"<=" if U else "=="} 0) ? ~0ull : 0;' + return f'VSET({d}, ({sa} {"<=" if U else "=="} 0) ? ~0ull : 0);' if opc == 10 and not U: - return f'c->v[{d}] = ({sa} < 0) ? ~0ull : 0;' + return f'VSET({d}, ({sa} < 0) ? ~0ull : 0);' if opc == 11: - return f'c->v[{d}] = (uint64_t)-{a};' if U else f'c->v[{d}] = ({sa} < 0) ? (uint64_t)-{a} : {a};' + return f'VSET({d}, (uint64_t)-{a});' if U else f'VSET({d}, ({sa} < 0) ? (uint64_t)-{a} : {a});' raise Unsupported('scalar misc opcode') def misc2_fp(self, w, scalar): @@ -634,7 +640,7 @@ class SimdC: a, b = f'({VT}){{{SF}({n})}}', f'({VT}){{}}' if swap: a, b = b, a - return f'c->v[{d}] = ({ut(2 + sz)})vcmp_{F}(c, {op}, {a}, {b})[0];' + return f'VSET({d}, ({ut(2 + sz)})vcmp_{F}(c, {op}, {a}, {b})[0]);' a, b = x, f'({VT}){{}}' if swap: a, b = b, a @@ -660,7 +666,7 @@ class SimdC: if opc in (0b11010, 0b11011) or (opc == 0b11100 and not a_): # FCVT[NPMZA][SU] rm = {(0b11010, 0): 0, (0b11010, 1): 1, (0b11011, 0): 2, (0b11011, 1): 3, (0b11100, 0): 4}[(opc, a_)] if scalar: - return f'c->v[{d}] = ({ut(2 + sz)})toint_{F}(c, {SF}({n}), {rm}, 0, {32 << sz}, {U});' + return f'VSET({d}, ({ut(2 + sz)})toint_{F}(c, {SF}({n}), {rm}, 0, {32 << sz}, {U}));' return VWr(VU, Q, d, f'vtoint_{F}(c, {x}, {rm}, 0, {U})') if opc == 0b11100: # URECPE / URSQRTE if scalar or sz: @@ -697,7 +703,7 @@ class SimdC: U, sz, opc, n, d = bits(w, 29, 1), bits(w, 22, 2), bits(w, 12, 5), bits(w, 5, 5), bits(w, 0, 5) if not U: if opc == 0b11011 and sz == 3: - return f'c->v[{d}] = (uint64_t)(VS64({n}) + VHI64({n}));' + return f'VSET({d}, (uint64_t)(VS64({n}) + VHI64({n})));' raise Unsupported('scalar pairwise') fop = {(0b01100, 0): 'FOP_MAXNM', (0b01100, 1): 'FOP_MINNM', (0b01101, 0): 'FOP_ADD', (0b01111, 0): 'FOP_MAX', (0b01111, 1): 'FOP_MIN'}.get((opc, sz >> 1)) @@ -725,7 +731,7 @@ class SimdC: if op and not (c4 == 14 or c4 == 15): # MVNI imm = ~imm & ((1 << 64) - 1) v = imm | (imm << 64) if Q else imm - return f'c->v[{d}] = {u128(v)};' + return vset(d, u128(v), v >> 64 != 0) # ------------------------------------------------------------ shift by immediate @@ -754,7 +760,7 @@ class SimdC: return f'{SET}({d}, fromint_{F}(c, {SU}({n}), {es}, {U}, {fb}));' return VWr(VT, Q, d, f'vfromint_{F}(c, {VR(VU, n)}, {fb}, {U})') if scalar: - return f'c->v[{d}] = ({ut(sz)})toint_{F}(c, {SF}({n}), 3, {fb}, {es}, {U});' + return f'VSET({d}, ({ut(sz)})toint_{F}(c, {SF}({n}), 3, {fb}, {es}, {U}));' return VWr(VU, Q, d, f'vtoint_{F}(c, {VQ(VT, Q, n)}, 3, {fb}, {U})') if narrow: if sz == 3: @@ -784,7 +790,7 @@ class SimdC: e = f'({T})({VR(S, n)} >> {min(rsh, es - 1)})' if opc == 2: e = f'{VR(T, d)} + {e}' - return VWr(T, Q, d, e) if not scalar else f'c->v[{d}] = (uint64_t)(({e})[0]);' + return VWr(T, Q, d, e) if not scalar else f'VSET({d}, (uint64_t)(({e})[0]));' if opc in (4, 6): return same(lop('RSHRI' if opc == 4 else 'RSRAI', U), sz, ne, d, n, -1, rsh) if opc == 8: @@ -795,7 +801,7 @@ class SimdC: if U: return same(lop('SLI'), sz, ne, d, n, -1, lsh) e = f'{VR(T, n)} << {lsh}' - return VWr(T, Q, d, e) if not scalar else f'c->v[{d}] = (uint64_t)(({e})[0]);' + return VWr(T, Q, d, e) if not scalar else f'VSET({d}, (uint64_t)(({e})[0]));' if opc == 12: if not U: raise Unsupported('sqshlu U') @@ -905,11 +911,11 @@ class SimdC: F, VT, VU, T, SF, SU, SET = FT[t] sign = '0x80000000u' if not t else '0x8000000000000000ull' if opc == 0: - return f'c->v[{d}] = {SU}({n});' + return f'VSET({d}, {SU}({n}));' if opc == 1: - return f'c->v[{d}] = {SU}({n}) & ~{sign};' + return f'VSET({d}, {SU}({n}) & ~{sign});' if opc == 2: - return f'c->v[{d}] = {SU}({n}) ^ {sign};' + return f'VSET({d}, {SU}({n}) ^ {sign});' if opc == 3: return f'fp_op_{F}(c, FOP_SQRT, {d}, {n}, {n});' if opc in (4, 5, 7): @@ -917,7 +923,7 @@ class SimdC: if to == t: raise Unsupported('fcvt same') if to == 3: - return f'c->v[{d}] = {"simd_s2h" if not t else "simd_dd2h"}(c, {SF}({n}));' + return f'VSET({d}, {"simd_s2h" if not t else "simd_dd2h"}(c, {SF}({n})));' return f'FP_SETD({d}, simd_s2d(c, VSF32({n})));' if to else f'FP_SETS({d}, simd_d2s(c, VSF64({n})));' rm = {8: 0, 9: 1, 10: 2, 11: 3, 12: 4, 14: 5, 15: 6}.get(opc) if rm is None: @@ -943,14 +949,14 @@ class SimdC: t = self._ftype(w) SU = FT[t][5] d, n, m, cnd = bits(w, 0, 5), bits(w, 5, 5), bits(w, 16, 5), bits(w, 12, 4) - return f'c->v[{d}] = {cond(cnd)} ? {SU}({n}) : {SU}({m});' + return f'VSET({d}, {cond(cnd)} ? {SU}({n}) : {SU}({m}));' def fp_imm(self, w): t = self._ftype(w) if bits(w, 5, 5): raise Unsupported('fmov imm5') imm8, d = bits(w, 13, 8), bits(w, 0, 5) - return f'c->v[{d}] = {hx(vfp_expand_imm(imm8, 32 << t))};' + return f'VSET({d}, {hx(vfp_expand_imm(imm8, 32 << t))});' def fp_fixed(self, w): sf, t, rmode, opc, scale, n, d = (bits(w, 31, 1), bits(w, 22, 2), bits(w, 19, 2), bits(w, 16, 3), bits(w, 10, 6), @@ -972,12 +978,12 @@ class SimdC: sf, t, rmode, opc, n, d = bits(w, 31, 1), bits(w, 22, 2), bits(w, 19, 2), bits(w, 16, 3), bits(w, 5, 5), bits(w, 0, 5) if opc in (6, 7): # FMOV if t == 2 and sf and rmode == 1: - return set_reg(1, d, f'VHI64({n})') if opc == 6 else f'VHI64({d}) = {X(n)};' + return set_reg(1, d, f'VHI64({n})') if opc == 6 else f'VINS(vu64, {d}, 1, {X(n)});' if rmode or t != sf or t > 1: raise Unsupported('fmov general') if opc == 6: return set_reg(sf, d, f'VS64({n})' if sf else f'VS32({n})') - return f'c->v[{d}] = {X(n) if sf else W(n)};' + return f'VSET({d}, {X(n) if sf else W(n)});' if t > 1: raise Unsupported('fp int type') F, VT, VU, T, SF, SU, SET = FT[t] @@ -1003,7 +1009,7 @@ class SimdC: if opc == 3: raise Unsupported('ldr literal opc') addr = (pc + sext(bits(w, 5, 19), 19) * 4) & ((1 << 64) - 1) - return f'c->v[{rt}] = {("LD32", "LD64", "LD128")[opc]}({hx(addr)});' + return vset(rt, f'{("LD32", "LD64", "LD128")[opc]}({hx(addr)})', opc == 2) if bits(w, 31, 1) == 0 and bits(w, 24, 6) in (0b001100, 0b001101): return self.ldst_struct(w) if bits(w, 27, 3) == 0b101: @@ -1026,8 +1032,8 @@ class SimdC: sz = 1 << scale lines = [f'uint64_t ea = {ea};'] if L: - lines.append(f'u128 a = {ld}(ea), b = {ld}(ea + {sz});') - lines.append(f'c->v[{rt}] = a; c->v[{rt2}] = b;') + lines.append(f'{"u128" if scale == 4 else "uint64_t"} a = {ld}(ea), b = {ld}(ea + {sz});') + lines.append(f'{vset(rt, "a", scale == 4)} {vset(rt2, "b", scale == 4)}') else: cast = {2: 'VS32', 3: 'VS64', 4: 'c->v'}[scale] v1, v2 = (f'c->v[{rt}]', f'c->v[{rt2}]') if scale == 4 else (f'{cast}({rt})', f'{cast}({rt2})') @@ -1077,7 +1083,7 @@ class SimdC: raise Unsupported('ldst simd') parts = [f'uint64_t ea = {ea};'] if L: - parts.append(f'c->v[{rt}] = {("LD8", "LD16", "LD32", "LD64", "LD128")[scale]}(ea);') + parts.append(vset(rt, f'{("LD8", "LD16", "LD32", "LD64", "LD128")[scale]}(ea)', scale == 4)) else: src = (f'VS8({rt})', f'VS16({rt})', f'VS32({rt})', f'VS64({rt})', f'c->v[{rt}]')[scale] parts.append(f'{("ST8", "ST16", "ST32", "ST64", "ST128")[scale]}(ea, {src});') @@ -1106,7 +1112,8 @@ class SimdC: regs = [(rt + i) % 32 for i in range(rpt)] step = 16 if Q else 8 if L: - acc = ' '.join(f'c->v[{r}] = {"LD128" if Q else "LD64"}(ea + {i * step});' for i, r in enumerate(regs)) + acc = ' '.join(vset(r, f'{"LD128" if Q else "LD64"}(ea + {i * step})', Q) + for i, r in enumerate(regs)) else: acc = ' '.join(f'{"ST128" if Q else "ST64"}(ea + {i * step}, {"c->v[%d]" % r if Q else "VS64(%d)" % r});' for i, r in enumerate(regs)) diff --git a/runtime/simd.h b/runtime/simd.h index aaeaa2ff..c3196230 100644 --- a/runtime/simd.h +++ b/runtime/simd.h @@ -34,9 +34,19 @@ typedef u128 __attribute__((aligned(1), may_alias)) u128u; // ---- registers: VREG = whole V register as a vector type; VQ = source with the upper half cleared when // Q=0; VW = write, zeroing the upper half when Q=0. Scalar views of lane 0: VS8..VS64, VSF32/VSF64. +// A write is one 16-byte store, also where the instruction sets a scalar (the bits above it become zero) or a single +// lane (the other lanes stay): a V register written as two 8-byte halves or as one lane makes the next 16-byte load +// of it wait until those stores have left the store buffer, because the processor forwards a store to a load only +// when the one store covers the load. Assigning a zero-extended u128 to c->v[d] compiles to the two halves, so the +// writes go through a vector type: +// VSET = a scalar of at most 64 bits in the low bits, zero above; VSET128 = all 128 bits of an integer value; +// VINS = lane i replaced by x, the other lanes kept. #define VREG(T, n) (*(T *)&c->v[n]) #define VQ(T, Q, n) ((T)((Q) ? c->v[n] : (u128)(uint64_t)c->v[n])) -#define VW(T, Q, d, e) do { T _w = (e); c->v[d] = (Q) ? (u128)_w : (u128)(uint64_t)(u128)_w; } while (0) +#define VSET(d, x) (VREG(vu64, d) = (vu64){(uint64_t)(x), 0}) +#define VSET128(d, x) (VREG(vu64, d) = (vu64)(u128)(x)) +#define VW(T, Q, d, e) do { T _w = (e); VREG(vu64, d) = (Q) ? (vu64)_w : (vu64){((vu64)_w)[0], 0}; } while (0) +#define VINS(T, d, i, x) do { T _l = VREG(T, d); _l[i] = (x); VREG(vu64, d) = simd_whole((vu64)_l); } while (0) #define VS8(n) (*(uint8_t *)&c->v[n]) #define VS16(n) (*(sa16 *)&c->v[n]) #define VS32(n) (*(sa32 *)&c->v[n]) @@ -44,6 +54,12 @@ typedef u128 __attribute__((aligned(1), may_alias)) u128u; #define VSF32(n) (*(saf32 *)&c->v[n]) #define VSF64(n) (*(saf64 *)&c->v[n]) #define VHI64(n) (((sa64 *)&c->v[n])[1]) +// v, with where it came from hidden from the compiler: a load of a V register, a change of one lane and a store of +// the 16 bytes would otherwise be compiled as a store of that lane alone. +SI vu64 simd_whole(vu64 v) { + __asm__("" : "+x"(v)); + return v; +} #define LD128(a) (*(const u128u *)(uintptr_t)(a)) #define ST128(a, v) (*(u128u *)(uintptr_t)(a) = (u128)(v)) #define SHUF __builtin_shufflevector @@ -384,8 +400,8 @@ SI float simd_h2s(Cpu *c, uint16_t h) { } // ---- scalar FP instructions (FP data processing) -#define FP_SETS(d, x) (c->v[d] = fb_s(x)) -#define FP_SETD(d, x) (c->v[d] = fb_d(x)) +#define FP_SETS(d, x) VSET(d, fb_s(x)) +#define FP_SETD(d, x) VSET(d, fb_d(x)) SI void fp_op_s(Cpu *c, int op, int d, int n, int m) { FP_SETS(d, op_s(c, op, VSF32(n), VSF32(m), 0)); } SI void fp_op_d(Cpu *c, int op, int d, int n, int m) { FP_SETD(d, op_d(c, op, VSF64(n), VSF64(m), 0)); } // FMADD family: neg_a / neg_n negate the addend / first factor before the fused op @@ -568,7 +584,7 @@ SI void sl_gen(Cpu *c, int op, int sz, int ne, int d, int szd, int offd, int kee uint64_t y = m < 0 ? bimm : lget(&b, szm, bidx >= 0 ? bidx : offm + i); lset(&r, szd, offd + i, lop(op, sz, lget(&a, szn, offn + i), y, lget(&o, szd, offd + i))); } - c->v[d] = r; + VSET128(d, r); } // SADDLP/UADDLP/SADALP/UADALP SI void sl_addlp(Cpu *c, int u, int acc, int Q, int sz, int d, int n) { @@ -578,7 +594,7 @@ SI void sl_addlp(Cpu *c, int u, int acc, int Q, int sz, int d, int n) { uint64_t s = u ? x + y : (uint64_t)(lsx(x, sz) + lsx(y, sz)); lset(&r, sz + 1, i, s + (acc ? lget(&o, sz + 1, i) : 0)); } - c->v[d] = r; + VSET128(d, r); } // ADDV, SADDLV/UADDLV, SMAXV/UMAXV, SMINV/UMINV (op 0..3; u = unsigned) SI void sl_across(Cpu *c, int op, int u, int Q, int sz, int d, int n) { @@ -591,12 +607,12 @@ SI void sl_across(Cpu *c, int op, int u, int Q, int sz, int d, int n) { else if (op == 2) acc = u ? ((uint64_t)x > (uint64_t)acc ? x : acc) : (x > acc ? x : acc); else acc = u ? ((uint64_t)x < (uint64_t)acc ? x : acc) : (x < acc ? x : acc); } - c->v[d] = (uint64_t)acc & lmask(op == 1 ? sz + 1 : sz); + VSET(d, (uint64_t)acc & lmask(op == 1 ? sz + 1 : sz)); } // PMULL2/PMULL 1Q (64x64 -> 128 carry-less) SI void sl_pmull64(Cpu *c, int part, int d, int n, int m) { uint64_t hi, lo = pmul64(((sa64 *)&c->v[n])[part], ((sa64 *)&c->v[m])[part], &hi); - c->v[d] = ((u128)hi << 64) | lo; + VSET128(d, ((u128)hi << 64) | lo); } // ---- FP vector helpers: lanes of the 64-bit form are computed on a zero upper half @@ -611,7 +627,7 @@ SI double pair_d(Cpu *c, int op, double a, double b) { return op_d(c, op, a, b, // FMAXV/FMINV/FMAXNMV/FMINNMV (4S): pairwise tree SI void sv_fred_s(Cpu *c, int op, int d, int n) { vf32 v = VREG(vf32, n); - c->v[d] = fb_s(pair_s(c, op, pair_s(c, op, v[0], v[1]), pair_s(c, op, v[2], v[3]))); + VSET(d, fb_s(pair_s(c, op, pair_s(c, op, v[0], v[1]), pair_s(c, op, v[2], v[3])))); } // lane-wise FP conversions SI vu32 vtoint_s(Cpu *c, vf32 v, int rm, int fbits, int uns) { @@ -656,7 +672,7 @@ SI void sv_fcvtl(Cpu *c, int sz, int part, int d, int n) { for (int i = 0; i < 2; i++) o[i] = simd_s2d(c, fv_s((uint32_t)lget(&a, 2, 2 * part + i))); r = (u128)o; } - c->v[d] = r; + VSET128(d, r); } SI void sv_fcvtn(Cpu *c, int sz, int part, int odd, int d, int n) { u128 a = c->v[n], r = part ? (uint64_t)c->v[d] : 0; @@ -668,7 +684,7 @@ SI void sv_fcvtn(Cpu *c, int sz, int part, int odd, int d, int n) { lset(&r, 2, 2 * part + i, fb_s(odd ? simd_d2s_odd(c, x) : simd_d2s(c, x))); } } - c->v[d] = r; + VSET128(d, r); } // TBL / TBX with len+1 table registers @@ -680,7 +696,7 @@ SI void sv_tbl(Cpu *c, int Q, int len, int tbx, int d, int n, int m) { unsigned k = (unsigned)lget(&idx, 0, i); lset(&r, 0, i, k < 16u * (len + 1) ? tab[k] : tbx ? lget(&o, 0, i) : 0); } - c->v[d] = r; + VSET128(d, r); } // ---- CRC32 / CRC32C (bytes = 1, 2, 4, 8) @@ -716,7 +732,7 @@ SI void sv_aes(Cpu *c, int op, int d, int n) { // 0 AESE, 1 AESD, 2 AESMC, 3 AES } } memcpy(&v, r, 16); - c->v[d] = v; + VSET128(d, v); } SI uint32_t rol32_(uint32_t x, int s) { return (x << s) | (x >> (32 - s)); } SI uint32_t ror32_(uint32_t x, int s) { return (x >> s) | (x << (32 - s)); } @@ -779,7 +795,7 @@ SI void sv_sha_misc(Cpu *c, int op, int d, int n, int m) { // 0 SHA1SU0, 1 SHA1S break; } default: - c->v[d] = rol32_(b[0], 30); + VSET(d, rol32_(b[0], 30)); return; } memcpy(&c->v[d], r, 16); @@ -809,7 +825,7 @@ SI void sl_ldm(Cpu *c, int Q, int sz, int selem, int rpt, int t, uint64_t ea) { for (int k = 0; k < rpt; k++) for (int e = 0; e < ne; e++) for (int s = 0; s < selem; s++, ea += 1 << sz) lset(&r[k + s], sz, e, simd_ld(ea, sz)); - for (int i = 0; i < rpt + selem - 1; i++) c->v[(t + i) & 31] = r[i]; + for (int i = 0; i < rpt + selem - 1; i++) VSET128((t + i) & 31, r[i]); } SI void sl_stm(Cpu *c, int Q, int sz, int selem, int rpt, int t, uint64_t ea) { u128 r[4]; @@ -822,9 +838,13 @@ SI void sl_stm(Cpu *c, int Q, int sz, int selem, int rpt, int t, uint64_t ea) { // single structure: lane idx of selem registers; LDnR replicates into every lane SI void sl_lds(Cpu *c, int sz, int selem, int idx, int t, uint64_t ea) { for (int s = 0; s < selem; s++) { - u128 v = c->v[(t + s) & 31]; - lset(&v, sz, idx, simd_ld(ea + (s << sz), sz)); - c->v[(t + s) & 31] = v; + uint64_t x = simd_ld(ea + (s << sz), sz); + switch (sz) { + case 0: VINS(vu8, (t + s) & 31, idx, (uint8_t)x); break; + case 1: VINS(vu16, (t + s) & 31, idx, (uint16_t)x); break; + case 2: VINS(vu32, (t + s) & 31, idx, (uint32_t)x); break; + default: VINS(vu64, (t + s) & 31, idx, x); break; + } } } SI void sl_sts(Cpu *c, int sz, int selem, int idx, int t, uint64_t ea) { @@ -836,6 +856,6 @@ SI void sl_ldr(Cpu *c, int Q, int sz, int selem, int t, uint64_t ea) { for (int s = 0; s < selem; s++) { u128 v = 0; for (int e = 0; e < (8 << Q) >> sz; e++) lset(&v, sz, e, x[s]); - c->v[(t + s) & 31] = v; + VSET128((t + s) & 31, v); } }