Files
Triforce-Source/nx2/fastvec.py
ruckus7382sfgsgdf e6f2f248f2 Triforce v0.7.0 "Sage Of Courage"
The 0.6 crashes after warps and area changes fixed (no write-back into memory the capture leaves out), invisible objects fixed (occlusion gate), DLSS frame generation flicker fixed, achievements, the renderer built with a play profile, faster translated game code, helper threads judged on settled scenes, memory, shader preparation, audio effects, launcher, controls and Linux fixes.
2026-11-22 11:59:06 -06:00

168 lines
8.9 KiB
Python

"""(Triforce, 11-21, the performance audit's LIFT-02, after Source A's) Vector moves the kit's translator writes slowly,
rewritten on the lifted text: two passes over one instruction's C, each with its helpers appended to a64_fpx.h. The
kit's files stay as they are (third_party/ is not edited: CONTRIBUTING.md); install(), called from nx2/lift.py beside
insn's and lift_fixes', wraps the kit's functions:
- a64_lift.vectorize_fpx: vectorize_bytes before it, on the emitter's own text (TBL with a one-register table, EXT),
so that widen_vstores, which comes after, no longer meets the loops' tails;
- a64_lift.op_text: vectorize_moves on its result (128-bit vector loads, DUP by element);
- a64_lift.fpx_header: VBYTE_HEADER and VMOVE_HEADER after the kit's VSET_HEADER, whose A64_OPAQUE and a64_vset64
they use.
Each rewrite leaves the same bytes in the same registers; tests/test_vbytes.py runs every shape both ways. Source A
counted, in its market's game code, TBL 2,995 sites, EXT 29,451, 128-bit loads 139,843 and DUP 26,529, and its game
work a frame went from 15.46 to about 14.3 ms with these and one more rewrite that Triforce had already
(a64_lift.py's vectorize_elem).
Switches, read at lift time (the lift key holds the A64_ variables): A64_NO_VBYTES=1 and A64_NO_VMOVES=1 keep the kit's
text for that pass. The lift's emit processes run nx2/lift.py's module level again (nx2/emit_pool.py), so they have
the wrappers too."""
import os
import re
import a64_lift
# TBL with a one-register table, and EXT. The emitter writes each as a loop of 8 or 16 single bytes through arrays on
# the stack, then a 16-byte reload of the result that cannot be forwarded from the byte stores. On x86 each is one SSSE3
# instruction. TBL: an index of 16 or more gives 0, which PSHUFB does when the index byte's top bit is set; the index
# plus 0x70, saturated, keeps 0 to 15 below 0x80 with the same low four bits and takes 16 to 255 to 0x80 or more.
# EXT of 16 bytes is PALIGNR, of 8 bytes two shifts of the low halves. Both sources are read before Vd is written, so
# Vd may be Vn or Vm.
VBYTE_TBL = re.compile(
r"\{ uint8_t _t\[16\],_x\[(8|16)\],_r\[\1\]; memcpy\(_t\+0,c->vreg\[(\d+)\],16\); memcpy\(_x,c->vreg\[(\d+)\],\1\); "
r"for\(int _i=0;_i<\1;_i\+\+\)\{ unsigned _k=_x\[_i\]; if\(_k<16U\) _r\[_i\]=_t\[_k\]; else _r\[_i\]=0; \} "
r"c->vreg\[(\d+)\]\[0\]=0; c->vreg\[\4\]\[1\]=0; memcpy\(c->vreg\[\4\],_r,\1\); \}")
VBYTE_EXT = re.compile(
r"\{ uint8_t _a\[(8|16)\],_b\[\1\],_r\[\1\]; memcpy\(_a,c->vreg\[(\d+)\],\1\); memcpy\(_b,c->vreg\[(\d+)\],\1\); "
r"for\(int _i=0;_i<\1;_i\+\+\)\{ int _k=_i\+(\d+); _r\[_i\] = \(_k<\1\) \? _a\[_k\] : _b\[_k-\1\]; \} "
r"c->vreg\[(\d+)\]\[0\]=0; c->vreg\[\5\]\[1\]=0; memcpy\(c->vreg\[\5\],_r,\1\); \}")
def vectorize_bytes(text):
if "uint8_t _" not in text or os.environ.get("A64_NO_VBYTES"):
return text
text = VBYTE_TBL.sub(lambda m: "a64_tbl1(c->vreg[%s],c->vreg[%s],c->vreg[%s],%d);"
% (m.group(4), m.group(2), m.group(3), m.group(1) == "16"), text)
return VBYTE_EXT.sub(lambda m: "A64_EXT%s(c->vreg[%s],c->vreg[%s],c->vreg[%s],%s);"
% (m.group(1), m.group(5), m.group(2), m.group(3), m.group(4)), text)
VBYTE_HEADER = r"""
/* ---- (Triforce, LIFT-02) TBL with one table register, and EXT, as one SSSE3 instruction each (nx2/fastvec.py
vectorize_bytes). Without SSSE3 (a baseline x86-64 build): plain C, the same bytes ---- */
#if defined(__SSSE3__)
#include <tmmintrin.h>
static inline __attribute__((always_inline)) void a64_tbl1(uint64_t *d, const uint64_t *t, const uint64_t *x, int q) {
const __m128i tv = _mm_loadu_si128((const __m128i *)t);
const __m128i xv = q ? _mm_loadu_si128((const __m128i *)x) : _mm_loadl_epi64((const __m128i *)x);
__m128i r = _mm_shuffle_epi8(tv, _mm_adds_epu8(xv, _mm_set1_epi8(0x70)));
if (!q) r = _mm_move_epi64(r);
A64_OPAQUE(r);
_mm_storeu_si128((__m128i *)d, r);
}
/* (EXT of 16 bytes: Vn's bytes from imm up, then Vm's; PALIGNR takes the high source first) */
#define A64_EXT16(d, n, m, imm) do { \
const __m128i a64_e_n_ = _mm_loadu_si128((const __m128i *)(n)), a64_e_m_ = _mm_loadu_si128((const __m128i *)(m)); \
__m128i a64_e_r_ = _mm_alignr_epi8(a64_e_m_, a64_e_n_, imm); \
A64_OPAQUE(a64_e_r_); \
_mm_storeu_si128((__m128i *)(d), a64_e_r_); } while (0)
#else
static inline void a64_tbl1(uint64_t *d, const uint64_t *t, const uint64_t *x, int q) {
uint8_t tb[16], xb[16], r[16] = {0};
memcpy(tb, t, 16); memcpy(xb, x, 16);
for (int i = 0; i < (q ? 16 : 8); i++) r[i] = xb[i] < 16 ? tb[xb[i]] : 0;
memcpy(d, r, 16);
}
#define A64_EXT16(d, n, m, imm) do { uint8_t a64_e_a_[32], a64_e_r_[16]; memcpy(a64_e_a_, (n), 16); \
memcpy(a64_e_a_ + 16, (m), 16); memcpy(a64_e_r_, a64_e_a_ + (imm), 16); memcpy((d), a64_e_r_, 16); } while (0)
#endif
/* (EXT of 8 bytes: the low halves, Vn's from byte imm up, then Vm's; the top half zero) */
#define A64_EXT8(d, n, m, imm) do { uint64_t a64_e_n_ = (n)[0], a64_e_m_ = (m)[0]; \
const uint64_t a64_e_r_ = (imm) ? (a64_e_n_ >> (8 * (imm))) | (a64_e_m_ << ((64 - 8 * (imm)) & 63)) : a64_e_n_; \
a64_vset64((d), a64_e_r_); } while (0)
"""
# Two moves the compiler leaves slow, on the final text (after widen_vstores):
# - a 128-bit vector load (LDR Q, LDP Q, LD1 of one register) is recomp_ldp64 into the register's two halves: two
# 8-byte stores, from which the next 16-byte read of the register cannot be forwarded. Now one 16-byte load and one
# 16-byte store (a64_vld16); guest memory is read the same way, recomp_load64 being a plain copy.
# - DUP Vd, Vn.T[i] is a loop through the stack and a 16-byte reload: one broadcast (a64_vdup). The element is read
# before Vd is written, so Vd may be Vn.
VMOVE_LDQ = re.compile(r"recomp_ldp64\(c,([^;]*?),&c->vreg\[(\d+)\]\[0\],&c->vreg\[\2\]\[1\]\);")
VMOVE_DUP = re.compile(
r"\{ uint8_t _s\[16\]; memcpy\(_s,c->vreg\[(\d+)\],16\); uint64_t _e=0; memcpy\(&_e,_s\+(\d+),(1|2|4|8)\); "
r"uint8_t _d\[16\]; memset\(_d,0,16\); for\(int _i=0;_i<(\d+);_i\+\+\) memcpy\(_d\+_i\*\3,&_e,\3\); "
r"a64_vst16\(c->vreg\[(\d+)\],_d\); \}")
def vectorize_moves(text):
if "c->vreg" not in text or os.environ.get("A64_NO_VMOVES"):
return text
text = VMOVE_LDQ.sub(lambda m: "a64_vld16(c->vreg[%s],%s);" % (m.group(2), m.group(1)), text)
def dup(m):
s, off, w, n, d = m.groups()
if int(w) * int(n) not in (8, 16): # (a shape that fills neither half nor the whole register: left alone)
return m.group(0)
return "a64_vdup(c->vreg[%s],c->vreg[%s],%s,%s,%d);" % (d, s, off, w, int(w) * int(n) == 16)
return VMOVE_DUP.sub(dup, text)
VMOVE_HEADER = r"""
/* ---- (Triforce, LIFT-02) a whole vector register loaded at once, and DUP by element as one broadcast (nx2/fastvec.py
vectorize_moves). Without the SSE2 host path: plain C, the same bytes ---- */
#if RECOMP_FPX_HOST == 2
static inline __attribute__((always_inline)) void a64_vld16(uint64_t *d, uint64_t a) {
__m128i x = _mm_loadu_si128((const __m128i *)A64_PTR(a));
A64_OPAQUE(x);
_mm_storeu_si128((__m128i *)d, x);
}
static inline __attribute__((always_inline)) void a64_vdup(uint64_t *d, const uint64_t *s, int off, int w, int q) {
uint64_t e = 0;
memcpy(&e, (const uint8_t *)s + off, w);
__m128i r = w == 1 ? _mm_set1_epi8((char)e) : w == 2 ? _mm_set1_epi16((short)e) : w == 4 ? _mm_set1_epi32((int)e)
: _mm_set1_epi64x((long long)e);
if (!q) r = _mm_move_epi64(r);
A64_OPAQUE(r);
_mm_storeu_si128((__m128i *)d, r);
}
#else
static inline void a64_vld16(uint64_t *d, uint64_t a) { memcpy(d, A64_PTR(a), 16); }
static inline void a64_vdup(uint64_t *d, const uint64_t *s, int off, int w, int q) {
uint8_t r[16] = {0};
uint64_t e = 0;
memcpy(&e, (const uint8_t *)s + off, w);
for (int i = 0; i < (q ? 16 : 8) / w; i++) memcpy(r + i * w, &e, w);
memcpy(d, r, 16);
}
#endif
"""
_installed = False
def install():
"""The two passes around the kit's vectorize_fpx and op_text, and their helpers into its fpx_header. Once"""
global _installed
if _installed:
return
_installed = True
base_vfpx, base_op_text, base_header = a64_lift.vectorize_fpx, a64_lift.op_text, a64_lift.fpx_header
def vectorize_fpx(text):
return base_vfpx(vectorize_bytes(text))
def op_text(text, fpx):
return vectorize_moves(base_op_text(text, fpx))
def fpx_header(used):
text = base_header(used)
at = a64_lift.VSET_HEADER + "\n"
if text.count(at) != 1: # (the kit changed: the helpers have nowhere known to go, and the lift must not go on)
raise SystemExit("fastvec: a64_fpx.h no longer holds the kit's VSET_HEADER once: where its helpers go")
return text.replace(at, at + VBYTE_HEADER + "\n" + VMOVE_HEADER + "\n")
a64_lift.vectorize_fpx, a64_lift.op_text, a64_lift.fpx_header = vectorize_fpx, op_text, fpx_header