Triforce v0.7.0 "Sage Of Courage"

The 0.6 crashes after warps and area changes fixed (no write-back into memory the capture leaves out), invisible objects fixed (occlusion gate), DLSS frame generation flicker fixed, achievements, the renderer built with a play profile, faster translated game code, helper threads judged on settled scenes, memory, shader preparation, audio effects, launcher, controls and Linux fixes.
This commit is contained in:
ruckus7382sfgsgdf
2026-11-22 11:59:06 -06:00
parent 70c210cee1
commit e6f2f248f2
529 changed files with 55958 additions and 1898 deletions

View File

@@ -5,6 +5,22 @@
Functions that call each other stay together, heaviest calls first, so what used to inline still can. The pieces cost about the
same to compile (chunks.fn_cost). code_hot.c stays as it was, renamed code_hot.c.whole, and build.py compiles code_hot_*.c at the
hot flags. A profile from the whole file fits the pieces: its counters go by function name and shape.
(Triforce, 11-22, the performance audit's LIFT-30, after Source A's) The pieces are sticky, and each declares only what it
uses. Every split dealt the functions out afresh by their sizes, so a few more lines in one hot function could move
functions between most of the pieces, and every piece began with code_hot.c's whole preamble, so one new declaration
changed all of them: an update that touched a few hot functions compiled up to eight pieces of about ten MB again, the
longest files of the build. Now:
- <work>/image/hot_pieces.txt keeps each hot function's piece. The next split leaves the functions it knows where they
are and puts a new one in the piece it shares the most calls with, else in the cheapest; when that leaves the costliest
piece more than STICKY_SLACK times a fresh split's costliest, or the file is not there, or it names more pieces than
this split makes, the split is the fresh one, as before, and the file is written again from it;
- a piece keeps a declaration of code_hot.c's preamble only if it uses the name: the lift's `void f_<address>(Ctx
*A64_NOESC);` lines and the patches' one-line `hos_` declarations (a function's, or a variable's with extern), each a
whole statement at the top level (declared()). Every other line of the preamble stays in every piece.
A64_HOT_STICKY=0 (at lift time; an A64_ name, in the lift key): the fresh split and the whole preamble, byte for byte as
before, and hot_pieces.txt is neither read nor written. The file is not in the lift key: any map gives the same functions.
tests/test_sticky_hot_pieces.py.
"""
import collections
import glob
@@ -21,21 +37,39 @@ HEAD = lifted_c.HEAD
CALL = re.compile(r"\b(f_[0-9a-f]{8})\(c\)")
def split(gen, pieces):
path = os.path.join(gen, "code_hot.c")
whole = path + ".whole"
if os.path.exists(path):
os.replace(path, whole)
for old in glob.glob(os.path.join(glob.escape(gen), "code_hot_*.c")):
os.remove(old)
with open(whole, encoding="utf-8", newline="") as f:
text = f.read()
marks = list(HEAD.finditer(text))
if not marks:
raise SystemExit("split_hot: no functions in %s" % whole)
preamble = text[:marks[0].start()]
bodies = {m.group(1): text[m.start():(marks[i + 1].start() if i + 1 < len(marks) else len(text))] for i, m in enumerate(marks)}
STICKY_SLACK = 1.15 # (how much dearer than a fresh split's the costliest sticky piece may be)
DECL = re.compile(r"void (f_[0-9a-f]+)\(Ctx \*A64_NOESC\);") # (the lift's declaration of a function a hot one names)
# (a one-line declaration of a hos_ name, nx2/oot_patches.py's: a function's, extern or not, or a variable's with extern. A
# variable's without extern is a definition, and stays in every piece as before)
EXTERN = re.compile(r"(?P<ext>extern\s+)?(?:[A-Za-z_]\w*[\s*]+)+(?P<name>hos_\w+)\s*(?P<args>\([^;{}=#]*\))?\s*;")
NAME = re.compile(r"\b(?:f_[0-9a-f]+|hos_\w+)\b")
TOKEN = re.compile(r'/\*|\*/|//|"(?:\\.|[^"\\])*"|\'(?:\\.|[^\'\\])*\'|[{}()]')
def sticky():
"""Whether the pieces are sticky with their own declarations: unless A64_HOT_STICKY=0"""
return os.environ.get("A64_HOT_STICKY", "1").strip() != "0"
def pieces_file(gen):
"""<work>/image/hot_pieces.txt for a gen folder (<work>/gen or <work>/sa/gen): beside chunks.txt, which a player's build keeps"""
up = os.path.dirname(os.path.abspath(gen))
work = os.path.dirname(up) if os.path.basename(up) == "sa" else up
return os.path.join(work, "image", "hot_pieces.txt")
def call_edges(bodies):
"""How often each hot function calls each other one: {(caller, callee): calls}"""
edges = collections.Counter()
for f, b in bodies.items():
for callee in CALL.findall(b):
if callee in bodies and callee != f:
edges[(f, callee)] += 1
return edges
def fresh_split(bodies, pieces, edges):
"""The split from the sizes alone, as every split was: (the pieces' function names, the calls that stay in one group)"""
# (groups joined along the calls: small callees first, those are what the optimiser inlines)
limit = sum(len(b) for b in bodies.values()) * 1.1 / pieces
parent = {f: f for f in bodies}
@@ -46,11 +80,6 @@ def split(gen, pieces):
parent[f] = parent[parent[f]]
f = parent[f]
return f
edges = collections.Counter()
for f, b in bodies.items():
for callee in CALL.findall(b):
if callee in bodies and callee != f:
edges[(f, callee)] += 1
for (f, callee), n in sorted(edges.items(), key=lambda e: (len(bodies[e[0][1]]) / e[1], e[0])):
a, b = root(f), root(callee)
if a != b and size[a] + size[b] <= limit:
@@ -68,17 +97,158 @@ def split(gen, pieces):
b = min(bins, key=lambda x: x[0])
b[0] += sum(cost[f] for f in g)
b[1].extend(g)
kept = sum(n for (f, c), n in edges.items() if root(f) == root(c))
return [names for _, names in bins], kept
def read_pieces(path):
"""{function: piece} from hot_pieces.txt, or {} when it is not there or not what save_pieces writes"""
try:
with open(path, encoding="utf-8") as f:
rows = [line.split() for line in f if line.strip()]
except OSError:
return {}
if not all(len(r) == 2 and r[1].isdigit() for r in rows):
return {}
return {name: int(piece) for name, piece in rows}
def save_pieces(path, bins):
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "w", encoding="utf-8", newline="\n") as f:
f.write("".join("%s %d\n" % (name, i) for i, names in enumerate(bins) for name in sorted(names)))
def sticky_split(bodies, pieces, saved, edges, fresh):
"""The pieces with the functions `saved` knows left where they are: (the pieces' names, why), or (fresh, why) when
there is nothing to keep or keeping it costs more than STICKY_SLACK times the fresh split's costliest piece"""
if not saved:
return fresh, "fresh (no pieces file)"
if max(saved.values()) >= pieces:
return fresh, "fresh (the pieces file is for another number of pieces)" # (fewer: the slack below decides)
cost = {f: chunks.fn_cost(len(b)) for f, b in bodies.items()}
bins = [[] for _ in range(pieces)]
where = {}
for f in sorted(bodies):
if f in saved:
bins[saved[f]].append(f)
where[f] = saved[f]
if not where:
return fresh, "fresh (none of the pieces file's functions is hot now)"
known = len(where)
load = [sum(cost[f] for f in names) for names in bins]
calls = collections.defaultdict(collections.Counter) # (each function's calls with every other, either way)
for (f, callee), n in edges.items():
calls[f][callee] += n
calls[callee][f] += n
for f in sorted(bodies): # (a new one: beside the function it shares the most calls with, else in the cheapest piece)
if f in where:
continue
shared = collections.Counter()
for other, n in calls[f].items():
if other in where:
shared[where[other]] += n
i = min(range(pieces), key=lambda i: (-shared[i], load[i], i))
bins[i].append(f)
where[f] = i
load[i] += cost[f]
fresh_top = max(sum(cost[f] for f in names) for names in fresh)
if max(load) > STICKY_SLACK * fresh_top:
return fresh, "fresh (kept where they were, the costliest piece is %.2f times a fresh split's)" % (max(load) / max(1, fresh_top))
return bins, "sticky (%d functions where they were, %d new)" % (known, len(where) - known)
def declared(lines):
"""For each line of code_hot.c's preamble the name it declares when a piece may leave it out, else None. Only a whole
statement at the top level is taken: not a line inside braces or parentheses, a comment, a macro's continued lines, or
after an unfinished statement (an attribute on the line before), where a line that reads like a declaration is code"""
out = []
depth = paren = 0
comment = cont = False
done = True
for line in lines:
s = line.strip()
name = None
if depth == 0 and paren == 0 and not comment and not cont and done and not s.startswith(("typedef", "static")):
m = DECL.fullmatch(s)
e = None if m else EXTERN.fullmatch(s)
name = m.group(1) if m else e.group("name") if e and (e.group("ext") or e.group("args")) else None
out.append(name)
directive = cont or s.startswith("#")
cont = line.rstrip("\r\n").endswith("\\")
code, pos = [], 0 # (the line without its comments, and the braces and parentheses it opens and closes)
for t in TOKEN.finditer(line):
g = t.group()
if comment:
if g == "*/":
comment, pos = False, t.end()
elif g in ("//", "/*"):
code.append(line[pos:t.start()])
pos = len(line)
if g == "//":
break
comment = True
elif g in ("{", "}", "(", ")") and not directive:
depth += (g == "{") - (g == "}")
paren += (g == "(") - (g == ")")
if not comment:
code.append(line[pos:])
code = " ".join(code).strip()
if not directive and code:
done = code.endswith((";", "{", "}"))
return out
def piece_preamble(preamble, names_used):
"""code_hot.c's preamble for one piece: a declaration of a function or of a hos_ name stays only if the piece uses the
name (names_used: every f_ and hos_ word of its functions); every other line stays"""
lines = preamble.splitlines(keepends=True)
names = declared(lines)
# (a name a line that always stays uses is used too: a macro or an inline helper the preamble may carry)
used = set(names_used)
for line, name in zip(lines, names):
if name is None:
used.update(NAME.findall(line))
return "".join(line for line, name in zip(lines, names) if name is None or name in used)
def split(gen, pieces):
path = os.path.join(gen, "code_hot.c")
whole = path + ".whole"
if os.path.exists(path):
os.replace(path, whole)
for old in glob.glob(os.path.join(glob.escape(gen), "code_hot_*.c")):
os.remove(old)
with open(whole, encoding="utf-8", newline="") as f:
text = f.read()
marks = list(lifted_c.definitions(text)) # ((Triforce, 11-22, the performance audit's IO-40) HEAD's matches, found by jumps)
if not marks:
raise SystemExit("split_hot: no functions in %s" % whole)
preamble = text[:marks[0].start()]
bodies = {m.group(1): text[m.start():(marks[i + 1].start() if i + 1 < len(marks) else len(text))] for i, m in enumerate(marks)}
edges = call_edges(bodies)
bins, kept = fresh_split(bodies, pieces, edges)
keep = sticky()
if keep: # ((Triforce, LIFT-30) the functions the last split placed stay where they are)
bins, how = sticky_split(bodies, pieces, read_pieces(pieces_file(gen)), edges, bins)
where = {f: i for i, names in enumerate(bins) for f in names}
kept = sum(n for (f, c), n in edges.items() if where[f] == where[c])
save_pieces(pieces_file(gen), bins)
print("split_hot: %s" % how)
cost = {f: chunks.fn_cost(len(b)) for f, b in bodies.items()}
costs = {}
for i, (c, names) in enumerate(bins):
for i, names in enumerate(bins):
c = sum(cost[f] for f in names)
head = piece_preamble(preamble, {n for name in names for n in NAME.findall(bodies[name])}) if keep else preamble
with open(os.path.join(gen, "code_hot_%d.c" % i), "w", encoding="utf-8", newline="") as f:
f.write(preamble)
f.write(head)
for name in sorted(names):
f.write(bodies[name])
costs["code_hot_%d.c" % i] = len(preamble) + c
costs["code_hot_%d.c" % i] = len(head) + c
print("split_hot: piece %d: %d functions, %d MB, cost %d MB" % (i, len(names), sum(len(bodies[f]) for f in names) >> 20, c >> 20))
chunks.write_costs(gen, costs)
kept = sum(n for (f, c), n in edges.items() if root(f) == root(c))
print("split_hot: %d of %d calls stay in one group" % (kept, sum(edges.values())))
print("split_hot: %d of %d calls stay in one %s" % (kept, sum(edges.values()), "piece" if keep else "group"))
if __name__ == "__main__":