Triforce v0.5.0

In-game mods (GameBanana browser, search and installs, with the Mask Salesman), code-patch and cheat mods, the devs' test18 to test45 work (full performance overlay, frame pacing on, steady frame rate, save export and import, bug capture, new options rows, Setup's Mods page and Add to Steam), occlusion queries and new defaults, Auto resolution that scales, replay helper threads and robust buffers off by default, the options in sections.
This commit is contained in:
ruckus7382sfgsgdf
2026-11-17 22:40:50 -06:00
parent 93eeb714c2
commit 6f338c8f56
600 changed files with 116254 additions and 2522 deletions

View File

@@ -13,6 +13,11 @@ The functions' C is made by several processes (fork, so Linux and macOS; one pro
consecutive functions from the lifter as it stands, and write() takes the texts in its own order, as if it had made
them. The lifter's counters (stats, unhandled, ijump, and any passed in `counters`, such as poll.stats) get each
process's additions. The output is the one-process output, byte for byte (tests/test_parallel_emit.py).
(Triforce) Where there is no fork (Windows) the processes are spawned and sent what they need (nx2/emit_pool.py), and the same
processes then cut the big chunks (split_big) and share the page marks (edit_c), a file each: 46 s, 6 s and 19 s of the lift's
127 s in one process on a 16-thread PC, 13 s, 1.4 s and 5 s of its 69 s with 8 processes. HOOKSHOT_LIFT_JOBS=1 is the one
process, as before, on every system.
"""
import bisect
import collections
@@ -20,9 +25,11 @@ import glob
import multiprocessing
import os
import re
import threading
import whole_write # (rewrites written beside and moved over: issue 1197)
import a64_lift
import emit_pool # ((Triforce) the emit's processes where there is no fork)
import lifted_c
FILE = "chunks.txt"
@@ -101,7 +108,8 @@ def _emit_run(es):
def _available_gb():
"""MemAvailable from /proc/meminfo in GB (None where there is none)"""
"""MemAvailable from /proc/meminfo in GB; (Triforce) on Windows the physical memory GlobalMemoryStatusEx calls available
(None where there is neither)"""
try:
with open("/proc/meminfo", encoding="ascii") as f:
for line in f:
@@ -109,22 +117,51 @@ def _available_gb():
return int(line.split()[1]) / (1024 * 1024)
except (OSError, ValueError, IndexError):
pass
if os.name == "nt":
try:
import ctypes
class Status(ctypes.Structure):
_fields_ = [("dwLength", ctypes.c_ulong), ("dwMemoryLoad", ctypes.c_ulong)] + [
(name, ctypes.c_ulonglong) for name in ("ullTotalPhys", "ullAvailPhys", "ullTotalPageFile", "ullAvailPageFile",
"ullTotalVirtual", "ullAvailVirtual", "ullAvailExtendedVirtual")]
s = Status()
s.dwLength = ctypes.sizeof(s)
if ctypes.WinDLL("kernel32").GlobalMemoryStatusEx(ctypes.byref(s)):
return s.ullAvailPhys / (1 << 30)
except (OSError, AttributeError, ValueError):
pass
return None
FORK = "fork" in multiprocessing.get_all_start_methods()
SPAWN_MOST = 8 # ((Triforce) spawned emit processes at most: the lift's own process cannot keep more busy, emit_pool.py)
SPAWN_GB = 1 # (and one per GB of available memory: one holds its run or a file of lifted C, 480 MB at the most seen)
SPAWN_LEAST = 2 * emit_pool.RUN_FUNCS # (functions below which the lift stays one process: starting the others costs more)
def emit_jobs():
"""Processes for the emit: HOOKSHOT_LIFT_JOBS, else the cores this process may use (at most 16; each holds its run's
texts and the pages of the lift it touches), and at most one per 2 GB of available memory: a Steam Deck (16 GB) ran
out with 8, and the system's out-of-memory handling stopped the lift ("lift.py stopped (signal 15): BrokenPipeError")
or one worker (the update then sat at "jump tables" for good). 1 where there is no fork"""
if "fork" not in multiprocessing.get_all_start_methods():
return 1
or one worker (the update then sat at "jump tables" for good).
(Triforce) Where there is no fork (Windows; before, always 1 there): half the processors, at most SPAWN_MOST, and at most
one per SPAWN_GB of available memory. Half, because the files that need no translated code are compiled beside the lift
there (make_game's EarlyCompile, one compiler a processor less one) and because the lift's own process, which sends the
runs and writes the C, is what more of them would wait for (emit_pool.py has the measure). 1 is the one process on
every system"""
if os.environ.get("HOOKSHOT_LIFT_JOBS"):
return max(1, int(os.environ["HOOKSHOT_LIFT_JOBS"]))
try:
n = len(os.sched_getaffinity(0))
except AttributeError:
n = os.cpu_count() or 1
if not FORK:
n = max(1, min(SPAWN_MOST, n // 2))
gb = _available_gb()
if gb is not None:
n = max(1, min(n, int(gb // SPAWN_GB)))
return n
n = max(1, min(16, n))
gb = _available_gb()
if gb is not None:
@@ -169,23 +206,65 @@ def parallel_emit(emit, order, texts, counters, jobs, run=2000):
yield (e,) + tuple(emit(e, texts))
def write(L, gen_dir, img_dir, rechunk=False, counters=(), jobs=None):
_workers = None # ((Triforce) the spawned processes of this lift, from write() until edit_c()'s marks end, emit_pool.Workers)
_marks = None # (the page marks in flight after edit_c(wait=False): the thread waiting for them, what ended it, the folder)
def stop_workers(kill=False):
"""(Triforce) The spawned processes ended, when there are any: after what they are doing, or (kill) now"""
global _workers
w, _workers = _workers, None
if w is not None:
w.stop(kill)
def drop_lifted(gen_dir):
"""(Triforce) The lifted C taken away after a lift that stopped while its processes were writing it: the chunks, their
parts, the hot file and its pieces, whole or cut short. What is left of such a lift looks like a lift (every file that
is there may end as a function does, and an earlier lift's a64_table.c still names every function), and the next lift
would take its chunk boundaries from it (seed_from_gen) and a build its files. The same files make_game takes away
before a lift"""
for f in os.listdir(gen_dir) if os.path.isdir(gen_dir) else ():
if f.startswith("code_") and f.endswith((".c", ".c.whole", ".part")):
try:
os.remove(os.path.join(gen_dir, f))
except OSError:
pass # (held by something else: the lift's own failure is what is said)
def gave_up(gen_dir):
"""(Triforce) A lift with spawned processes that failed: they are ended now and its C goes"""
stop_workers(kill=True)
drop_lifted(gen_dir)
def write(L, gen_dir, img_dir, rechunk=False, counters=(), jobs=None, setup=()):
"""L.write(gen_dir) with stable boundaries; returns its files. counters: dicts the emit adds to besides the
lifter's own (poll.stats); jobs: emit processes (emit_jobs())"""
lifter's own (poll.stats); jobs: emit processes (emit_jobs()); (Triforce) setup: what set the lifter's class up for
the emit in this process (poll.install), module-level functions the spawned processes call before they emit"""
global _workers
starts = [] if rechunk else load(img_dir)
if not starts and not rechunk:
starts = seed_from_gen(gen_dir)
b = Boundaries(a64_lift.CHUNK_BYTES, starts, L.funcs)
base_emit = L.emit_function
jobs = emit_jobs() if jobs is None else jobs
spawned = jobs > 1 and not FORK and len(L.funcs) >= SPAWN_LEAST
made = None
def emit_function(e, texts):
nonlocal made
global _workers
b.current = e
if made is None: # (the first call: write() has its texts now, and asks for the functions in sorted order)
made = parallel_emit(base_emit, sorted(L.funcs), texts,
[L.stats, L.unhandled, L.ijump] + list(counters), jobs)
every = [L.stats, L.unhandled, L.ijump] + list(counters)
if spawned:
stop_workers(kill=True) # (an earlier write()'s, which no edit_c() ended)
_workers = emit_pool.Workers(jobs, L, every, setup)
print("emit: %d processes (HOOKSHOT_LIFT_JOBS=<n> for another count, 1 for one process)" % _workers.jobs, flush=True)
made = emit_pool.emit(_workers, L, sorted(L.funcs), texts, every)
else:
made = parallel_emit(base_emit, sorted(L.funcs), texts, every, jobs)
got = next(made, None)
if got is None or got[0] != e:
raise SystemExit("chunks: write() asked for f_%08x, the emit made %s" % (e, "f_%08x" % got[0] if got else "nothing"))
@@ -193,6 +272,10 @@ def write(L, gen_dir, img_dir, rechunk=False, counters=(), jobs=None):
L.emit_function = emit_function
try:
files = L.write(gen_dir, chunk_bytes=b)
except BaseException:
if spawned: # ((Triforce) a process failed, died or hung, or this one was stopped: no part of a lift is left behind)
gave_up(gen_dir)
raise
finally:
del L.emit_function
if made is not None:
@@ -210,7 +293,12 @@ def write(L, gen_dir, img_dir, rechunk=False, counters=(), jobs=None):
for f in os.listdir(gen_dir):
if f == "code_hot.c.whole" or HOT_PIECE.fullmatch(f):
os.remove(os.path.join(gen_dir, f))
files += split_big(gen_dir, files)
try:
files += split_big(gen_dir, files, workers=_workers if spawned else None)
except BaseException:
if spawned:
gave_up(gen_dir)
raise
save(img_dir, starts or b.recorded)
print("chunks: %d, boundaries %s" % (len(files), "kept" if starts else "from sizes (saved)"))
return files
@@ -251,43 +339,57 @@ def write_costs(gen_dir, costs):
f.write("".join("%s %d\n" % kv for kv in sorted(known.items())))
def split_big(gen_dir, files, budget=None):
def split_one(gen_dir, path, budget):
"""split_big for one chunk (a spawned process's share of it too): the new parts' paths, {file name: cost} of the chunk or of
each of its parts, and what to say of a chunk that was cut (None of one left whole)"""
m = re.fullmatch(r"code_(\d{3})\.c", os.path.basename(path))
made, costs = [], {}
with open(path, encoding="utf-8", newline="") as f:
text = f.read()
marks = [h.start() for h in HEAD.finditer(text)]
ends = marks[1:] + [len(text)]
cost = [fn_cost(b - a) for a, b in zip(marks, ends)]
total = sum(cost)
if total <= budget * 3 // 2 or len(marks) < 2:
costs[os.path.basename(path)] = marks[0] + total if marks else len(text)
return made, costs, None
parts = -(-total // budget)
cuts, run, k = [0], 0, 1 # (indexes into marks where a part starts)
for i, c in enumerate(cost):
if i and run >= total * k // parts:
cuts.append(i)
k = run * parts // total + 1 # (past a giant function: the next share, not a sliver after it)
run += c
bounds = [marks[i] for i in cuts] + [len(text)]
preamble = text[:marks[0]]
for j in range(len(cuts)):
out = path if j == 0 else os.path.join(gen_dir, "code_%s_%d.c" % (m.group(1), j))
piece = text[:bounds[1]] if j == 0 else preamble + text[bounds[j]:bounds[j + 1]]
whole_write.write_whole(out, piece)
costs[os.path.basename(out)] = len(preamble) + sum(cost[cuts[j]:cuts[j + 1] if j + 1 < len(cuts) else len(cost)])
if j:
made.append(out)
return made, costs, "chunks: %s (%d MB, cost %d MB) in %d parts" % (os.path.basename(path), len(text) >> 20, total >> 20, len(cuts))
def split_big(gen_dir, files, budget=None, workers=None):
"""Chunks whose estimated compile time (fn_cost) is well past a plain chunk's, cut at function starts into parts of about equal
cost: code_NNN.c, code_NNN_1.c ... A chunk of the giant functions took 350 s, alone at the end of the build, where a chunk of
ordinary functions takes 45; the boundaries kept in chunks.txt also let chunks grow (44 MB beside 25). The first part keeps the
name, so seed_from_gen and the boundaries are as before. Writes chunk_costs.txt for every chunk; returns the new parts' paths"""
name, so seed_from_gen and the boundaries are as before. Writes chunk_costs.txt for every chunk; returns the new parts' paths.
(Triforce) workers: the spawned processes that take a chunk each (emit_pool.Workers; None: this process takes them all)"""
budget = budget or a64_lift.CHUNK_BYTES
made, costs = [], {}
for path in files:
m = re.fullmatch(r"code_(\d{3})\.c", os.path.basename(path))
if not m:
continue
with open(path, encoding="utf-8", newline="") as f:
text = f.read()
marks = [h.start() for h in HEAD.finditer(text)]
ends = marks[1:] + [len(text)]
cost = [fn_cost(b - a) for a, b in zip(marks, ends)]
total = sum(cost)
if total <= budget * 3 // 2 or len(marks) < 2:
costs[os.path.basename(path)] = marks[0] + total if marks else len(text)
continue
parts = -(-total // budget)
cuts, run, k = [0], 0, 1 # (indexes into marks where a part starts)
for i, c in enumerate(cost):
if i and run >= total * k // parts:
cuts.append(i)
k = run * parts // total + 1 # (past a giant function: the next share, not a sliver after it)
run += c
bounds = [marks[i] for i in cuts] + [len(text)]
preamble = text[:marks[0]]
for j in range(len(cuts)):
out = path if j == 0 else os.path.join(gen_dir, "code_%s_%d.c" % (m.group(1), j))
piece = text[:bounds[1]] if j == 0 else preamble + text[bounds[j]:bounds[j + 1]]
whole_write.write_whole(out, piece)
costs[os.path.basename(out)] = len(preamble) + sum(cost[cuts[j]:cuts[j + 1] if j + 1 < len(cuts) else len(cost)])
if j:
made.append(out)
print("chunks: %s (%d MB, cost %d MB) in %d parts" % (os.path.basename(path), len(text) >> 20, total >> 20, len(cuts)))
paths = [path for path in files if re.fullmatch(r"code_(\d{3})\.c", os.path.basename(path))]
if workers is None:
answers = (split_one(gen_dir, path, budget) for path in paths)
else:
answers = workers.each(split_one, (("the split of %s" % os.path.basename(path), (gen_dir, path, budget)) for path in paths))
for new, cost, said in answers: # (in the chunks' order, whoever cut them)
made += new
costs.update(cost)
if said:
print(said)
for f in os.listdir(gen_dir): # (parts of an earlier lift that this one did not make)
if PART.fullmatch(f) and os.path.join(gen_dir, f) not in made:
os.remove(os.path.join(gen_dir, f))
@@ -295,10 +397,82 @@ def split_big(gen_dir, files, budget=None):
return made
def edit_c(gen, stats, supported, whole=True):
def marks_one(path):
"""(Triforce) a64_lift.group_marks for one file, in a spawned process: whether it changed the file, and its counts"""
st = collections.Counter()
return a64_lift.group_marks([path], st), list(st.items())
def ask_marks(paths, workers):
"""(Triforce) a64_lift.group_marks asked of the spawned processes, a file each, the largest first (the hot file, 82 MB,
is the last by name and would end alone). Every file is asked for here, at once: a lift that ends on an error of its
own while they are at it asks for nothing more, and the processes finish the files, all of them. What take_marks() takes"""
by_size = sorted(paths, key=lambda p: (-os.path.getsize(p), p))
return by_size, workers.ask(marks_one, [("the page marks of %s" % os.path.basename(p), (p,)) for p in by_size])
def take_marks(paths, stats, workers, asked):
"""(Triforce) Waits for what ask_marks() asked: the counts go into stats in the paths' order, as one process adds them
when it groups the files itself. The files changed"""
by_size, calls = asked
answers = dict(zip(by_size, workers.answers(calls)))
changed = 0
for p in paths:
n, counts = answers[p]
changed += n
for k, v in counts:
stats[k] = stats.get(k, 0) + v
return changed
def edit_c(gen, stats, supported, whole=True, wait=True):
"""(nx2/lift.py, after the lift writes its C; here, beside the writing, so it imports without the lift's tools) The edits of the written C, in this order: the game's patches (nx2/oot_patches.py, which find the lift's exact text; the
supported build's whole standalone lift fails when an output-size edit finds no place), then, last, the page marks of each
group of stores shared (a64_lift.group_marks), which changes that text. stats: the lift's, for its report"""
group of stores shared (a64_lift.group_marks), which changes that text. stats: the lift's, for its report.
(Triforce) The marks are made by write()'s spawned processes when it left any, which end with them. wait=False: this
returns while they are at it, and marks_done() waits for them: the lift writes its last files meanwhile (hos_image.c takes
as long as the marks, and reads nothing of the C); stats is theirs until then"""
global _marks
import oot_patches
oot_patches.apply(gen, protect_hearts=supported, required=supported and whole)
a64_lift.group_marks(sorted(glob.glob(os.path.join(glob.escape(gen), "code_*.c"))), stats)
try:
oot_patches.apply(gen, protect_hearts=supported, required=supported and whole)
except BaseException:
stop_workers(kill=True) # (the patches' own failure, said as before: the C stays as it is, for whoever looks into it)
raise
paths = sorted(glob.glob(os.path.join(glob.escape(gen), "code_*.c")))
workers = _workers
if workers is None:
a64_lift.group_marks(paths, stats)
return
try:
asked = ask_marks(paths, workers)
except BaseException:
gave_up(gen)
raise
ended = []
def marks(): # (this thread only waits for the processes and adds their counts up)
try:
take_marks(paths, stats, workers, asked)
except BaseException as e:
ended.append(e)
thread = threading.Thread(target=marks)
thread.start()
_marks = (thread, ended, gen)
if wait:
marks_done()
def marks_done():
"""(Triforce) After edit_c(wait=False): waits for the page marks the spawned processes are making, and ends the
processes. Nothing to do when edit_c made the marks itself (one process, or wait)"""
global _marks
m, _marks = _marks, None
if m is None:
return
thread, ended, gen = m
thread.join()
if ended: # (a process failed in the middle of the files: some marked, some not, one perhaps cut short)
gave_up(gen)
raise ended[0]
stop_workers()