mirror of
https://git.triforceproject.com/triforce/source
synced 2026-11-19 16:20:01 -04:00
Triforce v0.5.0
In-game mods (GameBanana browser, search and installs, with the Mask Salesman), code-patch and cheat mods, the devs' test18 to test45 work (full performance overlay, frame pacing on, steady frame rate, save export and import, bug capture, new options rows, Setup's Mods page and Add to Steam), occlusion queries and new defaults, Auto resolution that scales, replay helper threads and robust buffers off by default, the options in sections.
This commit is contained in:
264
nx2/chunks.py
264
nx2/chunks.py
@@ -13,6 +13,11 @@ The functions' C is made by several processes (fork, so Linux and macOS; one pro
|
||||
consecutive functions from the lifter as it stands, and write() takes the texts in its own order, as if it had made
|
||||
them. The lifter's counters (stats, unhandled, ijump, and any passed in `counters`, such as poll.stats) get each
|
||||
process's additions. The output is the one-process output, byte for byte (tests/test_parallel_emit.py).
|
||||
|
||||
(Triforce) Where there is no fork (Windows) the processes are spawned and sent what they need (nx2/emit_pool.py), and the same
|
||||
processes then cut the big chunks (split_big) and share the page marks (edit_c), a file each: 46 s, 6 s and 19 s of the lift's
|
||||
127 s in one process on a 16-thread PC, 13 s, 1.4 s and 5 s of its 69 s with 8 processes. HOOKSHOT_LIFT_JOBS=1 is the one
|
||||
process, as before, on every system.
|
||||
"""
|
||||
import bisect
|
||||
import collections
|
||||
@@ -20,9 +25,11 @@ import glob
|
||||
import multiprocessing
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
import whole_write # (rewrites written beside and moved over: issue 1197)
|
||||
|
||||
import a64_lift
|
||||
import emit_pool # ((Triforce) the emit's processes where there is no fork)
|
||||
import lifted_c
|
||||
|
||||
FILE = "chunks.txt"
|
||||
@@ -101,7 +108,8 @@ def _emit_run(es):
|
||||
|
||||
|
||||
def _available_gb():
|
||||
"""MemAvailable from /proc/meminfo in GB (None where there is none)"""
|
||||
"""MemAvailable from /proc/meminfo in GB; (Triforce) on Windows the physical memory GlobalMemoryStatusEx calls available
|
||||
(None where there is neither)"""
|
||||
try:
|
||||
with open("/proc/meminfo", encoding="ascii") as f:
|
||||
for line in f:
|
||||
@@ -109,22 +117,51 @@ def _available_gb():
|
||||
return int(line.split()[1]) / (1024 * 1024)
|
||||
except (OSError, ValueError, IndexError):
|
||||
pass
|
||||
if os.name == "nt":
|
||||
try:
|
||||
import ctypes
|
||||
|
||||
class Status(ctypes.Structure):
|
||||
_fields_ = [("dwLength", ctypes.c_ulong), ("dwMemoryLoad", ctypes.c_ulong)] + [
|
||||
(name, ctypes.c_ulonglong) for name in ("ullTotalPhys", "ullAvailPhys", "ullTotalPageFile", "ullAvailPageFile",
|
||||
"ullTotalVirtual", "ullAvailVirtual", "ullAvailExtendedVirtual")]
|
||||
s = Status()
|
||||
s.dwLength = ctypes.sizeof(s)
|
||||
if ctypes.WinDLL("kernel32").GlobalMemoryStatusEx(ctypes.byref(s)):
|
||||
return s.ullAvailPhys / (1 << 30)
|
||||
except (OSError, AttributeError, ValueError):
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
FORK = "fork" in multiprocessing.get_all_start_methods()
|
||||
SPAWN_MOST = 8 # ((Triforce) spawned emit processes at most: the lift's own process cannot keep more busy, emit_pool.py)
|
||||
SPAWN_GB = 1 # (and one per GB of available memory: one holds its run or a file of lifted C, 480 MB at the most seen)
|
||||
SPAWN_LEAST = 2 * emit_pool.RUN_FUNCS # (functions below which the lift stays one process: starting the others costs more)
|
||||
|
||||
|
||||
def emit_jobs():
|
||||
"""Processes for the emit: HOOKSHOT_LIFT_JOBS, else the cores this process may use (at most 16; each holds its run's
|
||||
texts and the pages of the lift it touches), and at most one per 2 GB of available memory: a Steam Deck (16 GB) ran
|
||||
out with 8, and the system's out-of-memory handling stopped the lift ("lift.py stopped (signal 15): BrokenPipeError")
|
||||
or one worker (the update then sat at "jump tables" for good). 1 where there is no fork"""
|
||||
if "fork" not in multiprocessing.get_all_start_methods():
|
||||
return 1
|
||||
or one worker (the update then sat at "jump tables" for good).
|
||||
(Triforce) Where there is no fork (Windows; before, always 1 there): half the processors, at most SPAWN_MOST, and at most
|
||||
one per SPAWN_GB of available memory. Half, because the files that need no translated code are compiled beside the lift
|
||||
there (make_game's EarlyCompile, one compiler a processor less one) and because the lift's own process, which sends the
|
||||
runs and writes the C, is what more of them would wait for (emit_pool.py has the measure). 1 is the one process on
|
||||
every system"""
|
||||
if os.environ.get("HOOKSHOT_LIFT_JOBS"):
|
||||
return max(1, int(os.environ["HOOKSHOT_LIFT_JOBS"]))
|
||||
try:
|
||||
n = len(os.sched_getaffinity(0))
|
||||
except AttributeError:
|
||||
n = os.cpu_count() or 1
|
||||
if not FORK:
|
||||
n = max(1, min(SPAWN_MOST, n // 2))
|
||||
gb = _available_gb()
|
||||
if gb is not None:
|
||||
n = max(1, min(n, int(gb // SPAWN_GB)))
|
||||
return n
|
||||
n = max(1, min(16, n))
|
||||
gb = _available_gb()
|
||||
if gb is not None:
|
||||
@@ -169,23 +206,65 @@ def parallel_emit(emit, order, texts, counters, jobs, run=2000):
|
||||
yield (e,) + tuple(emit(e, texts))
|
||||
|
||||
|
||||
def write(L, gen_dir, img_dir, rechunk=False, counters=(), jobs=None):
|
||||
_workers = None # ((Triforce) the spawned processes of this lift, from write() until edit_c()'s marks end, emit_pool.Workers)
|
||||
_marks = None # (the page marks in flight after edit_c(wait=False): the thread waiting for them, what ended it, the folder)
|
||||
|
||||
|
||||
def stop_workers(kill=False):
|
||||
"""(Triforce) The spawned processes ended, when there are any: after what they are doing, or (kill) now"""
|
||||
global _workers
|
||||
w, _workers = _workers, None
|
||||
if w is not None:
|
||||
w.stop(kill)
|
||||
|
||||
|
||||
def drop_lifted(gen_dir):
|
||||
"""(Triforce) The lifted C taken away after a lift that stopped while its processes were writing it: the chunks, their
|
||||
parts, the hot file and its pieces, whole or cut short. What is left of such a lift looks like a lift (every file that
|
||||
is there may end as a function does, and an earlier lift's a64_table.c still names every function), and the next lift
|
||||
would take its chunk boundaries from it (seed_from_gen) and a build its files. The same files make_game takes away
|
||||
before a lift"""
|
||||
for f in os.listdir(gen_dir) if os.path.isdir(gen_dir) else ():
|
||||
if f.startswith("code_") and f.endswith((".c", ".c.whole", ".part")):
|
||||
try:
|
||||
os.remove(os.path.join(gen_dir, f))
|
||||
except OSError:
|
||||
pass # (held by something else: the lift's own failure is what is said)
|
||||
|
||||
|
||||
def gave_up(gen_dir):
|
||||
"""(Triforce) A lift with spawned processes that failed: they are ended now and its C goes"""
|
||||
stop_workers(kill=True)
|
||||
drop_lifted(gen_dir)
|
||||
|
||||
|
||||
def write(L, gen_dir, img_dir, rechunk=False, counters=(), jobs=None, setup=()):
|
||||
"""L.write(gen_dir) with stable boundaries; returns its files. counters: dicts the emit adds to besides the
|
||||
lifter's own (poll.stats); jobs: emit processes (emit_jobs())"""
|
||||
lifter's own (poll.stats); jobs: emit processes (emit_jobs()); (Triforce) setup: what set the lifter's class up for
|
||||
the emit in this process (poll.install), module-level functions the spawned processes call before they emit"""
|
||||
global _workers
|
||||
starts = [] if rechunk else load(img_dir)
|
||||
if not starts and not rechunk:
|
||||
starts = seed_from_gen(gen_dir)
|
||||
b = Boundaries(a64_lift.CHUNK_BYTES, starts, L.funcs)
|
||||
base_emit = L.emit_function
|
||||
jobs = emit_jobs() if jobs is None else jobs
|
||||
spawned = jobs > 1 and not FORK and len(L.funcs) >= SPAWN_LEAST
|
||||
made = None
|
||||
|
||||
def emit_function(e, texts):
|
||||
nonlocal made
|
||||
global _workers
|
||||
b.current = e
|
||||
if made is None: # (the first call: write() has its texts now, and asks for the functions in sorted order)
|
||||
made = parallel_emit(base_emit, sorted(L.funcs), texts,
|
||||
[L.stats, L.unhandled, L.ijump] + list(counters), jobs)
|
||||
every = [L.stats, L.unhandled, L.ijump] + list(counters)
|
||||
if spawned:
|
||||
stop_workers(kill=True) # (an earlier write()'s, which no edit_c() ended)
|
||||
_workers = emit_pool.Workers(jobs, L, every, setup)
|
||||
print("emit: %d processes (HOOKSHOT_LIFT_JOBS=<n> for another count, 1 for one process)" % _workers.jobs, flush=True)
|
||||
made = emit_pool.emit(_workers, L, sorted(L.funcs), texts, every)
|
||||
else:
|
||||
made = parallel_emit(base_emit, sorted(L.funcs), texts, every, jobs)
|
||||
got = next(made, None)
|
||||
if got is None or got[0] != e:
|
||||
raise SystemExit("chunks: write() asked for f_%08x, the emit made %s" % (e, "f_%08x" % got[0] if got else "nothing"))
|
||||
@@ -193,6 +272,10 @@ def write(L, gen_dir, img_dir, rechunk=False, counters=(), jobs=None):
|
||||
L.emit_function = emit_function
|
||||
try:
|
||||
files = L.write(gen_dir, chunk_bytes=b)
|
||||
except BaseException:
|
||||
if spawned: # ((Triforce) a process failed, died or hung, or this one was stopped: no part of a lift is left behind)
|
||||
gave_up(gen_dir)
|
||||
raise
|
||||
finally:
|
||||
del L.emit_function
|
||||
if made is not None:
|
||||
@@ -210,7 +293,12 @@ def write(L, gen_dir, img_dir, rechunk=False, counters=(), jobs=None):
|
||||
for f in os.listdir(gen_dir):
|
||||
if f == "code_hot.c.whole" or HOT_PIECE.fullmatch(f):
|
||||
os.remove(os.path.join(gen_dir, f))
|
||||
files += split_big(gen_dir, files)
|
||||
try:
|
||||
files += split_big(gen_dir, files, workers=_workers if spawned else None)
|
||||
except BaseException:
|
||||
if spawned:
|
||||
gave_up(gen_dir)
|
||||
raise
|
||||
save(img_dir, starts or b.recorded)
|
||||
print("chunks: %d, boundaries %s" % (len(files), "kept" if starts else "from sizes (saved)"))
|
||||
return files
|
||||
@@ -251,43 +339,57 @@ def write_costs(gen_dir, costs):
|
||||
f.write("".join("%s %d\n" % kv for kv in sorted(known.items())))
|
||||
|
||||
|
||||
def split_big(gen_dir, files, budget=None):
|
||||
def split_one(gen_dir, path, budget):
|
||||
"""split_big for one chunk (a spawned process's share of it too): the new parts' paths, {file name: cost} of the chunk or of
|
||||
each of its parts, and what to say of a chunk that was cut (None of one left whole)"""
|
||||
m = re.fullmatch(r"code_(\d{3})\.c", os.path.basename(path))
|
||||
made, costs = [], {}
|
||||
with open(path, encoding="utf-8", newline="") as f:
|
||||
text = f.read()
|
||||
marks = [h.start() for h in HEAD.finditer(text)]
|
||||
ends = marks[1:] + [len(text)]
|
||||
cost = [fn_cost(b - a) for a, b in zip(marks, ends)]
|
||||
total = sum(cost)
|
||||
if total <= budget * 3 // 2 or len(marks) < 2:
|
||||
costs[os.path.basename(path)] = marks[0] + total if marks else len(text)
|
||||
return made, costs, None
|
||||
parts = -(-total // budget)
|
||||
cuts, run, k = [0], 0, 1 # (indexes into marks where a part starts)
|
||||
for i, c in enumerate(cost):
|
||||
if i and run >= total * k // parts:
|
||||
cuts.append(i)
|
||||
k = run * parts // total + 1 # (past a giant function: the next share, not a sliver after it)
|
||||
run += c
|
||||
bounds = [marks[i] for i in cuts] + [len(text)]
|
||||
preamble = text[:marks[0]]
|
||||
for j in range(len(cuts)):
|
||||
out = path if j == 0 else os.path.join(gen_dir, "code_%s_%d.c" % (m.group(1), j))
|
||||
piece = text[:bounds[1]] if j == 0 else preamble + text[bounds[j]:bounds[j + 1]]
|
||||
whole_write.write_whole(out, piece)
|
||||
costs[os.path.basename(out)] = len(preamble) + sum(cost[cuts[j]:cuts[j + 1] if j + 1 < len(cuts) else len(cost)])
|
||||
if j:
|
||||
made.append(out)
|
||||
return made, costs, "chunks: %s (%d MB, cost %d MB) in %d parts" % (os.path.basename(path), len(text) >> 20, total >> 20, len(cuts))
|
||||
|
||||
|
||||
def split_big(gen_dir, files, budget=None, workers=None):
|
||||
"""Chunks whose estimated compile time (fn_cost) is well past a plain chunk's, cut at function starts into parts of about equal
|
||||
cost: code_NNN.c, code_NNN_1.c ... A chunk of the giant functions took 350 s, alone at the end of the build, where a chunk of
|
||||
ordinary functions takes 45; the boundaries kept in chunks.txt also let chunks grow (44 MB beside 25). The first part keeps the
|
||||
name, so seed_from_gen and the boundaries are as before. Writes chunk_costs.txt for every chunk; returns the new parts' paths"""
|
||||
name, so seed_from_gen and the boundaries are as before. Writes chunk_costs.txt for every chunk; returns the new parts' paths.
|
||||
(Triforce) workers: the spawned processes that take a chunk each (emit_pool.Workers; None: this process takes them all)"""
|
||||
budget = budget or a64_lift.CHUNK_BYTES
|
||||
made, costs = [], {}
|
||||
for path in files:
|
||||
m = re.fullmatch(r"code_(\d{3})\.c", os.path.basename(path))
|
||||
if not m:
|
||||
continue
|
||||
with open(path, encoding="utf-8", newline="") as f:
|
||||
text = f.read()
|
||||
marks = [h.start() for h in HEAD.finditer(text)]
|
||||
ends = marks[1:] + [len(text)]
|
||||
cost = [fn_cost(b - a) for a, b in zip(marks, ends)]
|
||||
total = sum(cost)
|
||||
if total <= budget * 3 // 2 or len(marks) < 2:
|
||||
costs[os.path.basename(path)] = marks[0] + total if marks else len(text)
|
||||
continue
|
||||
parts = -(-total // budget)
|
||||
cuts, run, k = [0], 0, 1 # (indexes into marks where a part starts)
|
||||
for i, c in enumerate(cost):
|
||||
if i and run >= total * k // parts:
|
||||
cuts.append(i)
|
||||
k = run * parts // total + 1 # (past a giant function: the next share, not a sliver after it)
|
||||
run += c
|
||||
bounds = [marks[i] for i in cuts] + [len(text)]
|
||||
preamble = text[:marks[0]]
|
||||
for j in range(len(cuts)):
|
||||
out = path if j == 0 else os.path.join(gen_dir, "code_%s_%d.c" % (m.group(1), j))
|
||||
piece = text[:bounds[1]] if j == 0 else preamble + text[bounds[j]:bounds[j + 1]]
|
||||
whole_write.write_whole(out, piece)
|
||||
costs[os.path.basename(out)] = len(preamble) + sum(cost[cuts[j]:cuts[j + 1] if j + 1 < len(cuts) else len(cost)])
|
||||
if j:
|
||||
made.append(out)
|
||||
print("chunks: %s (%d MB, cost %d MB) in %d parts" % (os.path.basename(path), len(text) >> 20, total >> 20, len(cuts)))
|
||||
paths = [path for path in files if re.fullmatch(r"code_(\d{3})\.c", os.path.basename(path))]
|
||||
if workers is None:
|
||||
answers = (split_one(gen_dir, path, budget) for path in paths)
|
||||
else:
|
||||
answers = workers.each(split_one, (("the split of %s" % os.path.basename(path), (gen_dir, path, budget)) for path in paths))
|
||||
for new, cost, said in answers: # (in the chunks' order, whoever cut them)
|
||||
made += new
|
||||
costs.update(cost)
|
||||
if said:
|
||||
print(said)
|
||||
for f in os.listdir(gen_dir): # (parts of an earlier lift that this one did not make)
|
||||
if PART.fullmatch(f) and os.path.join(gen_dir, f) not in made:
|
||||
os.remove(os.path.join(gen_dir, f))
|
||||
@@ -295,10 +397,82 @@ def split_big(gen_dir, files, budget=None):
|
||||
return made
|
||||
|
||||
|
||||
def edit_c(gen, stats, supported, whole=True):
|
||||
def marks_one(path):
|
||||
"""(Triforce) a64_lift.group_marks for one file, in a spawned process: whether it changed the file, and its counts"""
|
||||
st = collections.Counter()
|
||||
return a64_lift.group_marks([path], st), list(st.items())
|
||||
|
||||
|
||||
def ask_marks(paths, workers):
|
||||
"""(Triforce) a64_lift.group_marks asked of the spawned processes, a file each, the largest first (the hot file, 82 MB,
|
||||
is the last by name and would end alone). Every file is asked for here, at once: a lift that ends on an error of its
|
||||
own while they are at it asks for nothing more, and the processes finish the files, all of them. What take_marks() takes"""
|
||||
by_size = sorted(paths, key=lambda p: (-os.path.getsize(p), p))
|
||||
return by_size, workers.ask(marks_one, [("the page marks of %s" % os.path.basename(p), (p,)) for p in by_size])
|
||||
|
||||
|
||||
def take_marks(paths, stats, workers, asked):
|
||||
"""(Triforce) Waits for what ask_marks() asked: the counts go into stats in the paths' order, as one process adds them
|
||||
when it groups the files itself. The files changed"""
|
||||
by_size, calls = asked
|
||||
answers = dict(zip(by_size, workers.answers(calls)))
|
||||
changed = 0
|
||||
for p in paths:
|
||||
n, counts = answers[p]
|
||||
changed += n
|
||||
for k, v in counts:
|
||||
stats[k] = stats.get(k, 0) + v
|
||||
return changed
|
||||
|
||||
|
||||
def edit_c(gen, stats, supported, whole=True, wait=True):
|
||||
"""(nx2/lift.py, after the lift writes its C; here, beside the writing, so it imports without the lift's tools) The edits of the written C, in this order: the game's patches (nx2/oot_patches.py, which find the lift's exact text; the
|
||||
supported build's whole standalone lift fails when an output-size edit finds no place), then, last, the page marks of each
|
||||
group of stores shared (a64_lift.group_marks), which changes that text. stats: the lift's, for its report"""
|
||||
group of stores shared (a64_lift.group_marks), which changes that text. stats: the lift's, for its report.
|
||||
(Triforce) The marks are made by write()'s spawned processes when it left any, which end with them. wait=False: this
|
||||
returns while they are at it, and marks_done() waits for them: the lift writes its last files meanwhile (hos_image.c takes
|
||||
as long as the marks, and reads nothing of the C); stats is theirs until then"""
|
||||
global _marks
|
||||
import oot_patches
|
||||
oot_patches.apply(gen, protect_hearts=supported, required=supported and whole)
|
||||
a64_lift.group_marks(sorted(glob.glob(os.path.join(glob.escape(gen), "code_*.c"))), stats)
|
||||
try:
|
||||
oot_patches.apply(gen, protect_hearts=supported, required=supported and whole)
|
||||
except BaseException:
|
||||
stop_workers(kill=True) # (the patches' own failure, said as before: the C stays as it is, for whoever looks into it)
|
||||
raise
|
||||
paths = sorted(glob.glob(os.path.join(glob.escape(gen), "code_*.c")))
|
||||
workers = _workers
|
||||
if workers is None:
|
||||
a64_lift.group_marks(paths, stats)
|
||||
return
|
||||
try:
|
||||
asked = ask_marks(paths, workers)
|
||||
except BaseException:
|
||||
gave_up(gen)
|
||||
raise
|
||||
ended = []
|
||||
|
||||
def marks(): # (this thread only waits for the processes and adds their counts up)
|
||||
try:
|
||||
take_marks(paths, stats, workers, asked)
|
||||
except BaseException as e:
|
||||
ended.append(e)
|
||||
thread = threading.Thread(target=marks)
|
||||
thread.start()
|
||||
_marks = (thread, ended, gen)
|
||||
if wait:
|
||||
marks_done()
|
||||
|
||||
|
||||
def marks_done():
|
||||
"""(Triforce) After edit_c(wait=False): waits for the page marks the spawned processes are making, and ends the
|
||||
processes. Nothing to do when edit_c made the marks itself (one process, or wait)"""
|
||||
global _marks
|
||||
m, _marks = _marks, None
|
||||
if m is None:
|
||||
return
|
||||
thread, ended, gen = m
|
||||
thread.join()
|
||||
if ended: # (a process failed in the middle of the files: some marked, some not, one perhaps cut short)
|
||||
gave_up(gen)
|
||||
raise ended[0]
|
||||
stop_workers()
|
||||
|
||||
Reference in New Issue
Block a user