Triforce v0.5.0

In-game mods (GameBanana browser, search and installs, with the Mask Salesman), code-patch and cheat mods, the devs' test18 to test45 work (full performance overlay, frame pacing on, steady frame rate, save export and import, bug capture, new options rows, Setup's Mods page and Add to Steam), occlusion queries and new defaults, Auto resolution that scales, replay helper threads and robust buffers off by default, the options in sections.
This commit is contained in:
ruckus7382sfgsgdf
2026-11-17 22:40:50 -06:00
parent 93eeb714c2
commit 6f338c8f56
600 changed files with 116254 additions and 2522 deletions

View File

@@ -50,6 +50,11 @@ import win_manifest
NX2_RT = os.path.join(kit.HERE, "rt")
HOST = os.path.join(kit.HERE, "..", "host")
OPUS = os.path.join(kit.HERE, "..", "third_party", "opus")
# (Triforce) zstd's decoder as the runtime compiles it (third_party/zstd/lib; host/host_zstd.h): every file the decoder needs,
# and none of the rest. tests/test_host_zstd.py compiles the same list
HOST_ZSTD_SOURCES = ("common/debug.c", "common/entropy_common.c", "common/error_private.c", "common/fse_decompress.c",
"common/xxhash.c", "common/zstd_common.c", "decompress/huf_decompress.c", "decompress/zstd_ddict.c",
"decompress/zstd_decompress.c", "decompress/zstd_decompress_block.c")
OBJ_CACHE = os.path.join(kit.WORK, "objcache")
# (hookshot) work/hot_order.txt: the functions that run, hottest first (sampled), packed together by the linker; RX_NO_ORDER=1: off
PGO = os.environ.get("RX_PGO", "")
@@ -244,6 +249,109 @@ def vk_sources():
return sorted(out)
def strip_comments(data):
"""(Triforce) C or C++ text without its comments, for translation_text, and whether it has a raw string literal. A block
comment becomes a space, a line's comment goes with the spaces before it; everything else is kept as it is. A string or
character literal ends at its closing quote or at the end of its line (a backslash takes the next character with it, a
line end too), a number's ' are digit separators, an #include's <name> and a raw string literal are kept whole: what this
reads wrongly it keeps, it never takes code or a literal for a comment"""
ident = frozenset(b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_$")
out, raw, i, n = bytearray(), False, 0, len(data)
while i < n:
c = data[i]
if data.startswith(b"//", i):
j = data.find(b"\n", i)
i = n if j < 0 else j
while out and out[-1] in b" \t":
del out[-1]
continue
if data.startswith(b"/*", i):
j = data.find(b"*/", i + 2)
i = n if j < 0 else j + 2
out += b" "
continue
j = i + 1
if c in b"\"'":
while j < n and data[j] != c and data[j] != 10:
j += 2 if data[j] == 92 else 1
j = min(j + 1 if j < n and data[j] == c else j, n)
elif c == 60 and out[out.rfind(b"\n") + 1:].strip().replace(b" ", b"").replace(b"\t", b"") in (b"#include", b"#include_next",
b"#import"):
while j < n and data[j] != 62 and data[j] != 10:
j += 1
j = min(j + 1 if j < n and data[j] == 62 else j, n)
elif 48 <= c <= 57 or (c == 46 and i + 1 < n and 48 <= data[i + 1] <= 57): # (a pp-number)
while j < n:
if data[j] in b"eEpP" and j + 1 < n and data[j + 1] in b"+-":
j += 2
elif data[j] == 39 and j + 1 < n and data[j + 1] in ident:
j += 2
elif data[j] in ident or data[j] == 46:
j += 1
else:
break
elif c in ident:
while j < n and data[j] in ident:
j += 1
if data[i:j] in (b"R", b"LR", b"uR", b"UR", b"u8R") and j < n and data[j] == 34:
k = data.find(b"(", j + 1)
delim = data[j + 1:k] if k >= 0 else b"\n"
if len(delim) <= 16 and not any(ch in b" ()\\\t\v\f\n" for ch in delim):
end = data.find(b")" + delim + b'"', k + 1)
j, raw = n if end < 0 else end + len(delim) + 2, True
out += data[i:j]
i = j
return bytes(out), raw
def translation_text(rel, data):
"""(Triforce) A file of the translator as the translation stamp takes it (translation_stamp): its line ends as LF, and a C
or C++ file without the parts between an "Outside the translation stamp" mark and the next "Inside the translation stamp
again" (gpu/vk/vk_shader.cpp's: code that cannot change a translation, gpu/vk/program_carry.h), without its comments, and
without its blank lines and the spaces at its lines' ends (but for the line after one a backslash ends, and in a file with
a raw string literal). Marks that do not pair: the file whole, and the build says so"""
out_mark, in_mark = b"// ---- (Triforce) Outside the translation stamp", b"// ---- (Triforce) Inside the translation stamp again"
data = data.replace(bytes((13, 10)), bytes((10,)))
if not rel.endswith((".c", ".cc", ".cpp", ".h", ".hpp", ".inc")):
return data
lines, kept, out, paired = data.split(b"\n"), [], False, True
for line in lines:
mark = line.strip()
if mark.startswith(out_mark) or mark.startswith(in_mark):
paired = paired and out != mark.startswith(out_mark)
out = mark.startswith(out_mark)
elif not out:
kept.append(line)
if out or not paired:
print("build: %s: its translation stamp marks do not pair: the whole file is in the stamp (no program cache is carried "
"over this update)" % rel, flush=True)
kept = lines
data, raw = strip_comments(b"\n".join(kept))
if raw:
return data
lines = []
for line in data.split(b"\n"):
if lines and lines[-1].endswith(b"\\"):
lines.append(line)
elif line.rstrip(b" \t"):
lines.append(line.rstrip(b" \t"))
return b"\n".join(lines)
def translation_stamp(vk, translator):
"""(Triforce) The translation stamp (HOOKSHOT_TRANSLATION_STAMP, 16 hex digits), which the program cache is carried over an
update by (gpu/vk/program_carry.h has the rule): the files of the translator's stamp (HOOKSHOT_SHADER_STAMP), each as
translation_text takes it, but for gpu/vk/vk_api.h, the host's interface, which no translation reads a constant or a type
of (tests/test_program_carry.py). translator: the files of eden, sirit and compat"""
stamp = hashlib.sha1(usedforsecurity=False)
for f in sorted(translator) + [os.path.join(vk, n) for n in ("vk_shader.cpp", "vk_bank.cpp", "vk_state.h")]:
rel = os.path.relpath(f, vk).replace("\\", "/")
with open(f, "rb") as src:
text = translation_text(rel, src.read())
stamp.update(rel.encode() + b"\0" + len(text).to_bytes(8, "little") + text)
return stamp.hexdigest()[:16]
def cpm_dirs():
"""CPM packages: RX_CPM, packaged tools/cpm, then legacy project caches.
The packages provide Vulkan and SPIR-V headers, VMA and FFmpeg."""
@@ -415,21 +523,61 @@ def rss_mb(proc):
return 0
# (a function from this size on is counted by the square of its size: mem_need)
GIANT_BYTES = int(1.2 * (1 << 20))
GIANT_MB = 300
# (the giant functions are counted so only on a PC with less memory free than this when the compile starts: giants_counted)
GIANT_BELOW_MB = 14 * 1024
_giants = None
def giants_counted():
"""Whether mem_need counts giant functions by the square of their size: decided once, when it is first asked. On a PC
with GIANT_BELOW_MB or more free it does not. The count is each file's true peak, but the gate holds a file's whole
peak back from the moment its compiler starts, and the giants reach theirs only near their end and not together: with
21 GB free, counting them in full left 4 to 9 game files compiling at once for the first six minutes where 10 to 13
had run without the memory ever running short, and the compile took 657 s for 462. Where memory is short the same
files are what ran a PC out of it, so there they are counted. HOOKSHOT_MEM_GIANT=1 or 0 decides it outright."""
global _giants
if _giants is None:
asked = os.environ.get("HOOKSHOT_MEM_GIANT")
if asked in ("0", "1"):
_giants = asked == "1"
else:
free = free_mb()
_giants = free is not None and free < GIANT_BELOW_MB
print("the memory gate: giant functions %s (%s)" % (
"counted by the square of their size" if _giants else "counted as the other functions",
"HOOKSHOT_MEM_GIANT" if asked in ("0", "1") else "less than %d GB free" % (GIANT_BELOW_MB >> 10) if _giants
else "%d GB or more free, or the free memory not known" % (GIANT_BELOW_MB >> 10)), flush=True)
return _giants
def mem_need(src):
"""What compiling a file takes at its peak, in MB, a little over. Measured on a cold build (clang 23, x86-64): a chunk of lifted
code 0.4 GB + 50 MB per MB of C + 250 MB per MB of its largest function (1.5 GB for 24 MB of small functions, 2.2 GB for a 6 MB
function alone); a hot piece 0.8-1.3 GB whatever its size (-O2, the profile, inlining); the runtime's C files ~0.2 GB, the C++
files up to ~1 GB"""
files up to ~1 GB.
(Triforce, measured again with HOOKSHOT_MEM_LOG on a build of v0.4.0, clang 23.1.5) The few files with giant functions took
far more than that: 6.2 GB for a chunk that is one 5 MB function (counted 1.9), 3.7 and 3.1 GB for two others (counted 1.4
and 1.5), and 2.5 to 2.7 GB for the three hot pieces that hold functions near 2 MB (counted 1.5). The compiler's memory
grows with the square of a function's size, as its time does (nx2/chunks.py QUAD), so every function of GIANT_BYTES or
more now adds GIANT_MB for each MB squared, in a chunk and in a hot piece: of the 76 game files measured none is then
under by more than 250 MB, which the gate's reserve covers. Counted too low, such a file was started beside the others
on a PC without the memory for it, where Windows pages and Linux stops the compiler. Only on a PC short of memory:
giants_counted."""
name = os.path.basename(src)
if name.startswith("code_hot"):
return COMPILER_MB
if name.startswith("code_"):
with open(src, "rb") as f:
text = f.read()
# (each function from the newline before its definition, as this was first measured)
starts = [m.start() - 1 for m in lifted_c.HEAD_BYTES.finditer(text)] + [len(text)]
largest = max([b - a for a, b in zip(starts, starts[1:])] or [0])
return int(400 + 50 * len(text) / (1 << 20) + 250 * largest / (1 << 20))
sizes = [b - a for a, b in zip(starts, starts[1:])]
giant = GIANT_MB * sum((s / (1 << 20)) ** 2 for s in sizes if s >= GIANT_BYTES) if giants_counted() else 0
if name.startswith("code_hot"):
return int(COMPILER_MB + giant)
largest = max(sizes or [0])
return int(max(400 + 50 * len(text) / (1 << 20) + 250 * largest / (1 << 20), 300 + 50 * len(text) / (1 << 20) + giant))
return 1000 if src.endswith(".cpp") else 400
@@ -702,6 +850,19 @@ def compile_cost(src, costs):
return costs.get(name, os.path.getsize(src)) * (HOT_COST if name.startswith("code_hot") else 1)
# HOOKSHOT_MEM_LOG=1: each compile's peak memory beside what mem_need counted it at ("@@mem <file> <peak MB> <counted MB>
# <seconds>"), sampled twice a second over the compiler and the processes it starts. For fitting mem_need and the gate to
# what the compilers really take: off unless asked for, and nothing is sampled then
MEM_LOG = os.environ.get("HOOKSHOT_MEM_LOG", "0") not in ("", "0")
def sample_peak(proc, peak):
"""(a thread of its own, beside communicate) peak[0]: the most MB the compiler's process tree was seen holding"""
while proc.poll() is None:
peak[0] = max(peak[0], rss_mb(proc))
time.sleep(0.5)
def compile_one(job):
cc, src, obj, fl, deps = job
key = PLANNED.get((src, obj)) or object_key(src, fl, deps)
@@ -721,15 +882,20 @@ def compile_one(job):
return src, 0.0, None
lease = GATE.enter(src) if GATE else None
t = time.time()
peak = [0]
try:
with subprocess.Popen([cc] + fl + ["-c", src, "-o", obj], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) as p:
no_throttle(getattr(p, "_handle", None))
if GATE:
GATE.started(lease, p)
if MEM_LOG:
threading.Thread(target=sample_peak, args=(p, peak), daemon=True).start()
out, err = p.communicate()
finally:
if GATE:
GATE.leave(lease)
if MEM_LOG:
print("@@mem %s %d %d %.1f" % (os.path.basename(src), peak[0], mem_need(src), time.time() - t), flush=True)
r = subprocess.CompletedProcess(p.args, p.returncode, out, err)
if r.returncode:
failure = r.stderr or r.stdout or "Compiler exited with status %d without diagnostics." % r.returncode
@@ -739,6 +905,15 @@ def compile_one(job):
return src, time.time() - t, None
# RX_EARLY=1 (nx2/make_game.py, while the game is being translated): only the files that need nothing of the translated
# game are compiled, and nothing is linked: libopus, the WebGPU renderer and the Vulkan back end with its libraries, ~520 of
# the build's ~630 files and a ninth of its compile time. The lift was one process on Windows and left every other
# processor idle for two minutes (its emit has workers there now, nx2/emit_pool.py: half the processors at most); the build that follows finds these objects made (the same keys: their flags and
# dependencies hold nothing the lift writes) and compiles the game's own files alone. The lifted files are not read here:
# the lift is writing them
EARLY = os.environ.get("RX_EARLY") == "1"
def main():
kit.wait_for_quiet('the compile')
p = argparse.ArgumentParser()
@@ -759,7 +934,7 @@ def main():
obj_dir = os.path.join(out, "obj")
gen_dir = os.path.join(out, "gen") # (the logo and the fonts the Vulkan back end compiles in: made with it, below)
os.makedirs(obj_dir, exist_ok=True)
if os.environ.get("RX_VK_ONLY") or a.dry_run: # (the Vulkan back end needs no game: built so with no lift, as the AppImage's is)
if os.environ.get("RX_VK_ONLY") or a.dry_run or EARLY: # (the Vulkan back end needs no game: built so with no lift, as the AppImage's is)
os.makedirs(gen, exist_ok=True)
rt64 = rtgen.write(os.path.join(out, "rt64"))
cc = toolchain.need(toolchain.find("clang"))
@@ -769,15 +944,17 @@ def main():
stamp_def = version.define()
print("version: %s" % version.describe(), flush=True)
# (the lifted chunks include only the guest model: the emulator ABI header is a dependency of the shim alone)
deps = b"".join(open(os.path.join(d, f), "rb").read() for d in (gen, rt64) for f in sorted(os.listdir(d))
if f.endswith(".h"))
# (RX_EARLY: none: the lift is writing those headers, and no file compiled early depends on them)
deps = b"" if EARLY else b"".join(open(os.path.join(d, f), "rb").read() for d in (gen, rt64) for f in sorted(os.listdir(d))
if f.endswith(".h"))
shim_deps = deps + open(os.path.join(NX2_RT, "mkw_host.h"), "rb").read()
eden_gen, eden_out = os.path.join(kit.WORK, "gen"), os.path.join(kit.WORK, "build")
migrate_stamps(eden_gen, os.path.join(eden_out, "obj"),
flags(eden_gen, os.path.join(eden_out, "rt64"), a.opt, not a.no_polls) +
(["-DA64_POLL_CTX"] if a.dll else []), deps) # (as fl below: the flags its objects were built with)
migrate_stamps(gen, obj_dir, fl, deps)
chunks = sorted(f for f in os.listdir(gen) if f.startswith("code_") and f.endswith(".c"))
if not EARLY:
migrate_stamps(eden_gen, os.path.join(eden_out, "obj"),
flags(eden_gen, os.path.join(eden_out, "rt64"), a.opt, not a.no_polls) +
(["-DA64_POLL_CTX"] if a.dll else []), deps) # (as fl below: the flags its objects were built with)
migrate_stamps(gen, obj_dir, fl, deps)
chunks = [] if EARLY else sorted(f for f in os.listdir(gen) if f.startswith("code_") and f.endswith(".c"))
if a.only:
want = {"code_%03d.c" % int(n) for n in a.only.split(",")}
chunks = [c for c in chunks if c in want]
@@ -799,6 +976,8 @@ def main():
srcs += [os.path.join(NX2_RT, "eden_shim.c"), write_info(out)]
if os.environ.get("RX_VK_ONLY"): # (no lift needed: the back end alone)
srcs = [s for s in srcs if os.path.exists(s)]
if EARLY: # (the lift's files are on their way: none of them now)
srcs = []
if a.dry_run: # (the lift's files: there is no game here)
srcs = [s for s in srcs if os.path.dirname(s) != gen]
# the longest first, so the long ones do not end up alone at the end (compile_cost)
@@ -864,6 +1043,21 @@ def main():
for rel in open(os.path.join(OPUS, "sources.txt")).read().split():
jobs.append((cc, os.path.join(OPUS, rel), os.path.join(obj_dir, "opus_" + rel.replace("/", "_")[:-2] + ".o"),
oflags, b"opus-1.5.2"))
# (Triforce) zstd's decoder for the runtime too (third_party/zstd: the same files the Vulkan back end compiles into its
# DLL for the game's fonts, further down, where their names stay inside it). host/hos_zstd.c decodes the game's files
# with it when HOS_HOST_ZSTD is set (host/host_zstd.h). Its names are hidden (zstd.h marks its functions visible by
# itself), so that they are never among the names the program exports: on Linux those come first for every library
# loaded into it, and a GPU driver that uses the system's libzstd would be handed this decoder for half of its calls
if host_srcs:
hzstd = os.path.normpath(os.path.join(kit.HERE, "..", "third_party", "zstd", "lib"))
hzflags = target() + CPU + ["-O2", "-g0", "-DNDEBUG", "-Wno-everything", "-DZSTD_DISABLE_ASM", "-DZSTD_LEGACY_SUPPORT=0",
"-DZSTD_MULTITHREAD=0", "-fvisibility=hidden", "-DZSTDLIB_VISIBLE=", "-DZSTDERRORLIB_VISIBLE="]
hzdeps = b"".join(open(os.path.join(d, f), "rb").read()
for d in (hzstd, os.path.join(hzstd, "common"), os.path.join(hzstd, "decompress"))
for f in sorted(os.listdir(d)) if f.endswith(".h"))
for f in HOST_ZSTD_SOURCES:
jobs.append((cc, os.path.join(hzstd, *f.split("/")), os.path.join(obj_dir, "hostzstd_" + os.path.basename(f)[:-2] + ".o"),
hzflags, hzdeps))
# the renderer (C++20, NDEBUG: no asserts in it or suyu), when a wgpu-native release is there
wgpu = a.standalone and not a.only and not LINUX and os.path.isfile(os.path.join(WGPU, "include", "webgpu", "webgpu.h"))
if a.standalone and not a.only and not wgpu and not LINUX:
@@ -917,6 +1111,8 @@ def main():
with open(f, "rb") as src:
stamp.update(os.path.relpath(f, vk).replace("\\", "/").encode() + src.read().replace(bytes((13, 10)), bytes((10,))))
shader_stamp = "-DHOOKSHOT_SHADER_STAMP=0x%sull" % stamp.hexdigest()[:16]
# ((Triforce) and what of those translates: the program cache is carried over an update by it, translation_stamp)
translation_def = "-DHOOKSHOT_TRANSLATION_STAMP=0x%sull" % translation_stamp(vk, translator)
vflags = target() + CPU + (["-fPIC", "-I" + POSIX, "-D_GNU_SOURCE"] if LINUX else []) + [ # (hookshot: the replay is the frame's limit now)
# (what the code takes for granted, which newer C++ libraries bring in by themselves and older ones do not: issue 48)
"-include", "algorithm", "-include", "cstdio", "-include", "cstdlib", "-include", "cstring", "-include", "functional",
@@ -976,7 +1172,7 @@ def main():
"-DIMGUI_DISABLE_WIN32_FUNCTIONS", "-DIMGUI_DISABLE_DEMO_WINDOWS", "-DIMGUI_DISABLE_DEBUG_TOOLS"]
imgui_deps = b"".join(open(os.path.join(d, f), "rb").read() for d in (imgui, os.path.join(imgui, "backends"), zstd)
for f in sorted(os.listdir(d)) if f.endswith(".h"))
imgui_users = ("vk_imgui.cpp", "vk_menu.cpp")
imgui_users = ("vk_imgui.cpp", "vk_menu.cpp", "vk_salesman.cpp") # (vk_salesman.cpp: the game's files through vk_game_font.h)
for f in ("imgui.cpp", "imgui_draw.cpp", "imgui_tables.cpp", "imgui_widgets.cpp", "backends/imgui_impl_vulkan.cpp"):
vk_jobs.append((cxx, os.path.join(imgui, *f.split("/")),
os.path.join(obj_dir, "vk_imgui_" + os.path.basename(f)[:-4] + ".o"), vflags + imgui_flags, imgui_deps))
@@ -997,7 +1193,7 @@ def main():
name = os.path.relpath(src, vk).replace(os.sep, "_")[:-4]
lib = os.path.relpath(src, vk).split(os.sep)[0] in ("eden", "sirit")
vk_jobs.append((cxx, src, os.path.join(obj_dir, "vk_" + name + ".o"),
vflags + ([shader_stamp] if os.path.basename(src) == "vk_shader.cpp" else []) +
vflags + ([shader_stamp, translation_def] if os.path.basename(src) == "vk_shader.cpp" else []) +
([stamp_def] if os.path.basename(src) == "vk_menu.cpp" else []) +
(ngx_flags if os.path.basename(src) == "vk_dlss.cpp" else []) +
(sl_flags if os.path.basename(src) == "vk_dlssg.cpp" else []) +
@@ -1024,6 +1220,8 @@ def main():
vk_only = bool(os.environ.get("RX_VK_ONLY")) # (the renderer alone, the game's objects as they are)
if vk_only:
jobs = list(vk_jobs)
if EARLY: # (by their objects' names: libopus, the WebGPU renderer, the Vulkan back end; never the runtime's host_*.o)
jobs = [j for j in jobs if os.path.basename(j[2]).startswith(("opus_", "gpu_", "vk_"))]
if a.dry_run: # (the compile starts here: what it would read, said and checked, and nothing compiled)
missing = sorted({j[1] for j in jobs if not os.path.isfile(j[1])} |
({vk_prebuilt} if vk_prebuilt and not os.path.isfile(vk_prebuilt) else set()))
@@ -1087,6 +1285,8 @@ def main():
print("compiled in %.0fs, %d failed" % (time.time() - t0, still), flush=True)
if still:
return 1
if EARLY: # (nothing to link yet: the game's own objects come with the build after the lift)
return 0
vk_objs = {j[2] for j in vk_jobs}
objs = [j[2] for j in jobs if j[2] not in vk_objs]
rsp = os.path.join(out, "objs.rsp")
@@ -1171,9 +1371,9 @@ def main():
"-fuse-ld=lld", "-shared", "-o", vdll, "-Wl,-Map=" + vdll + ".map", "-Wl,-Bsymbolic", "-Wl,--gc-sections",
"-static-libstdc++", "-static-libgcc", "@" + vrsp] + \
([ngx_library(NGX_SDK, True)] if NGX_SDK else []) + ["-ldl", "-lpthread", "-lm"]
else:
else: # (Triforce: advapi32 for RivaTuner Statistics Server's folder in the registry, gpu/vk/rtss_sys_win.cpp)
link = [cc] + toolchain.native_target() + toolchain.native_link() + \
["-shared", "-o", vdll, toolchain.link_opt("/map:" + vdll[:-4] + ".map"), "@" + vrsp, "-luser32"] + (["-lc++", "-lunwind"] if toolchain.mingw() else []) + ngx_link(NGX_SDK, toolchain.mingw())
["-shared", "-o", vdll, toolchain.link_opt("/map:" + vdll[:-4] + ".map"), "@" + vrsp, "-luser32", "-ladvapi32"] + (["-lc++", "-lunwind"] if toolchain.mingw() else []) + ngx_link(NGX_SDK, toolchain.mingw())
r = subprocess.run(link, capture_output=True, text=True)
if r.returncode:
undefined = sorted({l.strip() for l in (r.stdout + r.stderr).splitlines() if "undefined symbol" in l})

View File

@@ -13,6 +13,11 @@ The functions' C is made by several processes (fork, so Linux and macOS; one pro
consecutive functions from the lifter as it stands, and write() takes the texts in its own order, as if it had made
them. The lifter's counters (stats, unhandled, ijump, and any passed in `counters`, such as poll.stats) get each
process's additions. The output is the one-process output, byte for byte (tests/test_parallel_emit.py).
(Triforce) Where there is no fork (Windows) the processes are spawned and sent what they need (nx2/emit_pool.py), and the same
processes then cut the big chunks (split_big) and share the page marks (edit_c), a file each: 46 s, 6 s and 19 s of the lift's
127 s in one process on a 16-thread PC, 13 s, 1.4 s and 5 s of its 69 s with 8 processes. HOOKSHOT_LIFT_JOBS=1 is the one
process, as before, on every system.
"""
import bisect
import collections
@@ -20,9 +25,11 @@ import glob
import multiprocessing
import os
import re
import threading
import whole_write # (rewrites written beside and moved over: issue 1197)
import a64_lift
import emit_pool # ((Triforce) the emit's processes where there is no fork)
import lifted_c
FILE = "chunks.txt"
@@ -101,7 +108,8 @@ def _emit_run(es):
def _available_gb():
"""MemAvailable from /proc/meminfo in GB (None where there is none)"""
"""MemAvailable from /proc/meminfo in GB; (Triforce) on Windows the physical memory GlobalMemoryStatusEx calls available
(None where there is neither)"""
try:
with open("/proc/meminfo", encoding="ascii") as f:
for line in f:
@@ -109,22 +117,51 @@ def _available_gb():
return int(line.split()[1]) / (1024 * 1024)
except (OSError, ValueError, IndexError):
pass
if os.name == "nt":
try:
import ctypes
class Status(ctypes.Structure):
_fields_ = [("dwLength", ctypes.c_ulong), ("dwMemoryLoad", ctypes.c_ulong)] + [
(name, ctypes.c_ulonglong) for name in ("ullTotalPhys", "ullAvailPhys", "ullTotalPageFile", "ullAvailPageFile",
"ullTotalVirtual", "ullAvailVirtual", "ullAvailExtendedVirtual")]
s = Status()
s.dwLength = ctypes.sizeof(s)
if ctypes.WinDLL("kernel32").GlobalMemoryStatusEx(ctypes.byref(s)):
return s.ullAvailPhys / (1 << 30)
except (OSError, AttributeError, ValueError):
pass
return None
FORK = "fork" in multiprocessing.get_all_start_methods()
SPAWN_MOST = 8 # ((Triforce) spawned emit processes at most: the lift's own process cannot keep more busy, emit_pool.py)
SPAWN_GB = 1 # (and one per GB of available memory: one holds its run or a file of lifted C, 480 MB at the most seen)
SPAWN_LEAST = 2 * emit_pool.RUN_FUNCS # (functions below which the lift stays one process: starting the others costs more)
def emit_jobs():
"""Processes for the emit: HOOKSHOT_LIFT_JOBS, else the cores this process may use (at most 16; each holds its run's
texts and the pages of the lift it touches), and at most one per 2 GB of available memory: a Steam Deck (16 GB) ran
out with 8, and the system's out-of-memory handling stopped the lift ("lift.py stopped (signal 15): BrokenPipeError")
or one worker (the update then sat at "jump tables" for good). 1 where there is no fork"""
if "fork" not in multiprocessing.get_all_start_methods():
return 1
or one worker (the update then sat at "jump tables" for good).
(Triforce) Where there is no fork (Windows; before, always 1 there): half the processors, at most SPAWN_MOST, and at most
one per SPAWN_GB of available memory. Half, because the files that need no translated code are compiled beside the lift
there (make_game's EarlyCompile, one compiler a processor less one) and because the lift's own process, which sends the
runs and writes the C, is what more of them would wait for (emit_pool.py has the measure). 1 is the one process on
every system"""
if os.environ.get("HOOKSHOT_LIFT_JOBS"):
return max(1, int(os.environ["HOOKSHOT_LIFT_JOBS"]))
try:
n = len(os.sched_getaffinity(0))
except AttributeError:
n = os.cpu_count() or 1
if not FORK:
n = max(1, min(SPAWN_MOST, n // 2))
gb = _available_gb()
if gb is not None:
n = max(1, min(n, int(gb // SPAWN_GB)))
return n
n = max(1, min(16, n))
gb = _available_gb()
if gb is not None:
@@ -169,23 +206,65 @@ def parallel_emit(emit, order, texts, counters, jobs, run=2000):
yield (e,) + tuple(emit(e, texts))
def write(L, gen_dir, img_dir, rechunk=False, counters=(), jobs=None):
_workers = None # ((Triforce) the spawned processes of this lift, from write() until edit_c()'s marks end, emit_pool.Workers)
_marks = None # (the page marks in flight after edit_c(wait=False): the thread waiting for them, what ended it, the folder)
def stop_workers(kill=False):
"""(Triforce) The spawned processes ended, when there are any: after what they are doing, or (kill) now"""
global _workers
w, _workers = _workers, None
if w is not None:
w.stop(kill)
def drop_lifted(gen_dir):
"""(Triforce) The lifted C taken away after a lift that stopped while its processes were writing it: the chunks, their
parts, the hot file and its pieces, whole or cut short. What is left of such a lift looks like a lift (every file that
is there may end as a function does, and an earlier lift's a64_table.c still names every function), and the next lift
would take its chunk boundaries from it (seed_from_gen) and a build its files. The same files make_game takes away
before a lift"""
for f in os.listdir(gen_dir) if os.path.isdir(gen_dir) else ():
if f.startswith("code_") and f.endswith((".c", ".c.whole", ".part")):
try:
os.remove(os.path.join(gen_dir, f))
except OSError:
pass # (held by something else: the lift's own failure is what is said)
def gave_up(gen_dir):
"""(Triforce) A lift with spawned processes that failed: they are ended now and its C goes"""
stop_workers(kill=True)
drop_lifted(gen_dir)
def write(L, gen_dir, img_dir, rechunk=False, counters=(), jobs=None, setup=()):
"""L.write(gen_dir) with stable boundaries; returns its files. counters: dicts the emit adds to besides the
lifter's own (poll.stats); jobs: emit processes (emit_jobs())"""
lifter's own (poll.stats); jobs: emit processes (emit_jobs()); (Triforce) setup: what set the lifter's class up for
the emit in this process (poll.install), module-level functions the spawned processes call before they emit"""
global _workers
starts = [] if rechunk else load(img_dir)
if not starts and not rechunk:
starts = seed_from_gen(gen_dir)
b = Boundaries(a64_lift.CHUNK_BYTES, starts, L.funcs)
base_emit = L.emit_function
jobs = emit_jobs() if jobs is None else jobs
spawned = jobs > 1 and not FORK and len(L.funcs) >= SPAWN_LEAST
made = None
def emit_function(e, texts):
nonlocal made
global _workers
b.current = e
if made is None: # (the first call: write() has its texts now, and asks for the functions in sorted order)
made = parallel_emit(base_emit, sorted(L.funcs), texts,
[L.stats, L.unhandled, L.ijump] + list(counters), jobs)
every = [L.stats, L.unhandled, L.ijump] + list(counters)
if spawned:
stop_workers(kill=True) # (an earlier write()'s, which no edit_c() ended)
_workers = emit_pool.Workers(jobs, L, every, setup)
print("emit: %d processes (HOOKSHOT_LIFT_JOBS=<n> for another count, 1 for one process)" % _workers.jobs, flush=True)
made = emit_pool.emit(_workers, L, sorted(L.funcs), texts, every)
else:
made = parallel_emit(base_emit, sorted(L.funcs), texts, every, jobs)
got = next(made, None)
if got is None or got[0] != e:
raise SystemExit("chunks: write() asked for f_%08x, the emit made %s" % (e, "f_%08x" % got[0] if got else "nothing"))
@@ -193,6 +272,10 @@ def write(L, gen_dir, img_dir, rechunk=False, counters=(), jobs=None):
L.emit_function = emit_function
try:
files = L.write(gen_dir, chunk_bytes=b)
except BaseException:
if spawned: # ((Triforce) a process failed, died or hung, or this one was stopped: no part of a lift is left behind)
gave_up(gen_dir)
raise
finally:
del L.emit_function
if made is not None:
@@ -210,7 +293,12 @@ def write(L, gen_dir, img_dir, rechunk=False, counters=(), jobs=None):
for f in os.listdir(gen_dir):
if f == "code_hot.c.whole" or HOT_PIECE.fullmatch(f):
os.remove(os.path.join(gen_dir, f))
files += split_big(gen_dir, files)
try:
files += split_big(gen_dir, files, workers=_workers if spawned else None)
except BaseException:
if spawned:
gave_up(gen_dir)
raise
save(img_dir, starts or b.recorded)
print("chunks: %d, boundaries %s" % (len(files), "kept" if starts else "from sizes (saved)"))
return files
@@ -251,43 +339,57 @@ def write_costs(gen_dir, costs):
f.write("".join("%s %d\n" % kv for kv in sorted(known.items())))
def split_big(gen_dir, files, budget=None):
def split_one(gen_dir, path, budget):
"""split_big for one chunk (a spawned process's share of it too): the new parts' paths, {file name: cost} of the chunk or of
each of its parts, and what to say of a chunk that was cut (None of one left whole)"""
m = re.fullmatch(r"code_(\d{3})\.c", os.path.basename(path))
made, costs = [], {}
with open(path, encoding="utf-8", newline="") as f:
text = f.read()
marks = [h.start() for h in HEAD.finditer(text)]
ends = marks[1:] + [len(text)]
cost = [fn_cost(b - a) for a, b in zip(marks, ends)]
total = sum(cost)
if total <= budget * 3 // 2 or len(marks) < 2:
costs[os.path.basename(path)] = marks[0] + total if marks else len(text)
return made, costs, None
parts = -(-total // budget)
cuts, run, k = [0], 0, 1 # (indexes into marks where a part starts)
for i, c in enumerate(cost):
if i and run >= total * k // parts:
cuts.append(i)
k = run * parts // total + 1 # (past a giant function: the next share, not a sliver after it)
run += c
bounds = [marks[i] for i in cuts] + [len(text)]
preamble = text[:marks[0]]
for j in range(len(cuts)):
out = path if j == 0 else os.path.join(gen_dir, "code_%s_%d.c" % (m.group(1), j))
piece = text[:bounds[1]] if j == 0 else preamble + text[bounds[j]:bounds[j + 1]]
whole_write.write_whole(out, piece)
costs[os.path.basename(out)] = len(preamble) + sum(cost[cuts[j]:cuts[j + 1] if j + 1 < len(cuts) else len(cost)])
if j:
made.append(out)
return made, costs, "chunks: %s (%d MB, cost %d MB) in %d parts" % (os.path.basename(path), len(text) >> 20, total >> 20, len(cuts))
def split_big(gen_dir, files, budget=None, workers=None):
"""Chunks whose estimated compile time (fn_cost) is well past a plain chunk's, cut at function starts into parts of about equal
cost: code_NNN.c, code_NNN_1.c ... A chunk of the giant functions took 350 s, alone at the end of the build, where a chunk of
ordinary functions takes 45; the boundaries kept in chunks.txt also let chunks grow (44 MB beside 25). The first part keeps the
name, so seed_from_gen and the boundaries are as before. Writes chunk_costs.txt for every chunk; returns the new parts' paths"""
name, so seed_from_gen and the boundaries are as before. Writes chunk_costs.txt for every chunk; returns the new parts' paths.
(Triforce) workers: the spawned processes that take a chunk each (emit_pool.Workers; None: this process takes them all)"""
budget = budget or a64_lift.CHUNK_BYTES
made, costs = [], {}
for path in files:
m = re.fullmatch(r"code_(\d{3})\.c", os.path.basename(path))
if not m:
continue
with open(path, encoding="utf-8", newline="") as f:
text = f.read()
marks = [h.start() for h in HEAD.finditer(text)]
ends = marks[1:] + [len(text)]
cost = [fn_cost(b - a) for a, b in zip(marks, ends)]
total = sum(cost)
if total <= budget * 3 // 2 or len(marks) < 2:
costs[os.path.basename(path)] = marks[0] + total if marks else len(text)
continue
parts = -(-total // budget)
cuts, run, k = [0], 0, 1 # (indexes into marks where a part starts)
for i, c in enumerate(cost):
if i and run >= total * k // parts:
cuts.append(i)
k = run * parts // total + 1 # (past a giant function: the next share, not a sliver after it)
run += c
bounds = [marks[i] for i in cuts] + [len(text)]
preamble = text[:marks[0]]
for j in range(len(cuts)):
out = path if j == 0 else os.path.join(gen_dir, "code_%s_%d.c" % (m.group(1), j))
piece = text[:bounds[1]] if j == 0 else preamble + text[bounds[j]:bounds[j + 1]]
whole_write.write_whole(out, piece)
costs[os.path.basename(out)] = len(preamble) + sum(cost[cuts[j]:cuts[j + 1] if j + 1 < len(cuts) else len(cost)])
if j:
made.append(out)
print("chunks: %s (%d MB, cost %d MB) in %d parts" % (os.path.basename(path), len(text) >> 20, total >> 20, len(cuts)))
paths = [path for path in files if re.fullmatch(r"code_(\d{3})\.c", os.path.basename(path))]
if workers is None:
answers = (split_one(gen_dir, path, budget) for path in paths)
else:
answers = workers.each(split_one, (("the split of %s" % os.path.basename(path), (gen_dir, path, budget)) for path in paths))
for new, cost, said in answers: # (in the chunks' order, whoever cut them)
made += new
costs.update(cost)
if said:
print(said)
for f in os.listdir(gen_dir): # (parts of an earlier lift that this one did not make)
if PART.fullmatch(f) and os.path.join(gen_dir, f) not in made:
os.remove(os.path.join(gen_dir, f))
@@ -295,10 +397,82 @@ def split_big(gen_dir, files, budget=None):
return made
def edit_c(gen, stats, supported, whole=True):
def marks_one(path):
"""(Triforce) a64_lift.group_marks for one file, in a spawned process: whether it changed the file, and its counts"""
st = collections.Counter()
return a64_lift.group_marks([path], st), list(st.items())
def ask_marks(paths, workers):
"""(Triforce) a64_lift.group_marks asked of the spawned processes, a file each, the largest first (the hot file, 82 MB,
is the last by name and would end alone). Every file is asked for here, at once: a lift that ends on an error of its
own while they are at it asks for nothing more, and the processes finish the files, all of them. What take_marks() takes"""
by_size = sorted(paths, key=lambda p: (-os.path.getsize(p), p))
return by_size, workers.ask(marks_one, [("the page marks of %s" % os.path.basename(p), (p,)) for p in by_size])
def take_marks(paths, stats, workers, asked):
"""(Triforce) Waits for what ask_marks() asked: the counts go into stats in the paths' order, as one process adds them
when it groups the files itself. The files changed"""
by_size, calls = asked
answers = dict(zip(by_size, workers.answers(calls)))
changed = 0
for p in paths:
n, counts = answers[p]
changed += n
for k, v in counts:
stats[k] = stats.get(k, 0) + v
return changed
def edit_c(gen, stats, supported, whole=True, wait=True):
"""(nx2/lift.py, after the lift writes its C; here, beside the writing, so it imports without the lift's tools) The edits of the written C, in this order: the game's patches (nx2/oot_patches.py, which find the lift's exact text; the
supported build's whole standalone lift fails when an output-size edit finds no place), then, last, the page marks of each
group of stores shared (a64_lift.group_marks), which changes that text. stats: the lift's, for its report"""
group of stores shared (a64_lift.group_marks), which changes that text. stats: the lift's, for its report.
(Triforce) The marks are made by write()'s spawned processes when it left any, which end with them. wait=False: this
returns while they are at it, and marks_done() waits for them: the lift writes its last files meanwhile (hos_image.c takes
as long as the marks, and reads nothing of the C); stats is theirs until then"""
global _marks
import oot_patches
oot_patches.apply(gen, protect_hearts=supported, required=supported and whole)
a64_lift.group_marks(sorted(glob.glob(os.path.join(glob.escape(gen), "code_*.c"))), stats)
try:
oot_patches.apply(gen, protect_hearts=supported, required=supported and whole)
except BaseException:
stop_workers(kill=True) # (the patches' own failure, said as before: the C stays as it is, for whoever looks into it)
raise
paths = sorted(glob.glob(os.path.join(glob.escape(gen), "code_*.c")))
workers = _workers
if workers is None:
a64_lift.group_marks(paths, stats)
return
try:
asked = ask_marks(paths, workers)
except BaseException:
gave_up(gen)
raise
ended = []
def marks(): # (this thread only waits for the processes and adds their counts up)
try:
take_marks(paths, stats, workers, asked)
except BaseException as e:
ended.append(e)
thread = threading.Thread(target=marks)
thread.start()
_marks = (thread, ended, gen)
if wait:
marks_done()
def marks_done():
"""(Triforce) After edit_c(wait=False): waits for the page marks the spawned processes are making, and ends the
processes. Nothing to do when edit_c made the marks itself (one process, or wait)"""
global _marks
m, _marks = _marks, None
if m is None:
return
thread, ended, gen = m
thread.join()
if ended: # (a process failed in the middle of the files: some marked, some not, one perhaps cut short)
gave_up(gen)
raise ended[0]
stop_workers()

480
nx2/code_mods.py Normal file
View File

@@ -0,0 +1,480 @@
"""(Triforce) Code mods: the mods' patches of the game's code, applied before the game is translated.
The game's code is compiled ahead of time, so a patch of it cannot be applied while the game runs (host/mods.c loads the
RomFS mods then). A build applies them instead: nx2/make_game.py calls build_step() on its own copy of the game's code
(<work>/exefs, which the lift reads), never on the player's files and never on the copy the game folder keeps for updates
(.triforce_exefs). The patched bytes are in the modules the lift key hashes (make_game.py lift_key), so a patch added,
taken away, switched off or edited makes the next build translate the game again, and nothing else does.
Where the patches are, as emulators keep them and as host/mods.c finds its mods:
<game folder>/mods/<name>/exefs/<build id>.ips IPS ("PATCH" ... "EOF") or IPS32 ("IPS32" ... "EEOF")
<game folder>/mods/<name>/exefs/<any name>.pchtxt IPSwitch text, its build id in the @nsobid- line
A mod may sit deeper (mods/contents/<title id>/<name>/exefs): any folder up to four levels down that holds a romfs, exefs
or cheats folder is a mod, and nothing below it is looked at. A folder whose name starts with '-' or '_' is off, one that
starts with '.' is skipped. Mods are taken in the order of their names (ASCII letters' case ignored), a mod's patch files
in the order of theirs, and where two write the same byte the later one wins; the build says so.
A patch names the build id of the module it is for (main, a subsdk, sdk or rtld): the .ips file's name, the .pchtxt
file's @nsobid- line, hex, trailing zeros left out or not. One for another build is refused with a line that says so.
Offsets are as emulators and Atmosphere read them: they count the NSO's 0x100-byte header before the module's image (a
.pchtxt offset after its "@flag offset_shift" is added). A file is applied whole or not at all: one that is cut short,
that cannot be read, or that writes outside the module's three segments (.text, .rodata, .data) is refused. That
leaves out code caves, patches that put new code into the empty space after .text: the translator has not been tried on
them.
What a build did is kept in the game folder, triforce_code_mods.txt (RECORD, write_record): the next build, an update
(--update) among them, reads the mods folder again, builds the same patches in again and says which of the last build's
are gone or changed; the game compares the file with the mods folder at its start (host/mods.c, host/mod_code.h) and
says in its log which patch files wait for a build.
Standard library only (lz4 too for a module that is LZ4-compressed, as the translator's own loader needs it). Not
imported by the lift: this file is not part of the lift key (make_game.py LIFT_KEY_NOT), what it does to the code is.
"""
import hashlib
import os
import re
import struct
NSO_HEADER = 0x100
SEGMENTS = (".text", ".rodata", ".data")
PATCH_SUFFIXES = (".ips", ".pchtxt")
PARTS = ("romfs", "exefs", "cheats") # (a folder holding one of these is a mod: host/mods.c find_mods)
MOD_DEPTH = 4
RECORD = "triforce_code_mods.txt"
RECORD_HEAD = ("# The code patches (.ips, .pchtxt) this game was built with: nx2/code_mods.py writes it at every build, and\n"
"# the game reads it at its start. Tab separated: applied or refused, the file's FNV-1a 64 and size, its\n"
"# path under the mods folder, then the module and the bytes changed, or why it was refused.\n")
class Refused(ValueError):
"""A patch file that is not applied; the text says why"""
# ---- the formats: [(offset in the module's image, bytes)] in the file's order
def parse_ips(data):
"""The writes of an IPS or IPS32 file. Refused when it is neither, or cut short"""
if data[:5] == b"PATCH":
width, end = 3, b"EOF"
elif data[:5] == b"IPS32":
width, end = 4, b"EEOF"
else:
raise Refused("it is not an IPS file (no PATCH or IPS32 at its start)")
writes, at = [], 5
while True:
# (the end's letters could be a record's offset too: they are the end only where the file ends, a truncation
# length of up to three bytes after them)
if data[at:at + len(end)] == end and len(data) - at - len(end) <= 3:
return writes
if at + width + 2 > len(data):
raise Refused("it is cut short (no %s at its end)" % end.decode())
offset = int.from_bytes(data[at:at + width], "big")
size = int.from_bytes(data[at + width:at + width + 2], "big")
at += width + 2
if size:
body = data[at:at + size]
at += size
if len(body) != size:
raise Refused("it is cut short (a record of %d bytes at file offset %#x has %d)" % (size, at - size, len(body)))
else: # (a run: its length, then the one byte it repeats)
if at + 3 > len(data):
raise Refused("it is cut short (in a run at file offset %#x)" % at)
body = data[at + 2:at + 3] * int.from_bytes(data[at:at + 2], "big")
at += 3
if offset < NSO_HEADER:
raise Refused("it writes the module's header (offset %#x: an offset counts the 0x100-byte NSO header)" % offset)
if body:
writes.append((offset - NSO_HEADER, bytes(body)))
ESCAPES = {"a": 7, "b": 8, "f": 12, "n": 10, "r": 13, "t": 9, "v": 11, "\\": 92, "'": 39, '"': 34, "?": 63, "0": 0}
PATCH_LINE = re.compile(r"([0-9A-Fa-f]{8})[ \t]+(\S.*)$")
def pchtxt_value(text, little_endian, where):
"""The bytes of a patch line's value: hex digits in byte order (reversed after @little-endian), or a string in double
quotes with C's one-letter escapes, as UTF-8 and without a zero after it"""
if text.startswith('"'):
out, i = bytearray(), 1
while i < len(text) and text[i] != '"':
ch = text[i]
if ch == "\\":
i += 1
if i >= len(text) or text[i] not in ESCAPES:
raise Refused("%s: an escape it does not know in a string" % where)
out.append(ESCAPES[text[i]])
else:
out += ch.encode("utf-8")
i += 1
if i >= len(text):
raise Refused("%s: a string without its closing quote" % where)
return bytes(out)
digits = re.split(r"[ \t/]", text, maxsplit=1)[0]
if len(digits) % 2 or not re.fullmatch(r"[0-9A-Fa-f]+", digits):
raise Refused("%s: %r is not a value (hex digits, two a byte)" % (where, digits[:40]))
value = bytes.fromhex(digits)
return value[::-1] if little_endian else value
def parse_pchtxt(text):
"""(build id or None, writes) of an IPSwitch file. Patch lines count between @enabled and @disabled, @stop ends the
file; a line that is neither a directive nor "<8 hex digits> <value>" is a title or a note and is passed over. A
patch line whose value cannot be read refuses the file"""
build, shift, enabled, little, writes = None, 0, False, False, []
for number, raw in enumerate(text.splitlines(), 1):
line = raw.strip()
if not line or line.startswith("//") or line.startswith("#"):
continue
if line.startswith("@"):
word = line.split()[0].lower()
if word == "@stop":
break
if word.startswith("@nsobid-"):
build = line.split()[0][len("@nsobid-"):]
elif word == "@enabled":
enabled = True
elif word == "@disabled":
enabled = False
elif word == "@little-endian":
little = True
elif word == "@big-endian":
little = False
elif word == "@flag":
f = line.split()
if len(f) >= 3 and f[1] == "offset_shift":
try:
shift = int(f[2], 0)
except ValueError:
raise Refused("line %d: %r is not a number for offset_shift" % (number, f[2])) from None
continue
m = PATCH_LINE.match(line)
if not m or not enabled:
continue
value = pchtxt_value(m.group(2), little, "line %d" % number)
offset = int(m.group(1), 16) + shift
if offset < NSO_HEADER:
raise Refused("line %d: it writes the module's header (offset %#x after the shift: an offset counts the "
"0x100-byte NSO header)" % (number, offset))
if value:
writes.append((offset - NSO_HEADER, value))
return build, writes
def same_build(named, build_id):
"""Whether a patch's build id (hex, any case, trailing zeros left out or not) is a module's (hex of its 32 bytes)"""
named = named.lower()
return bool(re.fullmatch(r"[0-9a-f]{1,64}", named)) and named.ljust(64, "0") == build_id.lower().ljust(64, "0")
# ---- the mods folder, as host/mods.c walks it
def name_key(name):
"""A name as host/mods.c orders it (_stricmp: bytes, ASCII letters' case ignored)"""
return bytes(b + 32 if 65 <= b <= 90 else b for b in name.encode("utf-8", "surrogateescape"))
def find_mods(root):
"""[(name, folder)] of the mods under root, by name: a mod's name is its path under root, with /"""
out = []
def walk(folder, name, depth):
if depth > 0 and any(os.path.exists(os.path.join(folder, part)) for part in PARTS):
out.append((name, folder))
return
if depth >= MOD_DEPTH:
return
try:
entries = sorted(os.listdir(folder))
except OSError:
return
for entry in entries:
sub = os.path.join(folder, entry)
if entry[:1] in (".", "-", "_") or not os.path.isdir(sub):
continue
walk(sub, name + "/" + entry if name else entry, depth + 1)
if os.path.isdir(root):
walk(root, "", 0)
return sorted(out, key=lambda mod: name_key(mod[0]))
def patch_files(root):
"""[(path under the mods folder, file)] of the mods' code patches, in the order they are applied"""
out = []
for name, folder in find_mods(root):
exefs = os.path.join(folder, "exefs")
try:
entries = sorted(os.listdir(exefs), key=name_key) if os.path.isdir(exefs) else []
except OSError:
entries = []
for entry in entries:
path = os.path.join(exefs, entry)
if entry.lower().endswith(PATCH_SUFFIXES) and os.path.isfile(path):
out.append(("%s/exefs/%s" % (name, entry), path))
return out
def fnv64(data):
"""FNV-1a 64 of a file's bytes, 16 hex digits: what the record names a patch file's contents by (host/mod_code.h
makes the same number)"""
h = 0xCBF29CE484222325
for b in data:
h = ((h ^ b) * 0x100000001B3) & 0xFFFFFFFFFFFFFFFF
return "%016x" % h
# ---- the modules of the copy that is translated
class Module:
"""One NSO of the exefs: its build id and its three segments; the image only once a patch is for it"""
def __init__(self, path):
self.path, self.name = path, os.path.basename(path)
with open(path, "rb") as f:
self.head = f.read(NSO_HEADER)
if len(self.head) < NSO_HEADER or self.head[:4] != b"NSO0":
raise ValueError("%s is not a game module" % path)
self.build_id = self.head[0x40:0x60].hex()
self.flags = struct.unpack_from("<I", self.head, 0xC)[0]
# (each: file offset, memory offset, size)
self.segments = [struct.unpack_from("<3I", self.head, 0x10 + 0x10 * i) for i in range(3)]
self.file_sizes = struct.unpack_from("<3I", self.head, 0x60)
self.raw = self.image = self.original = None
def load(self):
if self.image is not None:
return
if self.flags & 0x80:
raise ValueError("%s is still ZBIC-compressed (nx2/unpack.py makes it plain first)" % self.path)
with open(self.path, "rb") as f:
self.raw = f.read()
self.image = bytearray(max(memory + size for _, memory, size in self.segments))
for i, ((offset, memory, size), stored) in enumerate(zip(self.segments, self.file_sizes)):
data = self.raw[offset:offset + stored]
if self.flags & (1 << i):
import lz4.block # (as third_party/a64-kit/tools/nso.py reads it)
data = lz4.block.decompress(data, uncompressed_size=size)
if len(data) != size:
raise ValueError("%s: its %s has %d bytes, %d expected" % (self.path, SEGMENTS[i], len(data), size))
self.image[memory:memory + size] = data
self.original = bytes(self.image)
def segment_of(self, offset, size):
"""The segment that holds all of [offset, offset + size), or None"""
for i, (_, memory, length) in enumerate(self.segments):
if memory <= offset and offset + size <= memory + length:
return i
return None
def write(self):
"""The module written again as a plain NSO with the image as it is now: the header, what lies between it and
the first segment (the module's name), the three segments one after the other, their hashes made again"""
head = bytearray(self.head)
body = bytearray(self.raw[NSO_HEADER:max(NSO_HEADER, min(offset for offset, _, _ in self.segments))])
for i, (_, memory, size) in enumerate(self.segments):
data = bytes(self.image[memory:memory + size])
struct.pack_into("<I", head, 0x10 + 0x10 * i, NSO_HEADER + len(body))
struct.pack_into("<I", head, 0x60 + 4 * i, size)
head[0xA0 + 0x20 * i:0xC0 + 0x20 * i] = hashlib.sha256(data).digest()
body += data
struct.pack_into("<I", head, 0xC, self.flags & ~7) # (no segment compressed; the hash bits stay)
tmp = self.path + ".part" # (make_game.py check_modules passes over a .part)
with open(tmp, "wb") as f:
f.write(head)
f.write(body)
os.replace(tmp, self.path)
def load_modules(exefs):
"""The exefs's modules by name: every file of it that is an NSO (main.npdm is none)"""
out = {}
for name in sorted(os.listdir(exefs)):
path = os.path.join(exefs, name)
if os.path.isfile(path) and not name.endswith(".part"):
with open(path, "rb") as f:
magic = f.read(4)
if magic == b"NSO0":
out[name] = Module(path)
return out
# ---- applying
def read_patch(rel, data):
"""(the build id the file names, its writes). Refused when the file cannot be read as what its name says it is"""
if rel.lower().endswith(".ips"):
named = os.path.splitext(os.path.basename(rel))[0]
if not re.fullmatch(r"[0-9A-Fa-f]{1,64}", named):
raise Refused("its name is not a build id (an .ips patch is named after the build id of the code it is for)")
return named, parse_ips(data)
build, writes = parse_pchtxt(data.decode("utf-8-sig", errors="replace"))
if build is None:
raise Refused("it has no @nsobid- line (the build id of the code it is for)")
if not re.fullmatch(r"[0-9A-Fa-f]{1,64}", build):
raise Refused("its @nsobid- line is not a build id: %r" % build[:70])
return build, writes
def check_writes(module, writes):
"""Refused when a write is not whole inside one of the module's segments"""
for offset, data in writes:
if module.segment_of(offset, len(data)) is None:
end = max(memory + size for _, memory, size in module.segments)
where = "past the end of its image (%#x)" % end if offset + len(data) > end else \
"outside its .text, .rodata and .data (the empty space between them: a code cave is not supported)"
raise Refused("it writes %d bytes at %s+%#x, %s" % (len(data), module.name, offset, where))
def apply(exefs, mods_root, say=print):
"""The mods' code patches applied to the modules in exefs, in place. Returns the record's entries: a dict per patch
file, in the order they were taken: state ("applied" or "refused"), path (under the mods folder), fnv, size, and
module and changed (the bytes it changed that no later patch wrote over) or why. Says a line per file"""
files = patch_files(mods_root)
if not files:
return []
modules = load_modules(exefs)
game = ", ".join("%s %s" % (m.name, m.build_id.rstrip("0")[:40] or "0") for m in modules.values())
entries, owner = [], {} # (owner: {(module, offset): the entry whose byte is there})
for rel, path in files:
entry = {"state": "refused", "path": rel, "fnv": "0" * 16, "size": 0, "module": "", "changed": 0, "why": ""}
entries.append(entry)
try:
try:
with open(path, "rb") as f:
data = f.read()
except OSError as e:
raise Refused("it cannot be read (%s)" % (e.strerror or e)) from None
entry["fnv"], entry["size"] = fnv64(data), len(data)
named, writes = read_patch(rel, data)
module = next((m for m in modules.values() if same_build(named, m.build_id)), None)
if module is None:
raise Refused("it is for build %s, which is not this game's code (%s)" % (named.lower()[:40], game))
try:
module.load()
except (OSError, ValueError) as e:
raise Refused("the module it is for cannot be read (%s)" % e) from None
check_writes(module, writes)
except Refused as e:
entry["why"] = str(e)
say("code mods: %s is not applied: %s" % (rel, e))
continue
entry["state"], entry["module"] = "applied", module.name
over = {}
for offset, data in writes:
for i, b in enumerate(data):
at = (module.name, offset + i)
before = owner.get(at)
if before is not None and before is not entry and module.image[offset + i] != b:
over[before["path"]] = over.get(before["path"], 0) + 1
module.image[offset + i] = b
owner[at] = entry
for earlier, count in over.items():
say("code mods: %s writes over %d bytes of %s (it comes later by name: its bytes are the ones built in)"
% (rel, count, earlier))
for (name, offset), entry in owner.items(): # (what each changed in the end: a byte is the last patch's that wrote it)
if modules[name].image[offset] != modules[name].original[offset]:
entry["changed"] += 1
for entry in entries:
if entry["state"] == "applied":
say("code mods: %s applied to %s: %d bytes changed%s" % (
entry["path"], entry["module"], entry["changed"],
"" if entry["changed"] else " (it writes what the game has there already, or a later patch wrote over it)"))
written = []
for module in modules.values():
if module.image is not None and bytes(module.image) != module.original:
module.write()
written.append(module.name)
if written:
say("code mods: the game is translated with %s patched. If the translation stops or the game misbehaves, switch the "
"mod off (a - before its folder's name) and build again." % ", ".join(written))
return entries
# ---- what the game folder remembers
def record_text(entries):
lines = []
for e in entries:
tail = (e["module"], str(e["changed"])) if e["state"] == "applied" else (e["why"].replace("\t", " ").replace("\n", " "),)
path = e["path"].replace("\t", "?").replace("\n", "?").replace("\r", "?") # (a line a path, whatever the name)
lines.append("\t".join((e["state"], e["fnv"], str(e["size"]), path) + tail))
return RECORD_HEAD + "".join(line + "\n" for line in lines)
def read_record(out):
"""The entries of the game folder's record, [] when it has none (a folder built before code mods, or with none)"""
entries = []
try:
with open(os.path.join(out, RECORD), encoding="utf-8", errors="replace") as f:
for line in f:
f_ = line.rstrip("\r\n").split("\t")
if line.startswith("#") or len(f_) < 5 or f_[0] not in ("applied", "refused"):
continue
entry = {"state": f_[0], "fnv": f_[1], "path": f_[3], "module": "", "changed": 0, "why": ""}
try:
entry["size"] = int(f_[2])
if f_[0] == "applied":
entry["module"], entry["changed"] = f_[4], int(f_[5]) if len(f_) > 5 else 0
else:
entry["why"] = f_[4]
except ValueError:
continue
entries.append(entry)
except OSError:
pass
return entries
def write_record(out, entries):
"""The record of a finished build, written whole beside it and put in its place; with no patch files there is no
record (a game folder without code mods is as it was before them)"""
path = os.path.join(out, RECORD)
if not entries:
try:
os.remove(path)
except OSError:
pass
return
with open(path + ".new", "w", encoding="utf-8", newline="\n") as f:
f.write(record_text(entries))
os.replace(path + ".new", path)
def since_last(before, now):
"""Lines about what differs from the build before, for a build that takes the folder's patches again (an update):
the patches of that build that are gone, changed or no longer applied (a new one has its own line already)"""
lines = []
was = {e["path"]: e for e in before if e["state"] == "applied"}
have = {e["path"]: e for e in now}
for path, old in was.items():
new = have.get(path)
if new is None:
lines.append("code mods: %s was built into this game folder before and is gone from its mods folder (or "
"switched off): this build is made without it" % path)
elif new["state"] != "applied":
lines.append("code mods: %s was built into this game folder before and is refused now: this build is made "
"without it" % path)
elif (new["fnv"], new["size"]) != (old["fnv"], old["size"]):
lines.append("code mods: %s changed since the build before: this build has it as it is now" % path)
return lines
def mods_folder(out):
"""The mods folder a build reads: the game folder's, or HOS_MODS=<folder> as the game itself takes it"""
return os.environ.get("HOS_MODS") or os.path.join(out, "mods")
def build_step(exefs, out, say=print):
"""nx2/make_game.py, once its copy of the game's code is there and is the supported build: the patches of the game
folder's mods applied to that copy, with a line for each and for what changed since the folder's last build. Returns
the entries for write_record, which make_game.py calls when the build is done"""
before = read_record(out)
entries = apply(exefs, mods_folder(out), say)
for line in since_last(before, entries):
say(line)
applied = [e for e in entries if e["state"] == "applied"]
if entries:
say("code mods: %d of %d patch files applied (%s)" % (len(applied), len(entries), mods_folder(out)))
return entries

308
nx2/emit_pool.py Normal file
View File

@@ -0,0 +1,308 @@
"""(Triforce) The lift's emit in several processes where there is no fork (Windows).
nx2/chunks.py's forked workers inherit the lifter as it stands: 5 GB of decoded functions, tables and texts, shared for nothing.
A spawned process starts empty, and handing it that state whole costs more than it saves (425 MB of functions and 180 MB of texts
pickled, read again and held by every worker). So the lifter stays in the lift's own process, and each worker is sent what it
needs and no more:
- once, when it starts: every attribute of the lifter but NOT_SENT (the entries, the PLT stubs, the jump tables, the names, the
patches after instructions: 4 MB pickled), the functions that set the lifter's class up as this process did (poll.install),
and where the counters live;
- with each run of consecutive functions: those functions' bodies as the discovery left them and the texts of their
instructions as write() translated them (about 3 MB and 3.5 MB a run: 148 runs of 2000 functions were 425 and 513 MB).
A worker emits its run with the lifter's own emit_function and sends back the texts, the names each calls and what the run added
to the counters. The lift's process takes the runs in order and hands write() the functions one by one, as if it had made them:
the C is the one-process C, byte for byte (tests/test_spawn_emit.py on authored code; the game's own lift was compared file by
file). Nothing a worker computes is taken from anywhere but these two messages, so discovery is done once and is not assumed to
repeat itself.
Measured on a 16-thread PC (the game: 294,523 functions, 17.6 million instructions, 185 runs): write()'s emit loop takes 46 s in
one process, 37 s of it in emit_function, and 13 s with 8 processes (21 s with 3, on a PC that was not idle). What is left is
the lift's own process: about 7 s to send (3 s to pickle the bodies, 3 s to pick each run's texts, 1 s to pickle them) and what
write() itself needs for a gigabyte of C. More processes than keep up with it only wait for it, so emit_jobs (chunks.py) asks
for half the processors, at most 8, and the files compiled beside the lift keep the rest. A worker held 230 to 480 MB at its
most.
A worker that raises, dies or does not answer ends the lift with what it said (SystemExit, never a wait forever), and the
workers still running are ended. A worker whose parent is gone ends by itself: Windows leaves the children of a stopped process
running, and one of them could be writing a lifted file into the next build's folder.
"""
import collections
import concurrent.futures
import gc
import importlib
import multiprocessing
import os
import pickle
import sys
import threading
import traceback
from concurrent.futures.process import BrokenProcessPool
import a64_lift
# (the lifter's attributes a worker is not sent: the image and the functions (each run brings its own), write()'s stand-in for
# emit_function, which exists only in the lift's process, and what only the discovery reads)
NOT_SENT = ("img", "funcs", "emit_function", "eh_starts", "_preds")
RUN_FUNCS = 2000 # (a run: at most this many functions,
RUN_INSNS = 120000 # and no more once it holds this many instructions: 2000 functions are 120,000 on average, a few are 20,000 each)
AHEAD = 2 # (runs asked for beyond one a worker: none waits while the lift's process writes a chunk)
WAIT = 600 # (seconds without an answer to a run, a split or a file's marks, which take a second: the worker is taken for hung)
SLICE = 5.0
MOST = 60 # (Windows waits on at most 64 handles at once, and the pool keeps a few of its own)
# (a process that is gone: what it printed as it went, a Python error at its start for one, is in the lift's output above this)
GONE = "%s: a process of the lift stopped before it answered (out of memory, ended from outside, or it could not start): %s"
def plan(order, size, most_funcs=None, most_size=None):
"""order cut into runs of consecutive functions: a run ends at most_funcs functions, or with the function that brings it to
most_size (size(e): a function's instructions), so a run of giant functions is no longer than a run of small ones. Every
function is in exactly one run and the runs, joined, are order"""
most_funcs, most_size = most_funcs or RUN_FUNCS, most_size or RUN_INSNS
runs, cur, held = [], [], 0
for e in order:
cur.append(e)
held += size(e)
if len(cur) >= most_funcs or held >= most_size:
runs.append(cur)
cur, held = [], 0
if cur:
runs.append(cur)
return runs
def texts_of(funcs, es, texts):
"""The instruction texts the emit of the functions es reads, out of all of them: a64_lift's emit_function, dead_flags and
dirty_regs look a plain instruction's text up by its word, or by the word and its address where the text depends on the
address (pc_relative), and no other kind's"""
relative = a64_lift.pc_relative
need = {}
for e in es:
for pc, (k, w) in funcs[e][0].items():
if k[0] == "op":
key = (w, pc) if relative(w) else w
if key not in need:
need[key] = texts[key]
return need
def emit_print(cls):
"""What the class's emit_function is made of: its own name and, through the closures, the names of the functions it wraps
(poll.install keeps the one before it in its closure). A worker whose class has another emit_function than the lift's
process (a wrapper installed there and not named in `setup`) would write other C, without a word: it compares this first"""
out, todo, seen = [], [getattr(cls, "emit_function", None)], set()
while todo:
f = todo.pop()
f = getattr(f, "__func__", f)
if f is None or id(f) in seen:
continue
seen.add(id(f))
module = getattr(f, "__module__", None) or "?"
out.append("%s.%s" % ("__main__" if module == "__mp_main__" else module, getattr(f, "__qualname__", "?")))
for cell in getattr(f, "__closure__", None) or ():
try:
v = cell.cell_contents
except ValueError:
continue
if hasattr(v, "__code__") or hasattr(v, "__func__"):
todo.append(v)
return out
def where(L, counter):
"""How a worker finds a counter of the lift's process again: an attribute of the lifter (stats), or a module's global
(poll.stats). ("L", name) or (module, name)"""
for k, v in vars(L).items():
if v is counter:
return "L", k
for name, module in list(sys.modules.items()):
for k, v in list((getattr(module, "__dict__", None) or {}).items()):
if v is counter:
return name, k
raise SystemExit("emit: a counter that neither the lifter nor a module holds by name cannot be counted in other processes")
# ---- in a worker ------------------------------------------------------------------------------------------------------------
_mine = None # (a worker's own: its lifter and its counters, or what kept it from having them, as text)
def _no_throttle():
"""Windows' power throttling off for this process, as the build does for every process it starts: it slows the processes
of a window in the background to almost nothing on some PCs"""
if os.name != "nt":
return
try:
import ctypes
class State(ctypes.Structure):
_fields_ = [("Version", ctypes.c_ulong), ("ControlMask", ctypes.c_ulong), ("StateMask", ctypes.c_ulong)]
k = ctypes.WinDLL("kernel32")
k.GetCurrentProcess.restype = ctypes.c_void_p
k.SetProcessInformation.argtypes = [ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_ulong]
s = State(1, 1, 0)
k.SetProcessInformation(k.GetCurrentProcess(), 4, ctypes.byref(s), ctypes.sizeof(s))
except (OSError, ValueError, TypeError, AttributeError):
pass
def _leave_with_parent():
"""This process ends when the lift's does (stopped from outside: nothing tells its children)"""
parent = multiprocessing.parent_process()
if parent is None:
return
def watch():
parent.join()
os._exit(3)
threading.Thread(target=watch, daemon=True).start()
def _start(blob):
global _mine
try:
_no_throttle()
_leave_with_parent()
setup, cls, state, counters, want = pickle.loads(blob)
for f in setup:
f()
got = emit_print(cls)
if got != want:
raise RuntimeError("the lifter's emit_function here is %s, the lift's process has %s: what set it up there must be "
"named in `setup` (nx2/chunks.py write)" % (" over ".join(got), " over ".join(want)))
L = cls.__new__(cls)
vars(L).update(state)
_mine = (L, [vars(L)[k] if m == "L" else getattr(importlib.import_module(m), k) for m, k in counters])
except BaseException: # (said by the first run it is asked for: an initializer's own failure only breaks the pool)
_mine = traceback.format_exc()
def _emit_run(bodies, texts):
"""A run's functions emitted: [(e, text, refs)], and each counter's keys with what this run added (every key the counter
holds, in its order: the lift's process adds them in the runs' order, so its counters get their keys in the order one
process gives them)"""
if not isinstance(_mine, tuple):
raise RuntimeError("a process of the emit could not be set up:\n%s" % _mine)
L, counters = _mine
L.funcs = bodies
for c in counters:
for k in c:
c[k] = 0
out = [(e,) + tuple(L.emit_function(e, texts)) for e in bodies]
return out, [list(c.items()) for c in counters]
# ---- in the lift's process ----------------------------------------------------------------------------------------------------
class Workers:
"""The spawned processes. jobs: how many; L, counters, setup: what each is given when it starts (the lifter's state, where
its counters and the others in `counters` live, and the functions to call first)"""
def __init__(self, jobs, L, counters=(), setup=()):
blob = pickle.dumps((list(setup), type(L), {k: v for k, v in vars(L).items() if k not in NOT_SENT},
[where(L, c) for c in counters], emit_print(type(L))), pickle.HIGHEST_PROTOCOL)
self.jobs = max(1, min(int(jobs), MOST))
self.pool = concurrent.futures.ProcessPoolExecutor(max_workers=self.jobs, mp_context=multiprocessing.get_context("spawn"),
initializer=_start, initargs=(blob,))
def _ask(self, fn, call):
try:
return call[0], self.pool.submit(fn, *call[1])
except BrokenProcessPool as e: # (a process died since the last answer: the pool takes nothing more)
raise SystemExit(GONE % (call[0], e)) from None
def each(self, fn, calls, wait=None):
"""fn(*args) in the processes for every (what, args) of calls, the results in the calls' order. calls is read as the
processes come free (AHEAD more than there are), so what it makes for a call is not held for all of them at once.
A call that raises, a process that dies and one that gives no answer for `wait` seconds (WAIT) end it with SystemExit
and what was said, and with the processes ended; so does leaving it before its end"""
pending, calls, done = collections.deque(), iter(calls), False
try:
while True:
while calls is not None and len(pending) < self.jobs + AHEAD:
call = next(calls, None)
if call is None:
calls = None
else:
pending.append(self._ask(fn, call))
if not pending:
done = True
return
answer = self._answer(*pending.popleft(), wait or WAIT)
done = calls is None and not pending # (the last: whoever takes it and asks for no more leaves the processes be)
yield answer
finally:
if not done:
self.stop(kill=True)
def ask(self, fn, calls):
"""Every (what, args) of calls asked of the processes now: what answers() then takes. For work that must go to its
end once it has begun (the page marks: a file each, rewritten in place): nothing is left to ask for later, so a lift
that ends meanwhile for a reason of its own still has every file finished, the pool doing what it was asked before
the interpreter leaves"""
try:
return [self._ask(fn, call) for call in calls]
except BaseException:
self.stop(kill=True)
raise
def answers(self, asked, wait=None):
"""The answers to ask()'s calls, in their order; it fails as each() does"""
left = len(asked)
try:
for what, future in asked:
answer = self._answer(what, future, wait or WAIT)
left -= 1
yield answer
finally:
if left:
self.stop(kill=True)
def _answer(self, what, future, wait):
left = wait
while not concurrent.futures.wait([future], timeout=min(left, SLICE))[0]:
left -= SLICE # (in slices: a PC that slept through the wait is not a worker that hung)
if left <= 0:
raise SystemExit("%s: its process gave no answer for %d seconds and was stopped" % (what, wait))
try:
return future.result()
except KeyboardInterrupt:
raise
except BrokenProcessPool as e:
raise SystemExit(GONE % (what, e)) from None
except BaseException as e: # (what the process raised, SystemExit too, with where it raised it; the summary last, the line a build names)
raise SystemExit("%s%s: its process failed: %s: %s" % (
"%s\n" % str(e.__cause__).strip() if e.__cause__ is not None else "", what, type(e).__name__, e)) from None
def stop(self, kill=False):
"""The processes ended: asked to leave after what they are doing, or (kill) ended now"""
pool, self.pool = self.pool, None
if pool is None:
return
if kill:
for p in list((getattr(pool, "_processes", None) or {}).values()):
try:
p.kill()
except (OSError, ValueError, AttributeError):
pass
pool.shutdown(wait=True, cancel_futures=True)
def emit(workers, L, order, texts, counters, need=texts_of, runs=None):
"""(e, text, refs) for every function of order, in order, emitted by the workers; counters (the dicts Workers was given)
get each run's additions. need(funcs, es, texts): the texts a run is sent; runs: the runs, when not plan()'s"""
funcs = L.funcs
if runs is None:
runs = plan(order, lambda e: len(funcs[e][0]))
def calls():
for es in runs:
yield ("the emit of f_%08x to f_%08x" % (es[0], es[-1]), ({e: funcs[e] for e in es}, need(funcs, es, texts)))
gc.freeze() # (the lift's millions of objects out of the collector's sight while the runs' short-lived ones come and go)
try:
for out, counts in workers.each(_emit_run, calls()):
for c, added in zip(counters, counts):
for k, v in added:
c[k] = c.get(k, 0) + v
yield from out
finally:
gc.unfreeze()

View File

@@ -181,11 +181,14 @@ def main():
if not a.no_polls:
poll.install()
gen = os.path.join(out_dir, "gen")
# ((Triforce) setup: what the emit's spawned processes do to the lifter's class before they emit, as this one did above)
files = chunks.write(L, gen, os.path.join(a.work, "image"), a.rechunk and not a.standalone,
counters=[] if a.no_polls else [poll.stats])
counters=[] if a.no_polls else [poll.stats], setup=[] if a.no_polls else [poll.install])
import oot_patches
main_build = next((m["build_id"] for m in meta["modules"] if m["name"] == "main"), "")
chunks.edit_c(gen, L.stats, a.standalone and main_build == oot_patches.LIFE_BUILD_ID, whole=not a.max_func)
# ((Triforce) wait=False: where the emit's processes were spawned they share the page marks while the files below are
# written, which read none of the C; marks_done() waits for them before the report says their counts)
chunks.edit_c(gen, L.stats, a.standalone and main_build == oot_patches.LIFE_BUILD_ID, whole=not a.max_func, wait=False)
a64_rtgen.write(gen)
if a.standalone:
start, exit_ = standalone.write_image_c(gen, img, meta, a.exefs)
@@ -194,6 +197,7 @@ def main():
nvn = standalone.nvn_exports(nso.load_exefs(a.exefs))
standalone.write_nvn_exports(gen, nvn)
print("standalone: nvn2_exports.c (%d sdk NVN functions routed to host/nvn2.c)" % len(nvn))
chunks.marks_done()
L.report()
print("interrupt polls: %d loop headers in %d functions" % (poll.stats["polls"], poll.stats["functions"]))
print("wrote %d code files to %s (%.0fs)" % (len(files), gen, time.time() - t0))

View File

@@ -25,6 +25,7 @@ and one plain line the builders also read (older ones only show it): "the shader
build's last step, the shaders' preparation, was skipped or failed and is left to the game's first start (NOT_PREPARED)
"""
import argparse
import atexit
import contextlib
import filecmp
import hashlib
@@ -49,6 +50,7 @@ STARTED = time.time() # (the build's total time, kept in build_times.json and
sys.path[:0] = [HERE, os.path.join(ROOT, "third_party", "a64-kit", "tools")] # (python -I leaves the script folder off)
import toolchain as _toolchain
import disk as _disk
import code_mods as _code_mods # (Triforce: the mods' patches of the game's code, applied before the lift)
# (hookshot) RX_TARGET=windows on Linux: the Windows game, with llvm-mingw's Linux release (HOOKSHOT_MINGW, or llvm-mingw
# beside the builder), to run under Proton or copy to a Windows PC
LINUX, CROSS = not _toolchain.windows(), _toolchain.windows() and os.name != "nt"
@@ -97,7 +99,23 @@ def say_prepare(words):
print("progress 1.000 Done", flush=True)
STAGE_TIMES = [] # ((stage, when it began)): what each step of this build took, said at its end (stage_times)
def stage_times(now):
""" "before the game files 0:41, read 1:32, lift 2:10, compile 7:42, finish 1:17": each stage's time up to the next
one's start, the last one's up to now, and what came before the first (the tools' checks and what was fetched)"""
if not STAGE_TIMES:
return ""
parts = [("before the game files", STAGE_TIMES[0][1] - STARTED)]
for (name, began), (_, ended) in zip(STAGE_TIMES, STAGE_TIMES[1:] + [("", now)]):
parts.append((name, ended - began))
return ", ".join("%s %s" % (name, clock(seconds)) for name, seconds in parts)
def say(*words):
if words and words[0] == "stage" and len(words) > 1:
STAGE_TIMES.append((str(words[1]), time.time()))
if PREPARE:
return say_prepare([str(w) for w in words])
print("@@" + " ".join(str(w) for w in words), flush=True)
@@ -357,10 +375,36 @@ def file_progress(verb, clock=time.monotonic):
return progress
def stand_in_damaged_files(out, romfs):
def unpack_seen(game):
"""What the unpack itself saw of the romfs .zs files it wrote (nx2/nup.py ZS_CHECKED, in the folder it extracted to):
{"files": how many, "damaged": [paths under romfs]}, or None when it left no such note or one that cannot be right
(then every file is opened and looked at, as before)"""
import json
import nup
try:
with open(os.path.join(game, nup.ZS_CHECKED), encoding="utf-8") as f:
seen = json.load(f)
except (OSError, ValueError):
return None
if not isinstance(seen, dict) or not isinstance(seen.get("files"), int) or not isinstance(seen.get("damaged"), list):
return None
for rel in seen["damaged"]:
if not isinstance(rel, str):
return None
parts = rel.split("/")
if not rel.endswith(".zs") or "\\" in rel or ":" in rel or any(p in ("", ".", "..") for p in parts):
return None
return seen
def stand_in_damaged_files(out, romfs, seen=None):
"""An incomplete dump's damaged textures replaced by whole ones of the same kind, and said: without them the game
leaves out the walls and floors that use them (the Water Temple, #1002). Only in a romfs this builder unpacked (its
.triforce_romfs stamp), and once per unpack: .triforce_romfs_checked holds the stamp it was checked under"""
.triforce_romfs stamp), and once per unpack: .triforce_romfs_checked holds the stamp it was checked under.
seen: what this unpack saw of its own .zs files as it wrote them (unpack_seen): the damaged ones are then known and
no file is opened to find them. Opening all 102,185 took 292 s of an 18 minute build on a PC whose virus scanner
looks at each file opened; a folder unpacked by an earlier builder, or read from this .nup before without the
check's note, is still gone through file by file"""
good = os.path.join(out, ".triforce_romfs")
checked = os.path.join(out, ".triforce_romfs_checked")
if not os.path.isfile(good) or not os.path.isdir(romfs):
@@ -373,9 +417,14 @@ def stand_in_damaged_files(out, romfs):
if f.read() == stamp:
return
import nup
print("checking your game files for damaged textures (once after an unpack): every .zs file is opened, which can take "
"minutes", flush=True)
done = nup.stand_in_damaged(romfs, file_progress("checked"))
if seen is not None:
print("your game files were checked for damaged textures as they were unpacked: %s .zs files, %d damaged"
% (format(seen["files"], ","), len(seen["damaged"])), flush=True)
done = nup.stand_in_damaged(romfs, known=seen["damaged"])
else:
print("checking your game files for damaged textures (once after an unpack): every .zs file is opened, which can take "
"minutes", flush=True)
done = nup.stand_in_damaged(romfs, file_progress("checked"))
with open(checked, "w") as f:
f.write(stamp)
if not done:
@@ -975,6 +1024,22 @@ def ship_state_list(out):
print("the shipped pipeline state list: %d pipelines" % ((os.path.getsize(src) - 12) // 528), flush=True)
def ship_triforce_assets(out):
"""(Triforce, v0.5.0) Triforce's own files the game reads beside it (assets/triforce_assets: the Mods screen's mask, CC BY 4.0,
its credits in CREDITS.txt), as <out>/triforce_assets. None is a game file. Without them the game runs as before: the Mask
Salesman stands without the mask (gpu/vk/vk_salesman.cpp)"""
src = os.path.join(ROOT, "assets", "triforce_assets")
if not os.path.isdir(src):
return
dst = os.path.join(out, "triforce_assets")
os.makedirs(dst, exist_ok=True)
for name in sorted(os.listdir(src)):
s, d = os.path.join(src, name), os.path.join(dst, name)
if os.path.isfile(s) and not (os.path.isfile(d) and filecmp.cmp(s, d, shallow=False)):
copy_writable(s, d)
print("Triforce's own files beside the game: %s" % ", ".join(sorted(os.listdir(src))), flush=True)
# (the builders read this line's start: the step was skipped or failed, and the shaders' row says so instead of a check mark)
NOT_PREPARED = "the shaders were not prepared now"
# (a run that prints nothing this long is stopped. Not its whole length: that is 25 minutes on some AMD GPUs, and the driver's own
@@ -1140,11 +1205,189 @@ def shader_census_cmd(romfs, out):
LIFT_KEY = ".lift_key" # (in <work>/sa/gen: what the lifted C there was made from)
def compile_alone_failed(lines):
"""Whether a build that failed in nx2/build.py failed in its compile and never came to a link: build.py's own lines say
so ("compiled in 412s, 2 failed", and no "link FAILED" after it). The objects that compile did make are then whole,
each stamped and cached under the hash of its source and flags only once its compiler ended well, and are worth
keeping: a compiler that ran out of memory, or a disk that filled, used to cost the next build every one of the ~600
files again (make_game deleted them all), an hour and more on the PCs where a compile fails in the first place. A
build that got as far as its link and failed there, or that ended without saying how its compile went, is not
trusted: its objects go, as before (a stale object linked "compiled in 0s" at every try, issue 1197)"""
failed = None
for line in lines:
m = re.match(r"compiled in \d+s, (\d+) failed", line)
if m:
failed = int(m.group(1))
elif "link FAILED" in line:
return False
return bool(failed)
PROMPT_KEY = ".made_from" # (in <out>/prompts: prompts_key of what that folder was made from)
def prompts_key(out):
"""What the button prompts of other controllers are made from, as one hash: the tools that draw them (nx2/prompt_*.py,
the faces' drawings and parts in nx2/prompt_faces, the icon fonts in third_party/kenney_input), the packages they are
drawn with (nx2/prompt_requirements.txt) and the game's files, by the unpack's stamp (.triforce_romfs: the .nup's name
and size). None when the romfs has no such stamp (the player's own folders, linked in): its prompts are made at every
build, as before. (Triforce) The keyboard and mouse face's too: nx2/prompt_keys.py, which draws it, and the font its
labels are set in (third_party/fonts/libertinus: that one file, the menus' other fonts are not the prompts')"""
stamp = os.path.join(out, ".triforce_romfs")
if not os.path.isfile(stamp):
return None
# (each file under a name that does not depend on where the tools are installed)
files = [("nx2/" + n, os.path.join(HERE, n))
for n in ("prompt_assets.py", "prompt_draw.py", "prompt_faces.py", "prompt_requirements.txt", "prompt_keys.py")]
labels = "third_party/fonts/libertinus/LibertinusSans-Bold.ttf"
files.append((labels, os.path.join(ROOT, *labels.split("/"))))
for name, root in (("nx2/prompt_faces", os.path.join(HERE, "prompt_faces")),
("third_party/kenney_input", os.path.join(ROOT, "third_party", "kenney_input"))):
for folder, dirs, names in os.walk(root):
dirs.sort()
files += [(name + "/" + os.path.relpath(os.path.join(folder, n), root).replace(os.sep, "/"), os.path.join(folder, n))
for n in sorted(names)]
h = hashlib.sha1()
with open(stamp, "rb") as f:
h.update(f.read())
for name, path in files:
if os.path.isfile(path):
h.update(name.encode())
with open(path, "rb") as f:
h.update(f.read())
return h.hexdigest()
class EarlyCompile:
"""(Windows) The files that need nothing of the translated game, compiled while the game is being translated
(nx2/build.py RX_EARLY): libopus and the two renderers with their libraries, ~520 of the build's ~630 files. The lift was
one process there and left every other processor idle for its two minutes, and then the compile had those files to do
on top of the game's. (The lift's emit has processes of its own there now, nx2/emit_pool.py: at most half the
processors, and its first part, the discovery, is still one process.) finish() waits for it before the build proper starts, which then finds their objects made.
A failure here costs nothing: the build compiles whatever is missing, as before. HOOKSHOT_EARLY_COMPILE=0: not done"""
def __init__(self, work, env, jobs):
self.proc = None
if os.name != "nt" or os.environ.get("HOOKSHOT_EARLY_COMPILE") == "0":
return
self.log = os.path.join(work, "early_compile.log")
e = dict(os.environ, PYTHONIOENCODING="utf-8")
e.update(env)
e["RX_EARLY"] = "1"
e.pop("HOOKSHOT_TIMES", None) # (no countdown lines: nobody reads them here)
with open(self.log, "w", encoding="utf-8") as log:
self.proc = subprocess.Popen(PY + ["nx2/build.py", "--standalone", "--jobs", str(max(1, jobs - 1))], cwd=ROOT, env=e,
stdout=log, stderr=subprocess.STDOUT)
no_throttle(getattr(self.proc, "_handle", None))
def stop(self):
if self.proc is not None and self.proc.poll() is None:
self.proc.kill()
self.proc.wait()
def finish(self):
if self.proc is None:
return
waited = time.time()
code = self.proc.wait()
said = ""
try:
with open(self.log, encoding="utf-8", errors="replace") as f:
said = next((line.strip() for line in reversed(f.read().splitlines()) if line.startswith("compiled in ")), "")
except OSError:
pass
print("the files that need no translated code were compiled beside the lift: %s%s (%s more after the lift ended)%s"
% (said or "no result", "" if not code else ", stopped with status %d" % code, clock(time.time() - waited),
"" if not code else ": the build compiles what is missing"), flush=True)
self.proc = None
class Prompts:
"""The button prompts of other controllers (host/prompts.c): the game's icon font and textures redrawn per controller by
nx2/prompt_assets.py, with fontTools. It took 87 s at the end of every build and every update, with nothing else
running. It reads only the game's files, so it now starts as soon as they are read, beside the lift and the compile,
into <out>/prompts.new, and finish() puts the folder in place; and it is not run at all when the folder there was made
from the same tools and the same game files (prompts_key). Without it, or when it fails, the game shows its own icons,
as before"""
def __init__(self, romfs, out, work):
self.out, self.folder, self.new = out, os.path.join(out, "prompts"), os.path.join(out, "prompts.new")
self.key = prompts_key(out)
self.proc = self.log = None
self.state = "no fontTools" if importlib.util.find_spec("fontTools") is None else None
if self.state:
return
try:
with open(os.path.join(self.folder, PROMPT_KEY), encoding="utf-8") as f:
if self.key and f.read().strip() == self.key:
self.state = "kept"
return
except OSError:
pass
shutil.rmtree(self.new, ignore_errors=True)
self.log = os.path.join(work, "prompts.log")
with open(self.log, "w", encoding="utf-8") as log:
self.proc = subprocess.Popen(PY + [os.path.join(HERE, "prompt_assets.py"), "--romfs", romfs, "--out", self.new],
cwd=ROOT, env=dict(os.environ, PYTHONIOENCODING="utf-8"), stdout=log,
stderr=subprocess.STDOUT)
no_throttle(getattr(self.proc, "_handle", None))
def stop(self):
"""(the build failed or was stopped) the drawing ended and what it wrote taken away: the folder in place stays"""
if self.proc is not None and self.proc.poll() is None:
self.proc.kill()
self.proc.wait()
shutil.rmtree(self.new, ignore_errors=True)
def finish(self):
"""Waits for the drawing, says what it said, and puts the new folder in the old one's place"""
if self.state == "no fontTools":
shutil.rmtree(self.folder, ignore_errors=True)
print("the button prompts of other controllers were not made (no fontTools): the game shows its own", flush=True)
return
if self.state == "kept":
print("the button prompts of other controllers are as the last build made them (the same tools and game files): kept",
flush=True)
return
code = self.proc.wait()
try:
with open(self.log, encoding="utf-8", errors="replace") as f:
sys.stderr.write(f.read())
sys.stderr.flush()
except OSError:
pass
shutil.rmtree(self.folder, ignore_errors=True)
if code:
print("the button prompts of other controllers were not made: the game shows its own", flush=True)
shutil.rmtree(self.new, ignore_errors=True)
return
os.replace(self.new, self.folder)
if self.key:
with open(os.path.join(self.folder, PROMPT_KEY), "w", encoding="utf-8") as f:
f.write(self.key + chr(10))
# (Triforce) Files of nx2/ that are not the translator's and that the lift (nx2/lift.py, nx2/split_hot.py and what they
# import) cannot read: this driver, the steps before and after the lift, the version, the button prompts, and the tests
# kept in nx2/. They were part of the lift key, so an update that changed one of them, and every release changes the
# version and most change this file, translated the whole game again on every player's PC: 2:48 of a 3:05 update build
# on a 16-thread PC, for C that came out the same byte for byte. tests/test_lift_key_files.py holds the list to what
# the lift can import and to what its files name.
LIFT_KEY_NOT = ("make_game.py", "build.py", "nup.py", "version.py", "shader_census.py", "prompt_assets.py", "prompt_draw.py",
"prompt_faces.py", "prompt_requirements.txt", "prompt_keys.py",
"code_mods.py") # (the mods' code patches: what it does to the game's code is in the key, the exefs's files)
def lift_key_file(rel):
"""Whether a file of nx2/ (rel: its path under nx2/, with /) is part of the lift key: all but LIFT_KEY_NOT, the
prompt_faces folder and the tests (test_*.py)"""
name = rel.rsplit("/", 1)[-1]
return not (rel in LIFT_KEY_NOT or rel.startswith("prompt_faces/") or (name.startswith("test_") and name.endswith(".py")))
def lift_key(exefs, work, env):
"""What the lift's output depends on: the game's code, the translator (nx2/, the vendored a64-kit), the HLE functions host/ defines
(hooked in the lift; the rest of host/ is compiled after it, so a change there alone does not lift again), the hot list and chunk
boundaries, the lift's environment and the Python running it. A file the lift never reads may be in it (a new build then lifts
again); one it reads must be"""
"""What the lift's output depends on: the game's code, the translator (nx2/ but for LIFT_KEY_NOT, the vendored a64-kit), the HLE
functions host/ defines (hooked in the lift; the rest of host/ is compiled after it, so a change there alone does not lift again),
the hot list and chunk boundaries, the lift's environment and the Python running it. A file the lift never reads may be in it (a
new build then lifts again); one it reads must be"""
import kit # noqa: F401 (a64-kit on the path, for standalone)
import standalone
h = hashlib.sha1(sys.version.encode(), usedforsecurity=False)
@@ -1154,7 +1397,8 @@ def lift_key(exefs, work, env):
for top in (HERE, os.path.join(ROOT, "third_party", "a64-kit")):
for d, dirs, fs in os.walk(top):
dirs[:] = sorted(x for x in dirs if x != "__pycache__")
files += [(ROOT, os.path.join(d, f)) for f in sorted(fs) if f.endswith((".py", ".c", ".h", ".cpp", ".hpp", ".txt", ".json"))]
files += [(ROOT, os.path.join(d, f)) for f in sorted(fs) if f.endswith((".py", ".c", ".h", ".cpp", ".hpp", ".txt", ".json"))
and (top != HERE or lift_key_file(os.path.relpath(os.path.join(d, f), HERE).replace(os.sep, "/")))]
files += [(work, os.path.join(work, "hot_funcs.txt")), (work, os.path.join(work, "image", "chunks.txt"))]
for base, f in files:
if os.path.isfile(f):
@@ -1274,7 +1518,9 @@ def fsr_runtime_dir(work):
def fetch_fsr_runtime(work):
"""(--fsr, Windows) AMD's two signed FSR 4 libraries into fsr_runtime_dir, only what is missing. A download that fails
is said and the build goes on: the game then does not offer AMD FSR 4 until a build gets them"""
is said and the build goes on: the game then does not offer AMD FSR 4 until a build gets them. (Triforce) And its
frame generation library with them (scripts/fsr_sdk.py OPTIONAL): without that one the game has FSR 4 as before and
the Frame generation row no AMD FSR 4"""
fsr = fsr_script()
if LINUX or fsr is None:
return
@@ -1288,7 +1534,8 @@ def fetch_fsr_runtime(work):
def place_fsr_runtime(out, work, wanted):
"""(AMD FSR 4) a game built with it gets AMD's two libraries and their license text beside its exe, where
gpu/vk/vk_fsr4.cpp looks; a folder built with them before and without FSR 4 now loses them. Returns what was placed"""
gpu/vk/vk_fsr4.cpp looks, and (Triforce) AMD's frame generation library when it was fetched; a folder built with them
before and without FSR 4 now loses them, all three. Returns what was placed"""
fsr = fsr_script()
if LINUX or fsr is None:
return []
@@ -1447,8 +1694,8 @@ def parse_args():
p.add_argument("--fetch-ngx-sdk", action="store_true", help="download NVIDIA's DLSS SDK into HOOKSHOT_NGX_SDK before the build")
p.add_argument("--accept-nvidia-license", action="store_true", help="NVIDIA's DLSS SDK license, accepted (with --fetch-ngx-sdk)")
# (AMD FSR 4, off unless asked: nothing is downloaded. gpu/vk/vk_fsr4.cpp)
p.add_argument("--fsr", action="store_true", help="build with AMD FSR 4; on Windows AMD's two libraries are downloaded "
"from AMD's repository and put beside the game (scripts/fsr_sdk.py)")
p.add_argument("--fsr", action="store_true", help="build with AMD FSR 4; on Windows AMD's two libraries, and the one for "
"its frame generation, are downloaded from AMD's repository and put beside the game (scripts/fsr_sdk.py)")
p.add_argument("--no-fsr", action="store_true", help="with --update: build without AMD FSR 4 even if the folder had it")
# (RenoDX's HDR, in every build since 11-13: its shaders and the Slang compiler are downloaded and compiled here.
# scripts/renodx_shaders.py, gpu/vk/vk_renodx.cpp)
@@ -1582,6 +1829,7 @@ def main(a=None):
"or choose the file again in Hookshot Builder." % (a.nup, e))
stamp = "unpacked by this builder: %s %d" % (os.path.basename(a.nup), st.st_size)
same = False
unpacked_seen = None
if os.path.exists(good) and os.path.isdir(romfs) and os.path.isdir(kept):
with open(good) as f:
same = f.read().strip() == stamp
@@ -1610,6 +1858,7 @@ def main(a=None):
raise
if os.path.exists(romfs):
shutil.rmtree(romfs)
unpacked_seen = unpack_seen(game) # (the .zs files as the unpack saw them: stand_in_damaged_files below)
shutil.move(os.path.join(game, "romfs"), romfs)
shutil.move(staged_exefs, exefs)
shutil.rmtree(game, ignore_errors=True)
@@ -1619,7 +1868,7 @@ def main(a=None):
f.write(stamp + chr(10))
check_modules(exefs)
# (also when the .nup was read before: a folder built from a 23.3 GB dump by an earlier builder gets them too)
stand_in_damaged_files(out, romfs)
stand_in_damaged_files(out, romfs, unpacked_seen)
else:
check_cache_paths(out, (kept, romfs))
refuse_dump_links(a.exefs)
@@ -1630,6 +1879,10 @@ def main(a=None):
shutil.rmtree(kept, ignore_errors=True)
shutil.copytree(exefs, kept)
check_supported_build(exefs)
# (Triforce) code mods: the .ips and .pchtxt patches of the game folder's mods, applied to this copy of the game's code, which
# the lift reads and whose files its key hashes: never to the player's files nor to the copy kept for updates (kept, above).
# A patch added, taken away, switched off or edited so changes the key and the game is translated again (nx2/code_mods.py)
code_patches = _code_mods.build_step(exefs, out, say=lambda line: print(line, flush=True))
# (made once on the development PC: lists of addresses and counts, no game code)
for name in ("hot_funcs.txt", "hot_order.txt", "hookshot_pieces.prof", "hookshot.res"):
@@ -1648,6 +1901,9 @@ def main(a=None):
if os.path.exists(os.path.join(work, "hookshot_pieces.prof")) and profile_readable(os.path.join(work, "hookshot_pieces.prof")):
env["RX_PGO"] = os.path.join(work, "hookshot_pieces.prof")
prompts = Prompts(romfs, out, work) # (drawn beside the lift and the compile: finish() below)
atexit.register(prompts.stop) # (a build that stops for any reason takes its drawing with it)
say("stage", "lift", "Translating the game to PC code")
# (the same game, tools and settings as the lift before: its C is what this lift would write, and nothing recompiles from it. A
# rebuild after a change to host/ alone skips the minutes of the lift)
@@ -1676,9 +1932,12 @@ def main(a=None):
except OSError as e:
fail("Could not remove earlier translated file %s: %s. The new translation has not started." %
(os.path.join(gen_dir, f), e))
early = EarlyCompile(work, env, a.jobs) # (what needs no lifted code is compiled meanwhile)
atexit.register(early.stop)
run(PY + ["nx2/lift.py", "--standalone", "--write", "--exefs", exefs, "--work", work], env)
if os.path.isfile(os.path.join(work, "sa", "gen", "code_hot.c")):
run(PY + ["nx2/split_hot.py", os.path.join(work, "sa", "gen")], env) # (eight files side by side: ~6 minutes, not 79)
early.finish()
write_lift_key(work, key)
cut = lifted_cut_short(gen_dir)
if not cut:
@@ -1714,7 +1973,11 @@ def main(a=None):
mb = 0
return 23.5 * mb if name.startswith("code_hot") else 3.4 * mb + 1 if name.startswith("code_") else 1.3
said = [] # (build.py's lines about how its compile and its links went: compile_alone_failed)
def compiled(line):
if line.startswith("compiled in ") or "link FAILED" in line:
said.append(line)
m = re.match(r"(\d+) files, ", line)
if m:
try:
@@ -1751,9 +2014,15 @@ def main(a=None):
os.remove(key_file)
# (and its compiled objects go too: a cached runtime object from another version linked with every object "compiled in 0s"
# and a64_dirty, a64_icall, a64_lookup, recomp_cntpct undefined, at every try (issue 1197, Linux, v2.0.10))
for sub in (os.path.join(work, "objcache"), os.path.join(work, "sa", "build", "obj")):
shutil.rmtree(sub, ignore_errors=True)
print("the build failed: the next one translates and compiles the game again (nothing of this one is reused)", flush=True)
# (Triforce: but not when the compile alone failed, before any link: compile_alone_failed)
if compile_alone_failed(said):
print("the build failed in its compile: the files that did compile are kept, and the next build translates the game "
"again and compiles only the rest", flush=True)
else:
for sub in (os.path.join(work, "objcache"), os.path.join(work, "sa", "build", "obj")):
shutil.rmtree(sub, ignore_errors=True)
print("the build failed: the next one translates and compiles the game again (nothing of this one is reused)", flush=True)
prompts.stop()
raise
compile_seconds = time.time() - compile_started
@@ -1816,19 +2085,14 @@ def main(a=None):
if run(shader_census_cmd(romfs, os.path.join(out, "triforce_shaders.bin")), check=False, output=sys.stderr):
print("the shader list was not made: the game will compile its shaders as it plays", flush=True)
# (hookshot) the button prompts of other controllers (host/prompts.c): the game's icon font redrawn per controller, with fontTools.
# Without it the game shows its own icons
prompts = os.path.join(out, "prompts")
shutil.rmtree(prompts, ignore_errors=True)
if importlib.util.find_spec("fontTools") is None:
print("the button prompts of other controllers were not made (no fontTools): the game shows its own", flush=True)
elif run(PY + [os.path.join(HERE, "prompt_assets.py"), "--romfs", romfs, "--out", prompts], check=False, output=sys.stderr):
print("the button prompts of other controllers were not made: the game shows its own", flush=True)
shutil.rmtree(prompts, ignore_errors=True)
# Without it the game shows its own icons. (Triforce: drawn since the game files were read, or kept: class Prompts)
prompts.finish()
icon = os.path.join(ROOT, "assets", "triforce.png")
if LINUX and os.path.exists(icon): # (the window's icon: host/posix/display_sdl.c)
copy_writable(icon, os.path.join(out, "triforce.png")) # (a second build overwrites the first's)
seed_pipelines(out)
ship_state_list(out)
ship_triforce_assets(out)
language = os.path.join(out, "language.txt") # (the same text as host/nn_applet.c writes)
with open(language, "w", encoding="utf-8", newline="\n") as f:
f.write("# The game's language. Change the line below to one of the codes listed here, save, and start the game again.\n\n%s\n\n" % a.language)
@@ -1850,6 +2114,8 @@ def main(a=None):
# (the Python builder: builder is its pythonw.exe, and these the arguments before --update, host/hos_update.c)
if os.environ.get("HOOKSHOT_BUILDER_ARGS"):
f.write("builder_args=%s\n" % os.environ["HOOKSHOT_BUILDER_ARGS"])
# (Triforce) what code patches this game is built with, for the next build and for the game's start (host/mods.c)
_code_mods.write_record(out, code_patches)
# (the game's code as read: written again by every build, so not left behind in the work folder. The translated code (sa\gen)
# and the compiled objects stay: the next build skips the lift when nothing it reads changed (#428) and compiles only what
# changed; the builder says how much stays and lets it go. Kept in a development checkout)
@@ -1872,6 +2138,8 @@ def main(a=None):
print("the build took %s, compiled in %s%s" % (clock(total_seconds), clock(compile_seconds),
"" if previous is None else " (the last one %s)" % clock(previous)), flush=True)
say("times", "%.0f" % total_seconds, "%.0f" % compile_seconds, "-1" if previous is None else "%.0f" % previous)
# (where the time went, for a report of a slow build and for measuring a change to the build itself)
print("the build's steps: %s" % stage_times(time.time()), flush=True)
# (after the times: they stay comparable from build to build, without a GPU's minutes that can be switched off)
skipped = prepare_skipped(a)
if skipped:

View File

@@ -438,6 +438,25 @@ class ProgressSink:
return count
class HeadSink:
"""The first four bytes written, kept: extract's look at each romfs .zs file as it is unpacked (ZS_CHECKED)."""
def __init__(self, output):
self.output = output
self.head = b""
def write(self, data):
if len(self.head) < 4:
self.head += bytes(data[:4 - len(self.head)])
return self.output.write(data)
# (written into the folder a romfs was extracted to: how many of its .zs files were looked at as they were written, and
# which of them do not start as zstd data, by their path under romfs. nx2/make_game.py reads it in place of opening every
# .zs file again after the unpack, damaged_game_files: 102,185 files, 292 s of an 18 minute build where a virus scanner
# looks at each file that is opened)
ZS_CHECKED = "zs_checked.json"
def file_system(path):
"""The file system of the volume path is on (NTFS, exFAT, FAT32, ReFS): Windows only, None elsewhere"""
if os.name != "nt":
@@ -465,8 +484,13 @@ def out_of_space(error, dst, part, rel, size):
part, rel, size, free, ", " + kind if kind else ""))
def extract(nup, t, out, only=None, quiet=False):
def extract(nup, t, out, only=None, quiet=False, zs=None):
"""zs: a dict that is given "files", the romfs .zs files there are, and "damaged", those of them that do not start as
zstd data (their paths under romfs, "/" between the parts): each one looked at as it is written, and one that was
there already read back"""
total = n = written = 0
if zs is not None:
zs.update(files=0, damaged=[])
interval = 2
next_report = 0 if quiet else monotonic() + interval
@@ -483,15 +507,26 @@ def extract(nup, t, out, only=None, quiet=False):
continue
for rel, r in nup.walk(t[part]):
dst = extraction_path(out, part, rel)
looked = zs is not None and part == "romfs" and rel.endswith(".zs")
if os.path.exists(dst) and os.path.getsize(dst) == r[2]:
if looked:
with open(dst, "rb") as f:
zs["files"] += 1
if not zstd_start(f.read(4)):
zs["damaged"].append(rel)
continue # (re-runs pick up where they stopped)
os.makedirs(os.path.dirname(dst), exist_ok=True)
temporary = dst + ".part"
g = open(temporary, "wb")
try:
with g:
nup.copy_to(r, g if quiet else ProgressSink(g, progress), rel)
sink = HeadSink(g) if looked else g
nup.copy_to(r, sink if quiet else ProgressSink(sink, progress), rel)
os.replace(temporary, dst)
if looked:
zs["files"] += 1
if not zstd_start(sink.head):
zs["damaged"].append(rel)
except BaseException as error:
full = out_of_space(error, dst, part, rel, r[2]) # (measured before the member's space is freed)
# Free the failed member's space without hiding the extraction error.
@@ -583,11 +618,16 @@ def stand_in_for(path, damaged=()):
return best and best[1]
def stand_in_damaged(romfs, progress=None):
def stand_in_damaged(romfs, progress=None, known=None):
"""Each damaged .zs file in an unpacked romfs replaced by a copy of its stand-in: (damaged, stand-in or None) each.
Only for a romfs this tool unpacked: the player's own folder is never written to. progress: damaged_game_files'"""
Only for a romfs this tool unpacked: the player's own folder is never written to. progress: damaged_game_files'.
known: the damaged files as the unpack itself found them (extract's zs, paths under romfs with "/"): they are taken
as they are and no file is opened to look for them"""
done = []
damaged = damaged_game_files(romfs, progress)
if known is None:
damaged = damaged_game_files(romfs, progress)
else:
damaged = [os.path.normpath(os.path.join(romfs, *rel.split("/"))) for rel in known]
for path in damaged:
other = stand_in_for(path, set(damaged))
if other:
@@ -642,8 +682,13 @@ def main():
folder = parent
free = shutil.disk_usage(folder).free
print("extracting title %d (v%d) to %s (%.0f GB free)" % (t["index"], t["version"], a.extract, free / 1e9))
n, total = extract(nup, t, a.extract, a.only)
zs = {}
n, total = extract(nup, t, a.extract, a.only, zs=zs)
print("done: %d files written, %.2f GB" % (n, total / 1e9))
if a.only != "exefs": # (for nx2/make_game.py: the romfs .zs files as they were seen while they were written)
import json
with open(os.path.join(long_path(os.path.realpath(os.path.abspath(a.extract))), ZS_CHECKED), "w", encoding="utf-8") as f:
json.dump(zs, f)
if a.only != "romfs": # (main is ZBIC-compressed on firmware 22+: made a plain NSO for the tools)
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import unpack

View File

@@ -19,6 +19,7 @@ Each is a runtime switch where it can be, so the game's own path stays reachable
"""
import glob
import os
import re
import sys
import whole_write # (rewrites written beside and moved over: issue 1197)
@@ -176,6 +177,27 @@ def patch_menu_tick(source):
'#include "a64.h"\n', '#include "a64.h"\nextern void hos_oot_menu_tick(Ctx *c, uint64_t screen);\n', 1)
MOD_SIZE_CALL = "c->x[30] = 0x801b64e4ULL; f_836e3d30(c);\n"
MOD_SIZE_HOOK = (" c->x[0] = hos_mods_table_size(c->x[0], recomp_load64(c, c->x[29] + 24));"
" /* (nx2/oot_patches.py: room for the mods' files, host/mods.c) */\n")
def patch_mod_sizes(source):
"""((Triforce) 11-17) f_801b6340 asks the ResourceSizeTable (RESTBL, f_836e3d30: the CRC of the path, a binary search) how
much memory a file's load needs; the file's resource heap is made that size (f_801b6840). A mod's file served instead of the
game's (host/mods.c) can need more than the table gives it: a model with more materials and shapes than the original fails
its gsys::ModelResource allocations, which the game does not check (the Dark Link mod's Deku Shield and Slingshot: a write
to address 0 on loading a child save). The host raises the size for a mod's file to its real size and room beside it
(hos_mods_table_size); the game's own files keep the table's"""
if MOD_SIZE_CALL not in source:
return source, 0
if "hos_mods_table_size(" in source: # (made before: a lift patched again)
return source, 1
out = source.replace(MOD_SIZE_CALL, MOD_SIZE_CALL + MOD_SIZE_HOOK, 1)
out = out.replace('#include "a64.h"\n', '#include "a64.h"\nextern uint64_t hos_mods_table_size(uint64_t size, uint64_t path);\n', 1)
return out, 1
TEXT_HEAD = "void f_80b1e140(Ctx *__restrict A64_NOESC c) {"
# (the letters' block: past the early returns, the auto-advance timer and the hidden / no-text checks; reached once a call)
TEXT_OLD = ("L_80b1e1b4:;\n"
@@ -558,6 +580,78 @@ def patch_sprint(s):
return s, done
# (Triforce) The host's zstd decoder (host/host_zstd.h has what the two functions are and the rules; HOS_HOST_ZSTD, off unless
# set). The game's own zstd, linked into main: f_8010b380 (main+0xeb380) is ZSTD_decompressMultiFrame, f_80115c80 (main+0xf5c80)
# the stream feed its loading threads decode through. At the head of each the host is asked first: it answers the call (x0 the
# result, the function returns) or says 0 and the game's code runs as it is. hos_host_zstd is 0 once the switch was read as off,
# and the call is no longer made. A switch at run time, so that it takes no new translation to turn on or off.
# Only the supported build's: the heads are found by address, and by lines only these two functions have (their shape below), so
# a function of that name with other code (another game version, a changed lift) is left as it is.
# (Triforce) The shapes are matched with the store marks' spellings taken out (_unmarked, below). The page marks of a group of
# stores (a64_lift.mark_groups) are made after these patches (nx2/chunks.py edit_c): a store of a group becomes
# recomp_storeN_sp( and an A64_MARK line follows the group. The lift as it stands on disk has them, these patches see the text
# before them. The stream feed's begin stores (+48 and +0 through x19) are such a group: its shape, read from the lift on disk,
# was never in the text the patch saw, and every build said "found at 1 of 2 places" (the stream feed left out). The lines are
# written as the text is before the marks, the text the patch sees (tests/test_host_zstd.py holds them to that).
ZSTD_SITES = (
("void f_8010b380(Ctx *__restrict A64_NOESC c) {\n", "hos_zstd_multiframe",
# (the eighth argument kept and tested, and the dictionary read from a DDict's +8 and +16)
(" recomp_store64_sp(c,c->x[31] + 104,c->x[7]);\nif (c->x[7] != 0) { goto L_8010fd78; }\n",
"L_8010fd78:;\n c->x[8]=recomp_load64(c,c->x[31] + 104);\n"
" { uint64_t _b=c->x[8]; int64_t _o=8; { uint64_t _p0,_p1; recomp_ldp64(c,(_b+_o),&_p0,&_p1); c->x[5]=_p0; c->x[6]=_p1; } }\n")),
("void f_80115c80(Ctx *__restrict A64_NOESC c) {\n", "hos_zstd_stream",
# (the phase read from the state, the "done" result, the error code asked of zstd, and the stores of a begin. (Triforce)
# And every field host/host_zstd.h reads or writes, at its offset: the output and its room (+8, +16) read in phase 2, the
# DCtx's "bytes expected next" (HZ_DCTX_EXPECTED, 28776) read first there, the workspace (+24) read at the begin before
# the content size goes to +20; the DCtx to +48 and the phase to +0 at its end)
(" c->x[8]=recomp_load8(c,c->x[0] + 0);\n",
" c->x[0] = 0xfffffffdULL;\n",
"c->x[30] = 0x8011629cULL; f_80d5ba00(c);\n",
" c->x[22]=recomp_load64(c,c->x[21] + 28776);\n",
" c->x[23]=recomp_load64(c,c->x[19] + 8);\n"
" { uint64_t _r = (uint64_t)(int64_t)(int32_t)recomp_load32(c,c->x[19] + 16); c->x[2] = _r; }\n",
" c->x[21]=recomp_load64(c,c->x[19] + 24);\n recomp_store32(c,c->x[19] + 20,c->x[8]);\n",
" recomp_store64(c,c->x[19] + 48,c->x[21]);\n recomp_store8(c,c->x[19] + 0,c->x[8]);\n")),
)
ZSTD_HOOK = " if (hos_host_zstd && %s(c)) return; /* (nx2/oot_patches.py: the host's zstd decoder, HOS_HOST_ZSTD) */\n"
ZSTD_DECL = ("extern volatile int hos_host_zstd;\nextern int hos_zstd_multiframe(Ctx *c);\n"
"extern int hos_zstd_stream(Ctx *c);\n")
# (a64_lift.MG_MARK's line, and the _sp spelling of a marked store: mark_groups_lines, stack_stores, frame_stores)
MARK_LINE = re.compile(r"^ A64_MARK\(\(c->x\[\d+\] \+ \(uint64_t\)\(int64_t\)\(-?\d+\)\), \d+\);\n", re.M)
MARK_SP = re.compile(r"\brecomp_(store8|store16|store32|store64|stp32|stp64)_sp\(")
def _unmarked(text):
"""(Triforce) text without the store marks' spellings: a group's A64_MARK lines out, recomp_storeN_sp( as recomp_storeN(.
The code's shape alone, the same before a64_lift.mark_groups and after it (a second run over a finished lift), and with
A64_MARK_EACH, A64_MARK_STACK or A64_MARK_FRAME at lift time. They change only where a store's page is marked, never what
the guest's code does, so a shape can count on everything else"""
return MARK_SP.sub(r"recomp_\1(", MARK_LINE.sub("", text))
def patch_host_zstd(s):
"""The host decoder's two calls: (s, how many of the two heads are in s with their call, made now or before). A function
whose code is not the decoder's (another game version) is left as it is."""
done = 0
for head_text, name, shape in ZSTD_SITES:
at = _function(s, head_text)
if at is None:
continue
head, end = at
body = s[head:end]
hook = ZSTD_HOOK % name
if body.startswith(head_text + hook):
done += 1
elif hook not in body:
plain = _unmarked(body)
if all(plain.count(_unmarked(line)) == 1 for line in shape):
s = s[:head] + head_text + hook + body[len(head_text):] + s[end:]
done += 1
if done and ZSTD_DECL not in s:
s = s.replace('#include "a64.h"\n', '#include "a64.h"\n' + ZSTD_DECL, 1)
return s, done
def patch_text_speed(s):
"""The scale at the head of the letters' block of f_80b1e140; None when the file has no such function, s unchanged when the
function is there but its code differs (another game version)."""
@@ -616,6 +710,8 @@ def apply(gen, *, protect_hearts=False, required=False):
sound_jobs = solver_sites = 0 # (patch_sound_job, patch_solver_steps: the places found with their hooks)
floor_sites = sim_sites = record_sites = 0 # (patch_floor_carry, patch_sim_steps, patch_record_clock: the same)
sprint_sites = 0 # (patch_sprint: its two calls)
zstd_sites = 0 # (patch_host_zstd: its two calls, the supported build's alone)
mod_size_sites = 0 # ((Triforce) patch_mod_sizes: its one call, the supported build's alone)
for path in sorted(glob.glob(os.path.join(glob.escape(gen), "code_*.c"))):
with open(path, encoding="utf-8", newline="") as f:
s = f.read()
@@ -721,6 +817,17 @@ def apply(gen, *, protect_hearts=False, required=False):
if sprinted != out:
print("oot_patches: the sprint button in", os.path.basename(path))
out = sprinted
if protect_hearts: # (found by address: the supported build's functions)
zstd, found = patch_host_zstd(out)
zstd_sites += found
if zstd != out:
print("oot_patches: the host's zstd decoder's call(s) in", os.path.basename(path))
out = zstd
sized, found = patch_mod_sizes(out)
mod_size_sites += found
if sized != out:
print("oot_patches: room for the mods' files in", os.path.basename(path))
out = sized
if out != s:
whole_write.write_whole(path, out, newline="")
if sound_jobs != 1: # (without it, above 60 fps sounds and then the music can stop after a few loading zones)
@@ -736,6 +843,10 @@ def apply(gen, *, protect_hearts=False, required=False):
% (record_sites, sum(n for _, n in RECORD_SITES)))
if sprint_sites != 2: # (without them the Sprint button row does nothing)
print("oot_patches: WARNING: the sprint button's calls were found at %d of 2 places" % sprint_sites)
if protect_hearts and mod_size_sites != 1: # (without it a mod's bigger model can crash the game: host/mods.c)
print("oot_patches: WARNING: the resource size table's lookup was found at %d of 1 place" % mod_size_sites)
if protect_hearts and zstd_sites != len(ZSTD_SITES): # (without them HOS_HOST_ZSTD does nothing, or half: host/hos_zstd.c)
print("oot_patches: WARNING: the host's zstd decoder's calls were found at %d of %d places" % (zstd_sites, len(ZSTD_SITES)))
for what, n in done.items():
if n != 1:
print("oot_patches: WARNING: the %s was found %d times" % (what, n))

View File

@@ -18,10 +18,11 @@ MAIN = 0x80020000 # (main's base in this image: nx2/eden_image.py)
UI_M00, UI_M11 = "(2.0f / 3840.0f)", "(2.0f / 2160.0f)" # (the UI's full-screen orthographic projection, 3840x2160 grid)
# (the screens whose UI keeps the whole picture: the item-wheel and pause / system menu backgrounds redraw a capture of
# the whole picture; the event and loading fades cover it. Bit k of host/hos_main.c's hos_wide_keep_mask keeps screen k
# whole; the Insert menu's Ultrawide menus row clears the menus' bits 1 and 3 (vk_framegen.cpp): their layout then
# whole; the Insert menu's Ultrawide menus row clears the menus' bit 1 (vk_framegen.cpp): their layout then
# keeps its 16:9 shape in the middle, as the HUD's does, instead of being stretched over the whole picture. Bit 0, the
# item wheel's background (the last frame of play, blurred, HUD and all), and bit 2, the fades, keep covering it all: bit 0
# cleared, that capture went into the middle at twice its width)
# item wheel's background (the last frame of play, blurred, HUD and all), and bits 2 and 3, the two fades, keep covering
# it all: bit 0 cleared, that capture went into the middle at twice its width, and a fade in the middle alone left the
# picture showing at its sides while a place loaded)
KEEP_VTABLES = (MAIN + 0x4bed098, MAIN + 0x4be4fb0, MAIN + 0x4be61c8, MAIN + 0x4be7b30)

View File

@@ -3,12 +3,14 @@ and Kenney's Input Prompts (CC0, third_party/kenney_input).
python -I nx2/prompt_assets.py --romfs <dir> --out <game folder>/prompts
Writes <out>/<face>/External.bfotf for each face (switch, xbox, ps4, ps5, steamdeck). Every text in the game draws its button icons
Writes <out>/<face>/External.bfotf for each face (switch, xbox, ps4, ps5, steamdeck, keyboard). Every text in the game draws its button icons
from Font2/Scalable/External.*.bfotf (dialogue, the control hints, menus): A B X Y at U+E040..E043, L R ZL ZR at E044..E047, Plus
and Minus at E04B and E04C, the stick presses at E056 and E057, the d-pad at E060..E063. Each written font has one icon per pad
input at U+F700 + HOS_PAD_* (gpu/vk/vk_api.h), and each of those Switch buttons' code points in a cmap segment of its own: the
game (host/prompts.c) points each one at the icon of the pad input bound to that button when it serves the font, so the icons
follow the controls, changed ones too.
follow the controls, changed ones too. (Triforce) The keyboard face (the keyboard and mouse; nx2/prompt_keys.py draws it, from
nothing but a font of the repository's) has an icon per key that can be bound instead, at U+F700 + its key code, and what is
said of a pad input below is a key code's there.
The textures with button icons (the ocarina's notes, the letters on black, the controller in the pause menu) the same way, with
Pillow, astc_encoder and texture2ddecoder: <out>/Common.bntx is the game's UI/Layout2Archive/Common bntx decompressed, and
@@ -78,10 +80,19 @@ def zs_dictionary(romfs, which=b"zs.zsdic"):
def read_zs(romfs, rel, dictionary):
"""(Triforce, as NX2navi's tools/pad_prompts.py unpack) The file goes to the decompressor a piece at a time and the
pieces' outputs are joined once. Given the whole file in one call the library grows one bytes object by copying as the
output comes, and the textures (69 MB packed, 244 MB unpacked) took 20 s of this tool's 80 that way"""
import zstandard
d = zstandard.ZstdDecompressor(dict_data=zstandard.ZstdCompressionDict(dictionary))
d = zstandard.ZstdDecompressor(dict_data=zstandard.ZstdCompressionDict(dictionary)).decompressobj()
with open(os.path.join(romfs, rel), "rb") as f:
return d.decompressobj().decompress(f.read())
raw = f.read()
parts = []
for at in range(0, len(raw), 1 << 16):
parts.append(d.decompress(raw[at:at + (1 << 16)]))
if d.eof:
break
return b"".join(parts)
def size_table(romfs, dictionary, game_size, font_size):
@@ -415,43 +426,56 @@ def face_layers(b, tex, face, otf):
"""this face's images, as layers (in textures.bin's order), each a hand-made one if nx2/prompt_faces/<face>/ has it"""
from PIL import Image, ImageDraw
import prompt_draw
icons = Icons(face, otf)
keys = bool(FACES.get(face, {}).get("keys")) # ((Triforce) the keyboard and mouse: an icon per key code, no controller)
if keys:
import prompt_keys
icons = prompt_keys.Icons() if keys else Icons(face, otf)
# (what a Switch button can be on: its name in icons, its number in textures.bin, its name in a layer's)
inputs = [(vk, vk, "key%d" % vk) for vk in sorted(icons.code)] if keys else [(p, i, p) for i, p in enumerate(PADS)]
layers = []
made = {} # ((Triforce) an input's image is the same on every texture of one size: made once, and encoded once)
def keep(sw, pad): # (the Switch face's default controls: the game's own textures)
return pad in icons.code and not (face == "switch" and DEFAULT[SW[sw]] == pad)
for name, sw in NOTES.items():
t = tex[name]
w, h = t[2], t[3]
for pad in PADS:
if not keep(sw, pad):
continue
def note(pad, w, h):
if ("note", pad, w, h) not in made:
icon, shape, holes = icons.masks(pad, int(min(w, h) * 0.9))
im = Image.new("RGBA", (w, h), (0, 0, 0, 0))
o = ((w - icon.width) // 2, (h - icon.height) // 2)
im.paste(Image.new("RGBA", icon.size, (255, 255, 255, 255)), o, shape)
im.paste(Image.new("RGBA", icon.size, (0, 0, 0, 255)), o, holes)
layers.append(Layer(f"{name}.{pad}.png", name, SW[sw], PADS.index(pad), im))
for name, sw in LETTERS.items():
t = tex[name]
w, h = t[2], t[3]
for pad in PADS:
if not keep(sw, pad):
continue
made["note", pad, w, h] = im
return made["note", pad, w, h]
def letter_of(pad, w, h):
if ("letter", pad, w, h) not in made:
icon, shape, holes = icons.masks(pad, int(min(w, h) * 0.9))
letter = holes if holes.getbbox() else icon # (an icon with no letter in it: its shape)
# (an icon with no letter in it: its shape; a mouse's: the mouse, not what its outline leaves)
letter = holes if holes.getbbox() and pad not in getattr(icons, "plain", ()) else icon
im = Image.new("L", (w, h), 0)
im.paste(letter, ((w - letter.width) // 2, (h - letter.height) // 2))
layers.append(Layer(f"{name}.{pad}.png", name, SW[sw], PADS.index(pad), im))
made["letter", pad, w, h] = im
return made["letter", pad, w, h]
for name, sw in NOTES.items():
t = tex[name]
for pad, at, label in inputs:
if keep(sw, pad):
layers.append(Layer(f"{name}.{label}.png", name, SW[sw], at, note(pad, t[2], t[3])))
for name, sw in LETTERS.items():
t = tex[name]
for pad, at, label in inputs:
if keep(sw, pad):
layers.append(Layer(f"{name}.{label}.png", name, SW[sw], at, letter_of(pad, t[2], t[3])))
own = None
if face != "switch":
if face != "switch" and not keys: # ((Triforce) the keyboard has no picture: the page keeps the game's controller)
hand = override(face, PICTURES[0] + ".png")
own = Image.open(hand).convert("RGBA") if hand else prompt_draw.picture(
face, decode(b, tex[PICTURES[0]]).convert("RGBA"), Icons("switch", otf))
if own:
layers += picture_layers(b, tex, own)
for name in PICTURES if face != "switch" and not own else ():
for name in PICTURES if face != "switch" and not own and not keys else ():
t = tex[name]
im = decode(b, t)
px = im.load()
@@ -480,7 +504,7 @@ def face_layers(b, tex, face, otf):
def face_textures(b, tex, layers):
"""textures.bin's entries for these layers: [(Switch button, pad input, runs)], Switch button 0xff for those always written"""
from PIL import ImageChops
entries, made = [], {}
entries, made, encoded = [], {}, {}
for layer in layers:
t, im = tex[layer.texture], layer.im
made[layer.name] = im
@@ -493,7 +517,13 @@ def face_textures(b, tex, layers):
x0, y0, x1, y1 = min(x0, bx0), min(y0, by0), max(x1, bx1), max(y1, by1)
entries.append((layer.sw, layer.pad, runs(t, im, (x0, y0, x1, y1))))
else:
entries.append((layer.sw, layer.pad, runs(t, im)))
# ((Triforce) the same image on another texture of that size and format, a note or a letter on another Switch
# button: the same blocks, at that texture's place. The notes were encoded six times over, the letters four)
same = (id(im), t[1], t[2], t[3], t[4])
if same not in encoded:
encoded[same] = (t[5], runs(t, im))
data, rs = encoded[same]
entries.append((layer.sw, layer.pad, [[off - data + t[5], piece] for off, piece in rs]))
return entries
@@ -874,12 +904,15 @@ def hint_panes(blarc):
return out or None
def write_hint(path, otf, panes):
def write_hint(path, otf, panes, names=None):
"""(names: the glyph of each input's icon, in the inputs' order, None for a number that is no input: the pad inputs',
or the keyboard face's 256 key codes')"""
from fontTools.ttLib import TTFont
hmtx = TTFont(io.BytesIO(otf))["hmtx"]
names = names or [f"pad_{p}" for p in PADS]
with open(path, "wb") as f:
f.write(b"HPHN" + struct.pack("<I", len(PADS)))
f.write(b"".join(struct.pack("<f", round(max(0, hmtx[f"pad_{p}"][0] - HINT_FITS) * HINT_SCALE)) for p in PADS))
f.write(b"HPHN" + struct.pack("<I", len(names)))
f.write(b"".join(struct.pack("<f", round(max(0, hmtx[n][0] - HINT_FITS) * HINT_SCALE) if n else 0.0) for n in names))
f.write(struct.pack("<I", len(panes)) + b"".join(struct.pack("<IB3xf", at, slot, x) for at, slot, x in panes))
@@ -1074,10 +1107,23 @@ def main():
for face in a.faces.split(","):
d = os.path.join(a.out, face)
os.makedirs(d, exist_ok=True)
plain = face_font(otf, face)
names = None
if FACES.get(face, {}).get("keys"): # ((Triforce) the keyboard and mouse: its icons are drawn, nx2/prompt_keys.py)
try:
import prompt_keys
plain, binds = prompt_keys.game_font(otf)
except (Exception, SystemExit) as e: # (tools without the labels' font, a game font it cannot take: the
# controllers' faces are made all the same, and the game shows its own icons for the keyboard)
print(f"prompts: {face} was not made ({e})", file=sys.stderr)
os.rmdir(d)
continue
prompt_keys.write_binds(os.path.join(d, "keys.bin"), binds)
names = prompt_keys.hint_names()
else:
plain = face_font(otf, face)
font = wrap(key, plain)
if hint:
write_hint(os.path.join(d, "hint.bin"), plain, hint)
write_hint(os.path.join(d, "hint.bin"), plain, hint, names)
largest = max(largest, len(font))
with open(os.path.join(d, "External.bfotf"), "wb") as f:
f.write(font)

View File

@@ -21,6 +21,9 @@ Each face:
line ends on it), a button's centre and radius (its line ends just off the edge it comes from), a shoulder's top
edge (r 0). The four face buttons are framed. Not a pad input's: boxes the lines keep off (x0, y0, x1, y1: a
screen, a touchpad). SWITCH_PARTS: the game's Pro Controller's.
keys (Triforce) True for the keyboard and mouse, which is no controller and has none of the above: its icons are one
per key that can be bound, drawn by nx2/prompt_keys.py (the keys, their labels and what each code point of the
font shows are there), and the host picks them by the keys bound instead of the pad inputs.
"""
import os
@@ -96,6 +99,10 @@ FACES = {
icons=dict(_DPAD, south="button_a", east="button_b", west="button_x", north="button_y", back="button_view",
start="button_options", lshoulder="button_l1", rshoulder="button_r1", ltrigger="button_l2",
rtrigger="button_r2")),
# (Triforce) the keyboard and mouse: no controller, so no font of Kenney's, no drawing and no parts. keys: its icons are
# the keys' and the mouse's buttons', one per key that can be bound, drawn by nx2/prompt_keys.py; the controls page keeps
# the game's own controller picture
"keyboard": dict(keys=True),
}

View File

@@ -23,6 +23,13 @@ The Switch face keeps the game's own images for its default controls; only the o
Pad inputs: `south east west north back guide start lstick rstick lshoulder rshoulder up down left right ltrigger rtrigger`.
Switch buttons: `a b x y ls rs l r zl zr plus minus left up right down`.
The `keyboard` face (the keyboard and mouse) is drawn by `nx2/prompt_keys.py` from nothing but Libertinus Sans Bold
(`third_party/fonts`): a cap with its label for every key that can be bound, a mouse for each mouse button. Its notes and
letters are named by the key's code where the others have a pad input, `OcarinaButtonA_00.key69.png` for E, and a PNG of
that name in `nx2/prompt_faces/keyboard/` takes the image's place as for any face. It has no controller picture and no
leader lines: the controls page keeps the game's. `python -I nx2/prompt_keys.py --out <dir>` writes every icon into one
picture, to look at.
## Drawing one by hand
1. Write out what the build makes (and the game's own textures, to draw over):

500
nx2/prompt_keys.py Normal file
View File

@@ -0,0 +1,500 @@
"""(Triforce) The keyboard and mouse face of the button prompts: the game's button icons showing the key or the mouse button
each control is bound to (NX2navi's prompts = pc; nx2/prompt_assets.py builds it with the controllers' faces, host/prompts.c
serves it).
Nothing is read for it but a font the repository carries: a key is a cap drawn here, a rounded square (wider for a word) with
its label cut out of it in Libertinus Sans Bold (third_party/fonts/libertinus, SIL OFL: the font the menus fall back on); an
arrow key's label is an arrow; a mouse button is a mouse with that button filled; the camera is a mouse between arrows.
The controls can be changed (the Controls page), so the font has an icon for every key that can be bound, at U+F700 + its
Windows key code (KEYS: the keys of gpu/vk/vk_menu.cpp bindable_keys), and the host points each button's code point at the
icon of the first key bound to it when it serves the font, as it does with a pad's inputs. What the font's own code points
show:
the Switch's buttons (prompt_assets.SWITCH) the key of each (DEFAULT: host/nn_hid.c key_default's first keys)
the left stick's four directions (MOVES) the movement keys
the left stick, its two axes, the d-pad and its those keys side by side, as they lie on the keyboard (CLUSTERS)
two axes
the right stick and its directions (CAMERA) the mouse, with arrows for the way it moves
An icon of several keys cannot be one glyph per choice of keys. It is a glyph that calls a subroutine per key, and each key's
small cap is such a subroutine (a local one, as the game's font already has: mini). <face>/keys.bin lists where those calls
are in the font and each key's call, and the host writes the calls of the keys bound: "HPKB", u32 count, then count times
u32 offset (in the font, without its 8-byte header), u8 control (HOS_KEY_*), u8 0 x3; then u32 256 and for each key code the
two bytes of its call (0 0: no such key, the blank cap's).
python -I nx2/prompt_keys.py --out <dir> (every icon as it is drawn, in one PNG, to look at; needs Pillow)
"""
import io
import math
import os
import struct
import sys
HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
import prompt_assets as P # noqa: E402
LETTERS = os.path.join(os.path.dirname(HERE), "third_party", "fonts", "libertinus", "LibertinusSans-Bold.ttf")
BASE = P.PAD_BASE # (a key's icon is at BASE + its key code, as a pad input's is at BASE + HOS_PAD_*)
# every key a control can be bound to, by its Windows key code, and its label: a line, two ("Num" small over the key), or a
# drawing ("@up" an arrow, "@mouse_l" the mouse with its left button filled). F1 is the menu's.
KEYS = {ord(c): c for c in "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"}
KEYS.update({0x70 + n - 1: "F%d" % n for n in range(2, 13)})
KEYS.update({0x60 + n: "Num\n%d" % n for n in range(10)})
KEYS.update({
0x20: "Space", 0x10: "Shift", 0x11: "Ctrl", 0x12: "Alt", 0x09: "Tab", 0x0D: "Enter", 0x08: "Bksp", 0x14: "Caps",
0x26: "@up", 0x28: "@down", 0x25: "@left", 0x27: "@right", 0x24: "Home", 0x23: "End", 0x21: "PgUp", 0x22: "PgDn",
0x6B: "Num\n+", 0x6D: "Num\n-", 0x6A: "Num\n*", 0x6F: "Num\n/", 0x6E: "Num\n.",
0xBA: ";", 0xBB: "=", 0xBC: ",", 0xBD: "-", 0xBE: ".", 0xBF: "/", 0xC0: "`", 0xDB: "[", 0xDC: "\\", 0xDD: "]", 0xDE: "'",
0x01: "@mouse_l", 0x02: "@mouse_r", 0x04: "@mouse_m", 0x05: "@mouse_4", 0x06: "@mouse_5"})
# the first key of each control out of the box, in HOS_KEY_* order (host/nn_hid.c key_default): the Switch's buttons A B X Y
# LS RS L R ZL ZR Plus Minus Left Up Right Down, then move up, down, left, right and camera up, down, left, right
DEFAULT = [0x45, 0x10, 0x20, 0x01, 0x11, 0x43, 0x51, 0x52, 0x5A, 0x58, 0x09, 0x4D, 0x31, 0x32, 0x33, 0x34, 0x57, 0x53, 0x41,
0x44, 0x26, 0x28, 0x25, 0x27]
MOVE_UP, MOVE_DOWN, MOVE_LEFT, MOVE_RIGHT = 16, 17, 18, 19
PAD_LEFT, PAD_UP, PAD_RIGHT, PAD_DOWN = 12, 13, 14, 15
# the left stick pushed one way: that movement key (the font's code point: the control)
MOVES = {0xE052: MOVE_UP, 0xE050: MOVE_DOWN, 0xE05A: MOVE_LEFT, 0xE058: MOVE_RIGHT}
# the icons of several keys: each key's control and its place (column, row from the top)
CLUSTERS = {
0xE05E: [(MOVE_UP, 1, 0), (MOVE_LEFT, 0, 1), (MOVE_DOWN, 1, 1), (MOVE_RIGHT, 2, 1)], # (the left stick)
0xE05C: [(MOVE_UP, 0, 0), (MOVE_DOWN, 0, 1)], 0xE054: [(MOVE_LEFT, 0, 0), (MOVE_RIGHT, 1, 0)],
0xE049: [(PAD_UP, 1, 0), (PAD_LEFT, 0, 1), (PAD_DOWN, 1, 1), (PAD_RIGHT, 2, 1)], # (the d-pad)
0xE06A: [(PAD_UP, 1, 0), (PAD_LEFT, 0, 1), (PAD_DOWN, 1, 1), (PAD_RIGHT, 2, 1)],
0xE064: [(PAD_UP, 0, 0), (PAD_DOWN, 0, 1)], 0xE065: [(PAD_LEFT, 0, 0), (PAD_RIGHT, 1, 0)]}
# the right stick is the camera, which the mouse turns: the mouse and the ways it moves (the camera's keys, a second way
# to turn it, do not show)
CAMERA = {0xE05F: "xy", 0xE068: "xy", 0xE053: "y", 0xE051: "y", 0xE05D: "y", 0xE059: "x", 0xE05B: "x", 0xE055: "x"}
MICE = {vk for vk, label in KEYS.items() if label.startswith("@mouse")} # (their icons are no caps)
# ---- the drawings, in units of their own: a cap is H high, its corners R round, a word's as wide as the word up to
# P.WIDE caps; a mouse MOUSE wide. A contour that is ink runs counter-clockwise; a label is cut out of its cap by running
# the other way, which is the way Libertinus's outlines run (TrueType's)
H, R, MOUSE, STROKE = 1000, 180, 640, 80
KAPPA = 0.5523 # (a quarter circle's control points, of its radius)
MINI, MINI_R, MINI_GAP = 290, 52, 20 # (a cap in an icon of several keys, the game's font's units: two rows fill an icon)
MINI_MARK = [(16, 30), (30, 30), (30, 44), (16, 44)] # (a small cap's last contour, which ends at its last point: see mini)
def rounded(pen, x0, y0, x1, y1, r):
"""a rounded rectangle, counter-clockwise from (x0 + r, y0) (a side of no length, a round end's, is left out)"""
k = r * (1 - KAPPA)
at = (x0 + r, y0)
pen.moveTo(at)
for line, c1, c2, to in ((x1 - r, y0), (x1 - k, y0), (x1, y0 + k), (x1, y0 + r)), \
((x1, y1 - r), (x1, y1 - k), (x1 - k, y1), (x1 - r, y1)), \
((x0 + r, y1), (x0 + k, y1), (x0, y1 - k), (x0, y1 - r)), \
((x0, y0 + r), (x0, y0 + k), (x0 + k, y0), (x0 + r, y0)):
if line != at:
pen.lineTo(line)
pen.curveTo(c1, c2, to)
at = to
pen.closePath()
def polygon(pen, points):
pen.moveTo(points[0])
for p in points[1:]:
pen.lineTo(p)
pen.closePath()
def arc(pen, cx, cy, r, a0, a1):
"""a circle's arc from angle a0 to a1 (degrees, counter-clockwise), from the pen's place at its start"""
steps = max(1, math.ceil(abs(a1 - a0) / 90))
for i in range(steps):
t0, t1 = (math.radians(a0 + (a1 - a0) * j / steps) for j in (i, i + 1))
k = 4 / 3 * math.tan((t1 - t0) / 4) * r
pen.curveTo((cx + r * math.cos(t0) - k * math.sin(t0), cy + r * math.sin(t0) + k * math.cos(t0)),
(cx + r * math.cos(t1) + k * math.sin(t1), cy + r * math.sin(t1) - k * math.cos(t1)),
(cx + r * math.cos(t1), cy + r * math.sin(t1)))
def arrow(pen, cx, cy, size, way):
"""an arrow (counter-clockwise) pointing up, left, down or right, size long, about (cx, cy)"""
a, b = size / 2, size / 6
up = [(0, a), (-a, 0), (-b, 0), (-b, -a), (b, -a), (b, 0), (a, 0)]
turn = {"up": (1, 0), "left": (0, 1), "down": (-1, 0), "right": (0, -1)}[way] # (cos, sin of a quarter turn each)
polygon(pen, [(cx + x * turn[0] - y * turn[1], cy + x * turn[1] + y * turn[0]) for x, y in up])
class Letters:
"""Libertinus Sans Bold's outlines, for the labels"""
def __init__(self):
from fontTools.ttLib import TTFont
font = TTFont(LETTERS)
self.glyphs, self.cmap, self.advance, self.cap = font.getGlyphSet(), font.getBestCmap(), font["hmtx"], font["OS/2"].sCapHeight
def ink(self, text):
"""a line's ink, from its left to its right, in the font's units"""
from fontTools.pens.boundsPen import BoundsPen
x, lo, hi = 0, None, None
for ch in text:
g = self.cmap[ord(ch)]
b = BoundsPen(self.glyphs)
self.glyphs[g].draw(b)
if b.bounds:
lo = x + b.bounds[0] if lo is None else min(lo, x + b.bounds[0])
hi = x + b.bounds[2] if hi is None else max(hi, x + b.bounds[2])
x += self.advance[g][0]
return lo or 0, hi or 0
def draw(self, pen, text, scale, x, y):
"""a line from x on the baseline y, as the font's outlines run (clockwise); a glyph made of others as outlines too"""
from fontTools.pens.recordingPen import DecomposingRecordingPen
from fontTools.pens.transformPen import TransformPen
for ch in text:
g = self.cmap[ord(ch)]
outline = DecomposingRecordingPen(self.glyphs)
self.glyphs[g].draw(outline)
outline.replay(TransformPen(pen, (scale, 0, 0, scale, x, y)))
x += self.advance[g][0] * scale
def lines_of(letters, label, h, room):
"""a label's lines in a cap h high with room for the ink: [(text, scale, the ink's width, baseline from the cap's middle)].
One letter is half the cap high, a word less, and what is wider than the room is made smaller (one letter has a square
cap's room: a W's cap is no wider than an I's); of two lines the upper is the smaller ("Num")"""
def line(text, cap_height, middle):
s = cap_height / letters.cap
lo, hi = letters.ink(text)
fits = min(room, h * 0.64) if len(text) == 1 else room
if (hi - lo) * s > fits:
s = fits / (hi - lo)
return text, s, (hi - lo) * s, middle - s * letters.cap / 2
lines = label.split("\n")
if len(lines) == 1:
return [line(lines[0], h * (0.5 if len(lines[0]) == 1 else 0.4), 0)]
return [line(lines[0], h * 0.2, h * 0.24), line(lines[1], h * 0.38, -h * 0.13)]
def key_width(letters, vk):
"""how wide a key's icon is (it is H high)"""
label = KEYS.get(vk, "")
if vk in MICE:
return MOUSE
if not label or label.startswith("@"):
return H
widest = max(w for _, _, w, _ in lines_of(letters, label, H, H * (P.WIDE - 0.3)))
return int(min(H * P.WIDE, max(H, widest + H * 0.36)))
def cap(pen, letters, label, w, h, r, room):
"""a cap w by h from (0, 0), its label cut out of it"""
from fontTools.pens.reverseContourPen import ReverseContourPen
from fontTools.pens.transformPen import TransformPen
rounded(pen, 0, 0, w, h, r)
if label.startswith("@mouse"): # (in an icon of several keys: a small mouse cut out of the cap, its lines 30 thick)
s = h * 0.72 / H
mouse(TransformPen(ReverseContourPen(pen), (s, 0, 0, s, w / 2 - s * MOUSE / 2, h * 0.14)), label[7:], 30 / s)
elif label.startswith("@"):
arrow(ReverseContourPen(pen), w / 2, h / 2, h * 0.5, label[1:])
elif label:
for text, s, width, base in lines_of(letters, label, h, room):
lo, _ = letters.ink(text)
letters.draw(pen, text, s, w / 2 - width / 2 - lo * s, h / 2 + base)
def mouse(pen, button, stroke=STROKE):
"""a mouse MOUSE by H from (0, 0): its outline, and the button filled ("l", "r", "m"; "4", "5": the number on its palm;
"": none). No part touches another: the same contours cut a small mouse out of a cap"""
from fontTools.pens.reverseContourPen import ReverseContourPen
from fontTools.pens.transformPen import TransformPen
half, gap = MOUSE / 2, max(40, stroke / 2)
rounded(pen, 0, 0, MOUSE, H, half)
rounded(ReverseContourPen(pen), stroke, stroke, MOUSE - stroke, H - stroke, half - stroke)
top, r = H - half, half - stroke - gap # (the buttons: the upper end's inside, a gap off the outline and each other)
if button in ("l", "r"):
side = pen if button == "l" else TransformPen(ReverseContourPen(pen), (-1, 0, 0, 1, MOUSE, 0))
x = half - gap / 2
a = math.degrees(math.acos((x - half) / r))
side.moveTo((half - r, top - 120))
side.lineTo((x, top - 120))
side.lineTo((x, top + r * math.sin(math.radians(a))))
arc(side, half, top, r, a, 180)
side.closePath()
elif button == "m": # (the wheel)
wheel = min(55, 0.3 * r)
rounded(pen, half - wheel, top - 0.35 * r, half + wheel, top + 0.6 * r, wheel)
elif button: # (the side buttons have no place in this picture: their number)
cap_height = (half - stroke - 20) / 0.625
s = cap_height / letters_once().cap
lo, hi = letters_once().ink(button)
letters_once().draw(ReverseContourPen(pen), button, s, half - s * (lo + hi) / 2, half - cap_height / 2)
def letters_once(_made=[]):
if not _made:
_made.append(Letters())
return _made[0]
def camera_size(axes):
return {"xy": (820, 1000), "x": (820, 500), "y": (320, 1000)}[axes]
def camera(pen, axes):
"""the mouse moving: a small mouse, an arrowhead each way it moves (axes "xy", "x" or "y"), from (0, 0); camera_size"""
from fontTools.pens.reverseContourPen import ReverseContourPen
from fontTools.pens.transformPen import TransformPen
w, h = camera_size(axes)
cx, cy = w / 2, h / 2
body = TransformPen(pen, (1, 0, 0, 1, cx - 160, cy - 250))
rounded(body, 0, 0, 320, 500, 160)
rounded(ReverseContourPen(body), 60, 60, 260, 440, 100)
polygon(pen, [(cx - 30, cy + 60), (cx + 30, cy + 60), (cx + 30, cy + 150), (cx - 30, cy + 150)]) # (its wheel)
if "y" in axes:
polygon(pen, [(cx, cy + 500), (cx - 120, cy + 340), (cx + 120, cy + 340)])
polygon(pen, [(cx, cy - 500), (cx + 120, cy - 340), (cx - 120, cy - 340)])
if "x" in axes:
polygon(pen, [(cx - 410, cy), (cx - 250, cy - 120), (cx - 250, cy + 120)])
polygon(pen, [(cx + 410, cy), (cx + 250, cy + 120), (cx + 250, cy - 120)])
def key(pen, letters, vk):
"""a key's icon from (0, 0): key_width wide, H high (a key code that is no key: the blank cap)"""
if vk in MICE:
mouse(pen, KEYS[vk][7:])
else:
cap(pen, letters, KEYS.get(vk, ""), key_width(letters, vk), H, R, H * (P.WIDE - 0.3))
def mini(pen, letters, vk):
"""a key's small cap, MINI square, for an icon of several keys. Its outline begins at (MINI_R, 0) with the cap and ends
at MINI_MARK's last point whatever the key: the last contour is a small square inside the cap's ink (it runs the cap's
way: filled all the same, and no label reaches that corner), so that the glyph that calls it knows where the pen is
before and after, and any key's cap can stand in for another's"""
cap(pen, letters, KEYS.get(vk, ""), MINI, MINI, MINI_R, MINI * 0.74)
polygon(pen, MINI_MARK)
def standalone():
"""the icons as a font of their own (OpenType, each at BASE + its key code, the camera's at BASE + 0x100 on), in the
drawings' units, a cap half an em high (prompt_assets.Icons draws an icon on a sheet two ems wide): what the textures'
images are drawn from"""
from fontTools.fontBuilder import FontBuilder
from fontTools.pens.t2CharStringPen import T2CharStringPen
letters = letters_once()
names, charstrings, metrics, cmap = [".notdef"], {}, {".notdef": (H, 0)}, {}
pen = T2CharStringPen(H, None)
charstrings[".notdef"] = pen.getCharString()
for vk in sorted(KEYS):
name, w = "key_%d" % vk, key_width(letters, vk)
pen = T2CharStringPen(w + 100, None)
key(pen, letters, vk)
names.append(name)
charstrings[name], metrics[name], cmap[BASE + vk] = pen.getCharString(), (w + 100, 0), name
for i, axes in enumerate(("xy", "x", "y")):
name, w = "camera_" + axes, camera_size(axes)[0]
pen = T2CharStringPen(w + 100, None)
camera(pen, axes)
names.append(name)
charstrings[name], metrics[name], cmap[BASE + 0x100 + i] = pen.getCharString(), (w + 100, 0), name
fb = FontBuilder(2 * H, isTTF=False)
fb.setupGlyphOrder(names)
fb.setupCharacterMap(cmap)
fb.setupCFF("TriforceKeys", {"FullName": "Triforce Keys"}, charstrings, {})
fb.setupHorizontalMetrics(metrics)
fb.setupHorizontalHeader(ascent=H, descent=0)
fb.setupNameTable({"familyName": "Triforce Keys", "styleName": "Regular"})
fb.setupOS2(sTypoAscender=H, sTypoDescender=0, usWinAscent=H, usWinDescent=0)
fb.setupPost()
out = io.BytesIO()
fb.save(out)
return out.getvalue()
class Icons(P.Icons):
"""the keys' icons for the textures (the ocarina's notes, the letters on black), by key code: prompt_assets.Icons drawn
from standalone(). plain: the icons whose letter on black is the whole icon, not what is cut out of it (the mice)"""
def __init__(self):
self.path, self.code, self.plain = io.BytesIO(standalone()), {vk: BASE + vk for vk in KEYS}, MICE
def subr_bias(count):
"""the number a charstring subtracts from a subroutine's index, by how many subroutines the font has (CFF)"""
return 107 if count < 1240 else 1131 if count < 33900 else 32768
def game_font(otf, default=DEFAULT):
"""the game's font (decrypted) as the keyboard and mouse face, the controls' keys out of the box `default`: (the font,
binds). binds: {"patches": [(offset in the font, control)], "calls": {key code: the two bytes of its small cap's call}},
what the host writes for other keys (write_binds); no patches when the font cannot take the subroutines (none of its
own, or too many): the icons of several keys then show the keys out of the box whatever is bound"""
from fontTools.misc.psCharStrings import T2CharString
from fontTools.pens.boundsPen import BoundsPen
from fontTools.pens.t2CharStringPen import T2CharStringPen
from fontTools.pens.transformPen import TransformPen
from fontTools.ttLib import TTFont
from fontTools.ttLib.tables.DefaultTable import DefaultTable
game = TTFont(io.BytesIO(otf))
gset, cmap = game.getGlyphSet(), dict(game.getBestCmap())
top = game["CFF "].cff.topDictIndex[0]
chars, order, hmtx = top.CharStrings, list(game.getGlyphOrder()), game["hmtx"]
own = chars[cmap[P.SWITCH[0]]] # (a charstring of the font: every new one shares its private dictionary and subroutines)
letters = letters_once()
def bounds(name):
p = BoundsPen(gset)
gset[name].draw(p)
return p.bounds
def fitted(box, w, h, advance):
"""the transform of a drawing w by h into a glyph's box (as prompt_assets.face_font's draw: a square about the box's
middle, as high as it and up to P.WIDE times as wide, the glyph widened), and the glyph's advance"""
side = max(box[2] - box[0], box[3] - box[1])
s = min(side * P.WIDE / w, side / h)
over = max(0, s * w - side)
return (s, 0, 0, s, (box[0] + box[2] + over) / 2 - s * w / 2, (box[1] + box[3]) / 2 - s * h / 2), advance + round(over)
def charstring(pen):
return pen.getCharString(private=own.private, globalSubrs=own.globalSubrs)
# each key's icon, in the box of the game's A
a_box, a_advance = bounds(cmap[P.SWITCH[0]]), hmtx[cmap[P.SWITCH[0]]][0]
for vk in [0] + sorted(KEYS):
t, width = fitted(a_box, key_width(letters, vk), H, a_advance)
pen = T2CharStringPen(width, gset)
key(TransformPen(pen, t), letters, vk)
name = "key_%d" % vk
chars.charStringsIndex.append(charstring(pen))
chars.charStrings[name] = len(chars.charStringsIndex) - 1
top.charset.append(name)
order.append(name)
hmtx.metrics[name] = (width, 0)
# the camera's: the mouse moving, inside the right stick's glyphs' boxes (their advances stay)
for code, axes in CAMERA.items():
box = bounds(cmap[code]) if code in cmap else None
if box:
w, h = camera_size(axes)
s = min((box[2] - box[0]) / w, (box[3] - box[1]) / h)
pen = T2CharStringPen(hmtx[cmap[code]][0], gset)
camera(TransformPen(pen, (s, 0, 0, s, (box[0] + box[2] - s * w) / 2, (box[1] + box[3] - s * h) / 2)), axes)
chars[cmap[code]] = charstring(pen)
# each key's small cap as a subroutine, then the icons of several keys calling them. The calls the host rewrites are
# all two bytes long (a subroutine's number less the bias from 108 to 1131): empty subroutines fill up to the first
subrs = getattr(own.private, "Subrs", None)
codes = [0] + sorted(KEYS)
bias = subr_bias(len(subrs)) if subrs is not None else 0
first = max(len(subrs), bias + 108) if subrs is not None else 0
live = subrs is not None and subr_bias(first + len(codes)) == bias and first + len(codes) - 1 - bias <= 1131
index = {}
if live:
while len(subrs) < first:
subrs.append(T2CharString(program=["return"], private=own.private, globalSubrs=own.globalSubrs))
for vk in codes:
pen = T2CharStringPen(None, gset)
mini(pen, letters, vk)
program = charstring(pen).program
at = next(i for i, token in enumerate(program) if isinstance(token, str))
assert program[at] in ("rmoveto", "hmoveto", "vmoveto") and program[-1] == "endchar", program[:4]
index[vk] = len(subrs)
subrs.append(T2CharString(program=program[at + 1:-1] + ["return"], private=own.private,
globalSubrs=own.globalSubrs))
clusters = [] # ((the glyph's program, [(where in it a call's number is, its control)]))
for code, cells in CLUSTERS.items():
if code not in cmap or not bounds(cmap[code]):
continue
name, step = cmap[code], MINI + MINI_GAP
box, advance = bounds(name), hmtx[name][0]
cols, rows = max(c for _, c, _ in cells) + 1, max(r for _, _, r in cells) + 1
w, h = cols * step - MINI_GAP, rows * step - MINI_GAP
over = max(0, w - (box[2] - box[0]))
x0, y0 = round((box[0] + box[2] + over) / 2 - w / 2), round((box[1] + box[3]) / 2 - h / 2)
width = advance + round(over)
program, calls, at = [width], [], (0, 0)
pen = T2CharStringPen(width, gset)
for control, col, row in cells:
ox, oy = x0 + col * step, y0 + (rows - 1 - row) * step
if live:
program += [ox + MINI_R - at[0], oy - at[1], "rmoveto"]
calls.append((len(program), control))
program += [index[default[control]] - bias, "callsubr"]
at = (ox + MINI_MARK[-1][0], oy + MINI_MARK[-1][1])
else:
mini(TransformPen(pen, (1, 0, 0, 1, ox, oy)), letters, default[control])
if live:
chars[name] = T2CharString(program=program + ["endchar"], private=own.private, globalSubrs=own.globalSubrs)
clusters.append((program + ["endchar"], calls))
else:
chars[name] = charstring(pen)
hmtx.metrics[name] = (width, hmtx[name][1])
game.setGlyphOrder(order)
gid = {n: i for i, n in enumerate(order)}
mapping = {c: gid[n] for c, n in cmap.items() if c <= 0xFFFF}
for vk in codes:
mapping[BASE + vk] = gid["key_%d" % vk]
single = {c: control for control, c in enumerate(P.SWITCH)}
single.update(MOVES)
for c, control in single.items():
mapping[c] = gid["key_%d" % default[control]]
table = DefaultTable("cmap")
table.data = P.cmap4(mapping, set(single))
game["cmap"] = table
out = io.BytesIO()
game.save(out)
font = out.getvalue()
def compiled(program):
cs = T2CharString(program=list(program))
cs.compile()
return bytes(cs.bytecode)
patches, seen = [], set()
for program, calls in clusters: # (where each call's number is in the font as written: the glyph's bytes, found whole)
code = compiled(program)
if code in seen:
continue
seen.add(code)
found, at = [], font.find(code)
while at >= 0:
found.append(at)
at = font.find(code, at + 1)
if len(found) != sum(1 for p, _ in clusters if compiled(p) == code):
raise SystemExit("prompts: an icon of several keys is in the font %d times" % len(found))
for at in found:
for where, control in calls:
off = at + len(compiled(program[:where] + ["endchar"])) - 1
assert 247 <= font[off] <= 250, font[off]
patches.append((off, control))
calls = {vk: compiled([index[vk] - bias, "callsubr"])[:2] for vk in index}
return font, {"patches": sorted(patches), "calls": calls}
def write_binds(path, binds):
with open(path, "wb") as f:
f.write(b"HPKB" + struct.pack("<I", len(binds["patches"])))
for off, control in binds["patches"]:
f.write(struct.pack("<IB3x", off, control))
f.write(struct.pack("<I", 256) + b"".join(binds["calls"].get(vk, b"\0\0") for vk in range(256)))
def hint_names():
"""the glyph of each key code's icon in game_font's font (None: no such key), for prompt_assets.write_hint"""
return ["key_%d" % vk if vk == 0 or vk in KEYS else None for vk in range(256)]
def main():
import argparse
from PIL import Image, ImageDraw, ImageFont
ap = argparse.ArgumentParser()
ap.add_argument("--out", required=True)
a = ap.parse_args()
font = ImageFont.truetype(io.BytesIO(standalone()), 192)
codes = [BASE + vk for vk in sorted(KEYS)] + [BASE + 0x100 + i for i in range(3)]
sheet = Image.new("L", (12 * 180, (len(codes) + 11) // 12 * 130), 0)
for i, code in enumerate(codes):
ImageDraw.Draw(sheet).text((i % 12 * 180 + 10, i // 12 * 130 + 10), chr(code), font=font, fill=255)
os.makedirs(a.out, exist_ok=True)
sheet.save(os.path.join(a.out, "keys.png"))
print("prompts: keys.png, %d icons" % len(codes), file=sys.stderr)
if __name__ == "__main__":
main()

View File

@@ -194,6 +194,14 @@ def write_image_c(gen, img, meta, exefs):
start_hook, exit_hook = thread_hooks(img, meta)
lines.append("const uint64_t hos_thread_start_hook = %#xULL, hos_thread_exit_hook = %#xULL;"
% (start_hook, exit_hook))
# (Triforce) for the mods' cheat files (host/mod_cheats.c): where the module main begins, which a cheat's addresses
# count from, and each module's code, which a cheat must not write (it is compiled ahead of time)
code = [(m["segments"][0]["addr"], m["segments"][0]["addr"] + m["segments"][0]["size"]) for m in meta["modules"]]
lines += ["const uint64_t hos_main_base = %#xULL; /* (the module main) */"
% next((m["base"] for m in meta["modules"] if m["name"] == "main"), meta["base"]),
"const int hos_code_count = %d;" % len(code),
"const uint64_t hos_code[][2] = {%s}; /* (each module's .text: from, to) */"
% ", ".join("{%#xULL, %#xULL}" % span for span in code)]
text = "\n".join(lines) + "\n"
path = os.path.join(gen, "hos_image.c")
if not os.path.exists(path) or open(path).read() != text:

View File

@@ -68,18 +68,18 @@ mg_spec.loader.exec_module(make_game)
calls = []
reported = [] # (whether each call was given a progress(done, total): the check says how far it is, #1764)
sys.modules["nup"] = type(sys)("nup")
sys.modules["nup"].stand_in_damaged = lambda romfs, progress=None: calls.append(romfs) or reported.append(callable(progress)) or []
sys.modules["nup"].stand_in_damaged = lambda romfs, progress=None, known=None: calls.append(romfs) or reported.append(callable(progress)) or []
with tempfile.TemporaryDirectory() as out:
romfs = os.path.join(out, "romfs")
os.makedirs(romfs)
make_game.stand_in_damaged_files(out, romfs)
check("no stamp: not the builder's romfs", calls, [])
with open(os.path.join(out, ".hookshot_romfs"), "w") as f:
with open(os.path.join(out, ".triforce_romfs"), "w") as f:
f.write("unpacked by this builder: game.nup 1\n")
make_game.stand_in_damaged_files(out, romfs)
make_game.stand_in_damaged_files(out, romfs)
check("checked once", calls, [romfs])
with open(os.path.join(out, ".hookshot_romfs"), "w") as f:
with open(os.path.join(out, ".triforce_romfs"), "w") as f:
f.write("unpacked by this builder: other.nup 2\n")
make_game.stand_in_damaged_files(out, romfs)
check("checked again for another unpack", calls, [romfs, romfs])

View File

@@ -7,6 +7,11 @@ image.bin nor its log's folder and closed with nothing on screen. activeCodePage
every A function and the C runtime's narrow file functions UTF-8 for the whole process, its DLLs too (triforce_vk.dll,
SDL2, FFmpeg). host/hos_main.c says so in a message box when the Windows is too old for it and the folder's name needs it.
(Triforce) longPathAware: where Windows' own LongPathsEnabled is on (Windows 10 1607 and newer, a setting of the PC's), the A and
W functions and the C runtime's file functions take paths of MAX_PATH (260) and more, for the files the game finds in a deep
folder (the game's own nn::fs paths add the long-path prefix themselves, host/nn_fs.c fs_wide; host/hos_paths.h's and gpu/vk/vk_paths.h's
probe too). Without that setting it changes nothing.
python nx2/win_manifest.py [out.res]
"""
import os
@@ -19,6 +24,7 @@ MANIFEST = b"""<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<application xmlns="urn:schemas-microsoft-com:asm.v3">
<windowsSettings>
<activeCodePage xmlns="http://schemas.microsoft.com/SMI/2019/WindowsSettings">UTF-8</activeCodePage>
<longPathAware xmlns="http://schemas.microsoft.com/SMI/2016/WindowsSettings">true</longPathAware>
</windowsSettings>
</application>
<trustInfo xmlns="urn:schemas-microsoft-com:asm.v3">