mirror of
https://git.triforceproject.com/triforce/source
synced 2026-11-20 00:29:57 -04:00
Triforce v0.5.0
In-game mods (GameBanana browser, search and installs, with the Mask Salesman), code-patch and cheat mods, the devs' test18 to test45 work (full performance overlay, frame pacing on, steady frame rate, save export and import, bug capture, new options rows, Setup's Mods page and Add to Steam), occlusion queries and new defaults, Auto resolution that scales, replay helper threads and robust buffers off by default, the options in sections.
This commit is contained in:
234
nx2/build.py
234
nx2/build.py
@@ -50,6 +50,11 @@ import win_manifest
|
||||
NX2_RT = os.path.join(kit.HERE, "rt")
|
||||
HOST = os.path.join(kit.HERE, "..", "host")
|
||||
OPUS = os.path.join(kit.HERE, "..", "third_party", "opus")
|
||||
# (Triforce) zstd's decoder as the runtime compiles it (third_party/zstd/lib; host/host_zstd.h): every file the decoder needs,
|
||||
# and none of the rest. tests/test_host_zstd.py compiles the same list
|
||||
HOST_ZSTD_SOURCES = ("common/debug.c", "common/entropy_common.c", "common/error_private.c", "common/fse_decompress.c",
|
||||
"common/xxhash.c", "common/zstd_common.c", "decompress/huf_decompress.c", "decompress/zstd_ddict.c",
|
||||
"decompress/zstd_decompress.c", "decompress/zstd_decompress_block.c")
|
||||
OBJ_CACHE = os.path.join(kit.WORK, "objcache")
|
||||
# (hookshot) work/hot_order.txt: the functions that run, hottest first (sampled), packed together by the linker; RX_NO_ORDER=1: off
|
||||
PGO = os.environ.get("RX_PGO", "")
|
||||
@@ -244,6 +249,109 @@ def vk_sources():
|
||||
return sorted(out)
|
||||
|
||||
|
||||
def strip_comments(data):
|
||||
"""(Triforce) C or C++ text without its comments, for translation_text, and whether it has a raw string literal. A block
|
||||
comment becomes a space, a line's comment goes with the spaces before it; everything else is kept as it is. A string or
|
||||
character literal ends at its closing quote or at the end of its line (a backslash takes the next character with it, a
|
||||
line end too), a number's ' are digit separators, an #include's <name> and a raw string literal are kept whole: what this
|
||||
reads wrongly it keeps, it never takes code or a literal for a comment"""
|
||||
ident = frozenset(b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_$")
|
||||
out, raw, i, n = bytearray(), False, 0, len(data)
|
||||
while i < n:
|
||||
c = data[i]
|
||||
if data.startswith(b"//", i):
|
||||
j = data.find(b"\n", i)
|
||||
i = n if j < 0 else j
|
||||
while out and out[-1] in b" \t":
|
||||
del out[-1]
|
||||
continue
|
||||
if data.startswith(b"/*", i):
|
||||
j = data.find(b"*/", i + 2)
|
||||
i = n if j < 0 else j + 2
|
||||
out += b" "
|
||||
continue
|
||||
j = i + 1
|
||||
if c in b"\"'":
|
||||
while j < n and data[j] != c and data[j] != 10:
|
||||
j += 2 if data[j] == 92 else 1
|
||||
j = min(j + 1 if j < n and data[j] == c else j, n)
|
||||
elif c == 60 and out[out.rfind(b"\n") + 1:].strip().replace(b" ", b"").replace(b"\t", b"") in (b"#include", b"#include_next",
|
||||
b"#import"):
|
||||
while j < n and data[j] != 62 and data[j] != 10:
|
||||
j += 1
|
||||
j = min(j + 1 if j < n and data[j] == 62 else j, n)
|
||||
elif 48 <= c <= 57 or (c == 46 and i + 1 < n and 48 <= data[i + 1] <= 57): # (a pp-number)
|
||||
while j < n:
|
||||
if data[j] in b"eEpP" and j + 1 < n and data[j + 1] in b"+-":
|
||||
j += 2
|
||||
elif data[j] == 39 and j + 1 < n and data[j + 1] in ident:
|
||||
j += 2
|
||||
elif data[j] in ident or data[j] == 46:
|
||||
j += 1
|
||||
else:
|
||||
break
|
||||
elif c in ident:
|
||||
while j < n and data[j] in ident:
|
||||
j += 1
|
||||
if data[i:j] in (b"R", b"LR", b"uR", b"UR", b"u8R") and j < n and data[j] == 34:
|
||||
k = data.find(b"(", j + 1)
|
||||
delim = data[j + 1:k] if k >= 0 else b"\n"
|
||||
if len(delim) <= 16 and not any(ch in b" ()\\\t\v\f\n" for ch in delim):
|
||||
end = data.find(b")" + delim + b'"', k + 1)
|
||||
j, raw = n if end < 0 else end + len(delim) + 2, True
|
||||
out += data[i:j]
|
||||
i = j
|
||||
return bytes(out), raw
|
||||
|
||||
|
||||
def translation_text(rel, data):
|
||||
"""(Triforce) A file of the translator as the translation stamp takes it (translation_stamp): its line ends as LF, and a C
|
||||
or C++ file without the parts between an "Outside the translation stamp" mark and the next "Inside the translation stamp
|
||||
again" (gpu/vk/vk_shader.cpp's: code that cannot change a translation, gpu/vk/program_carry.h), without its comments, and
|
||||
without its blank lines and the spaces at its lines' ends (but for the line after one a backslash ends, and in a file with
|
||||
a raw string literal). Marks that do not pair: the file whole, and the build says so"""
|
||||
out_mark, in_mark = b"// ---- (Triforce) Outside the translation stamp", b"// ---- (Triforce) Inside the translation stamp again"
|
||||
data = data.replace(bytes((13, 10)), bytes((10,)))
|
||||
if not rel.endswith((".c", ".cc", ".cpp", ".h", ".hpp", ".inc")):
|
||||
return data
|
||||
lines, kept, out, paired = data.split(b"\n"), [], False, True
|
||||
for line in lines:
|
||||
mark = line.strip()
|
||||
if mark.startswith(out_mark) or mark.startswith(in_mark):
|
||||
paired = paired and out != mark.startswith(out_mark)
|
||||
out = mark.startswith(out_mark)
|
||||
elif not out:
|
||||
kept.append(line)
|
||||
if out or not paired:
|
||||
print("build: %s: its translation stamp marks do not pair: the whole file is in the stamp (no program cache is carried "
|
||||
"over this update)" % rel, flush=True)
|
||||
kept = lines
|
||||
data, raw = strip_comments(b"\n".join(kept))
|
||||
if raw:
|
||||
return data
|
||||
lines = []
|
||||
for line in data.split(b"\n"):
|
||||
if lines and lines[-1].endswith(b"\\"):
|
||||
lines.append(line)
|
||||
elif line.rstrip(b" \t"):
|
||||
lines.append(line.rstrip(b" \t"))
|
||||
return b"\n".join(lines)
|
||||
|
||||
|
||||
def translation_stamp(vk, translator):
|
||||
"""(Triforce) The translation stamp (HOOKSHOT_TRANSLATION_STAMP, 16 hex digits), which the program cache is carried over an
|
||||
update by (gpu/vk/program_carry.h has the rule): the files of the translator's stamp (HOOKSHOT_SHADER_STAMP), each as
|
||||
translation_text takes it, but for gpu/vk/vk_api.h, the host's interface, which no translation reads a constant or a type
|
||||
of (tests/test_program_carry.py). translator: the files of eden, sirit and compat"""
|
||||
stamp = hashlib.sha1(usedforsecurity=False)
|
||||
for f in sorted(translator) + [os.path.join(vk, n) for n in ("vk_shader.cpp", "vk_bank.cpp", "vk_state.h")]:
|
||||
rel = os.path.relpath(f, vk).replace("\\", "/")
|
||||
with open(f, "rb") as src:
|
||||
text = translation_text(rel, src.read())
|
||||
stamp.update(rel.encode() + b"\0" + len(text).to_bytes(8, "little") + text)
|
||||
return stamp.hexdigest()[:16]
|
||||
|
||||
|
||||
def cpm_dirs():
|
||||
"""CPM packages: RX_CPM, packaged tools/cpm, then legacy project caches.
|
||||
The packages provide Vulkan and SPIR-V headers, VMA and FFmpeg."""
|
||||
@@ -415,21 +523,61 @@ def rss_mb(proc):
|
||||
return 0
|
||||
|
||||
|
||||
# (a function from this size on is counted by the square of its size: mem_need)
|
||||
GIANT_BYTES = int(1.2 * (1 << 20))
|
||||
GIANT_MB = 300
|
||||
# (the giant functions are counted so only on a PC with less memory free than this when the compile starts: giants_counted)
|
||||
GIANT_BELOW_MB = 14 * 1024
|
||||
_giants = None
|
||||
|
||||
|
||||
def giants_counted():
|
||||
"""Whether mem_need counts giant functions by the square of their size: decided once, when it is first asked. On a PC
|
||||
with GIANT_BELOW_MB or more free it does not. The count is each file's true peak, but the gate holds a file's whole
|
||||
peak back from the moment its compiler starts, and the giants reach theirs only near their end and not together: with
|
||||
21 GB free, counting them in full left 4 to 9 game files compiling at once for the first six minutes where 10 to 13
|
||||
had run without the memory ever running short, and the compile took 657 s for 462. Where memory is short the same
|
||||
files are what ran a PC out of it, so there they are counted. HOOKSHOT_MEM_GIANT=1 or 0 decides it outright."""
|
||||
global _giants
|
||||
if _giants is None:
|
||||
asked = os.environ.get("HOOKSHOT_MEM_GIANT")
|
||||
if asked in ("0", "1"):
|
||||
_giants = asked == "1"
|
||||
else:
|
||||
free = free_mb()
|
||||
_giants = free is not None and free < GIANT_BELOW_MB
|
||||
print("the memory gate: giant functions %s (%s)" % (
|
||||
"counted by the square of their size" if _giants else "counted as the other functions",
|
||||
"HOOKSHOT_MEM_GIANT" if asked in ("0", "1") else "less than %d GB free" % (GIANT_BELOW_MB >> 10) if _giants
|
||||
else "%d GB or more free, or the free memory not known" % (GIANT_BELOW_MB >> 10)), flush=True)
|
||||
return _giants
|
||||
|
||||
|
||||
def mem_need(src):
|
||||
"""What compiling a file takes at its peak, in MB, a little over. Measured on a cold build (clang 23, x86-64): a chunk of lifted
|
||||
code 0.4 GB + 50 MB per MB of C + 250 MB per MB of its largest function (1.5 GB for 24 MB of small functions, 2.2 GB for a 6 MB
|
||||
function alone); a hot piece 0.8-1.3 GB whatever its size (-O2, the profile, inlining); the runtime's C files ~0.2 GB, the C++
|
||||
files up to ~1 GB"""
|
||||
files up to ~1 GB.
|
||||
(Triforce, measured again with HOOKSHOT_MEM_LOG on a build of v0.4.0, clang 23.1.5) The few files with giant functions took
|
||||
far more than that: 6.2 GB for a chunk that is one 5 MB function (counted 1.9), 3.7 and 3.1 GB for two others (counted 1.4
|
||||
and 1.5), and 2.5 to 2.7 GB for the three hot pieces that hold functions near 2 MB (counted 1.5). The compiler's memory
|
||||
grows with the square of a function's size, as its time does (nx2/chunks.py QUAD), so every function of GIANT_BYTES or
|
||||
more now adds GIANT_MB for each MB squared, in a chunk and in a hot piece: of the 76 game files measured none is then
|
||||
under by more than 250 MB, which the gate's reserve covers. Counted too low, such a file was started beside the others
|
||||
on a PC without the memory for it, where Windows pages and Linux stops the compiler. Only on a PC short of memory:
|
||||
giants_counted."""
|
||||
name = os.path.basename(src)
|
||||
if name.startswith("code_hot"):
|
||||
return COMPILER_MB
|
||||
if name.startswith("code_"):
|
||||
with open(src, "rb") as f:
|
||||
text = f.read()
|
||||
# (each function from the newline before its definition, as this was first measured)
|
||||
starts = [m.start() - 1 for m in lifted_c.HEAD_BYTES.finditer(text)] + [len(text)]
|
||||
largest = max([b - a for a, b in zip(starts, starts[1:])] or [0])
|
||||
return int(400 + 50 * len(text) / (1 << 20) + 250 * largest / (1 << 20))
|
||||
sizes = [b - a for a, b in zip(starts, starts[1:])]
|
||||
giant = GIANT_MB * sum((s / (1 << 20)) ** 2 for s in sizes if s >= GIANT_BYTES) if giants_counted() else 0
|
||||
if name.startswith("code_hot"):
|
||||
return int(COMPILER_MB + giant)
|
||||
largest = max(sizes or [0])
|
||||
return int(max(400 + 50 * len(text) / (1 << 20) + 250 * largest / (1 << 20), 300 + 50 * len(text) / (1 << 20) + giant))
|
||||
return 1000 if src.endswith(".cpp") else 400
|
||||
|
||||
|
||||
@@ -702,6 +850,19 @@ def compile_cost(src, costs):
|
||||
return costs.get(name, os.path.getsize(src)) * (HOT_COST if name.startswith("code_hot") else 1)
|
||||
|
||||
|
||||
# HOOKSHOT_MEM_LOG=1: each compile's peak memory beside what mem_need counted it at ("@@mem <file> <peak MB> <counted MB>
|
||||
# <seconds>"), sampled twice a second over the compiler and the processes it starts. For fitting mem_need and the gate to
|
||||
# what the compilers really take: off unless asked for, and nothing is sampled then
|
||||
MEM_LOG = os.environ.get("HOOKSHOT_MEM_LOG", "0") not in ("", "0")
|
||||
|
||||
|
||||
def sample_peak(proc, peak):
|
||||
"""(a thread of its own, beside communicate) peak[0]: the most MB the compiler's process tree was seen holding"""
|
||||
while proc.poll() is None:
|
||||
peak[0] = max(peak[0], rss_mb(proc))
|
||||
time.sleep(0.5)
|
||||
|
||||
|
||||
def compile_one(job):
|
||||
cc, src, obj, fl, deps = job
|
||||
key = PLANNED.get((src, obj)) or object_key(src, fl, deps)
|
||||
@@ -721,15 +882,20 @@ def compile_one(job):
|
||||
return src, 0.0, None
|
||||
lease = GATE.enter(src) if GATE else None
|
||||
t = time.time()
|
||||
peak = [0]
|
||||
try:
|
||||
with subprocess.Popen([cc] + fl + ["-c", src, "-o", obj], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) as p:
|
||||
no_throttle(getattr(p, "_handle", None))
|
||||
if GATE:
|
||||
GATE.started(lease, p)
|
||||
if MEM_LOG:
|
||||
threading.Thread(target=sample_peak, args=(p, peak), daemon=True).start()
|
||||
out, err = p.communicate()
|
||||
finally:
|
||||
if GATE:
|
||||
GATE.leave(lease)
|
||||
if MEM_LOG:
|
||||
print("@@mem %s %d %d %.1f" % (os.path.basename(src), peak[0], mem_need(src), time.time() - t), flush=True)
|
||||
r = subprocess.CompletedProcess(p.args, p.returncode, out, err)
|
||||
if r.returncode:
|
||||
failure = r.stderr or r.stdout or "Compiler exited with status %d without diagnostics." % r.returncode
|
||||
@@ -739,6 +905,15 @@ def compile_one(job):
|
||||
return src, time.time() - t, None
|
||||
|
||||
|
||||
# RX_EARLY=1 (nx2/make_game.py, while the game is being translated): only the files that need nothing of the translated
|
||||
# game are compiled, and nothing is linked: libopus, the WebGPU renderer and the Vulkan back end with its libraries, ~520 of
|
||||
# the build's ~630 files and a ninth of its compile time. The lift was one process on Windows and left every other
|
||||
# processor idle for two minutes (its emit has workers there now, nx2/emit_pool.py: half the processors at most); the build that follows finds these objects made (the same keys: their flags and
|
||||
# dependencies hold nothing the lift writes) and compiles the game's own files alone. The lifted files are not read here:
|
||||
# the lift is writing them
|
||||
EARLY = os.environ.get("RX_EARLY") == "1"
|
||||
|
||||
|
||||
def main():
|
||||
kit.wait_for_quiet('the compile')
|
||||
p = argparse.ArgumentParser()
|
||||
@@ -759,7 +934,7 @@ def main():
|
||||
obj_dir = os.path.join(out, "obj")
|
||||
gen_dir = os.path.join(out, "gen") # (the logo and the fonts the Vulkan back end compiles in: made with it, below)
|
||||
os.makedirs(obj_dir, exist_ok=True)
|
||||
if os.environ.get("RX_VK_ONLY") or a.dry_run: # (the Vulkan back end needs no game: built so with no lift, as the AppImage's is)
|
||||
if os.environ.get("RX_VK_ONLY") or a.dry_run or EARLY: # (the Vulkan back end needs no game: built so with no lift, as the AppImage's is)
|
||||
os.makedirs(gen, exist_ok=True)
|
||||
rt64 = rtgen.write(os.path.join(out, "rt64"))
|
||||
cc = toolchain.need(toolchain.find("clang"))
|
||||
@@ -769,15 +944,17 @@ def main():
|
||||
stamp_def = version.define()
|
||||
print("version: %s" % version.describe(), flush=True)
|
||||
# (the lifted chunks include only the guest model: the emulator ABI header is a dependency of the shim alone)
|
||||
deps = b"".join(open(os.path.join(d, f), "rb").read() for d in (gen, rt64) for f in sorted(os.listdir(d))
|
||||
if f.endswith(".h"))
|
||||
# (RX_EARLY: none: the lift is writing those headers, and no file compiled early depends on them)
|
||||
deps = b"" if EARLY else b"".join(open(os.path.join(d, f), "rb").read() for d in (gen, rt64) for f in sorted(os.listdir(d))
|
||||
if f.endswith(".h"))
|
||||
shim_deps = deps + open(os.path.join(NX2_RT, "mkw_host.h"), "rb").read()
|
||||
eden_gen, eden_out = os.path.join(kit.WORK, "gen"), os.path.join(kit.WORK, "build")
|
||||
migrate_stamps(eden_gen, os.path.join(eden_out, "obj"),
|
||||
flags(eden_gen, os.path.join(eden_out, "rt64"), a.opt, not a.no_polls) +
|
||||
(["-DA64_POLL_CTX"] if a.dll else []), deps) # (as fl below: the flags its objects were built with)
|
||||
migrate_stamps(gen, obj_dir, fl, deps)
|
||||
chunks = sorted(f for f in os.listdir(gen) if f.startswith("code_") and f.endswith(".c"))
|
||||
if not EARLY:
|
||||
migrate_stamps(eden_gen, os.path.join(eden_out, "obj"),
|
||||
flags(eden_gen, os.path.join(eden_out, "rt64"), a.opt, not a.no_polls) +
|
||||
(["-DA64_POLL_CTX"] if a.dll else []), deps) # (as fl below: the flags its objects were built with)
|
||||
migrate_stamps(gen, obj_dir, fl, deps)
|
||||
chunks = [] if EARLY else sorted(f for f in os.listdir(gen) if f.startswith("code_") and f.endswith(".c"))
|
||||
if a.only:
|
||||
want = {"code_%03d.c" % int(n) for n in a.only.split(",")}
|
||||
chunks = [c for c in chunks if c in want]
|
||||
@@ -799,6 +976,8 @@ def main():
|
||||
srcs += [os.path.join(NX2_RT, "eden_shim.c"), write_info(out)]
|
||||
if os.environ.get("RX_VK_ONLY"): # (no lift needed: the back end alone)
|
||||
srcs = [s for s in srcs if os.path.exists(s)]
|
||||
if EARLY: # (the lift's files are on their way: none of them now)
|
||||
srcs = []
|
||||
if a.dry_run: # (the lift's files: there is no game here)
|
||||
srcs = [s for s in srcs if os.path.dirname(s) != gen]
|
||||
# the longest first, so the long ones do not end up alone at the end (compile_cost)
|
||||
@@ -864,6 +1043,21 @@ def main():
|
||||
for rel in open(os.path.join(OPUS, "sources.txt")).read().split():
|
||||
jobs.append((cc, os.path.join(OPUS, rel), os.path.join(obj_dir, "opus_" + rel.replace("/", "_")[:-2] + ".o"),
|
||||
oflags, b"opus-1.5.2"))
|
||||
# (Triforce) zstd's decoder for the runtime too (third_party/zstd: the same files the Vulkan back end compiles into its
|
||||
# DLL for the game's fonts, further down, where their names stay inside it). host/hos_zstd.c decodes the game's files
|
||||
# with it when HOS_HOST_ZSTD is set (host/host_zstd.h). Its names are hidden (zstd.h marks its functions visible by
|
||||
# itself), so that they are never among the names the program exports: on Linux those come first for every library
|
||||
# loaded into it, and a GPU driver that uses the system's libzstd would be handed this decoder for half of its calls
|
||||
if host_srcs:
|
||||
hzstd = os.path.normpath(os.path.join(kit.HERE, "..", "third_party", "zstd", "lib"))
|
||||
hzflags = target() + CPU + ["-O2", "-g0", "-DNDEBUG", "-Wno-everything", "-DZSTD_DISABLE_ASM", "-DZSTD_LEGACY_SUPPORT=0",
|
||||
"-DZSTD_MULTITHREAD=0", "-fvisibility=hidden", "-DZSTDLIB_VISIBLE=", "-DZSTDERRORLIB_VISIBLE="]
|
||||
hzdeps = b"".join(open(os.path.join(d, f), "rb").read()
|
||||
for d in (hzstd, os.path.join(hzstd, "common"), os.path.join(hzstd, "decompress"))
|
||||
for f in sorted(os.listdir(d)) if f.endswith(".h"))
|
||||
for f in HOST_ZSTD_SOURCES:
|
||||
jobs.append((cc, os.path.join(hzstd, *f.split("/")), os.path.join(obj_dir, "hostzstd_" + os.path.basename(f)[:-2] + ".o"),
|
||||
hzflags, hzdeps))
|
||||
# the renderer (C++20, NDEBUG: no asserts in it or suyu), when a wgpu-native release is there
|
||||
wgpu = a.standalone and not a.only and not LINUX and os.path.isfile(os.path.join(WGPU, "include", "webgpu", "webgpu.h"))
|
||||
if a.standalone and not a.only and not wgpu and not LINUX:
|
||||
@@ -917,6 +1111,8 @@ def main():
|
||||
with open(f, "rb") as src:
|
||||
stamp.update(os.path.relpath(f, vk).replace("\\", "/").encode() + src.read().replace(bytes((13, 10)), bytes((10,))))
|
||||
shader_stamp = "-DHOOKSHOT_SHADER_STAMP=0x%sull" % stamp.hexdigest()[:16]
|
||||
# ((Triforce) and what of those translates: the program cache is carried over an update by it, translation_stamp)
|
||||
translation_def = "-DHOOKSHOT_TRANSLATION_STAMP=0x%sull" % translation_stamp(vk, translator)
|
||||
vflags = target() + CPU + (["-fPIC", "-I" + POSIX, "-D_GNU_SOURCE"] if LINUX else []) + [ # (hookshot: the replay is the frame's limit now)
|
||||
# (what the code takes for granted, which newer C++ libraries bring in by themselves and older ones do not: issue 48)
|
||||
"-include", "algorithm", "-include", "cstdio", "-include", "cstdlib", "-include", "cstring", "-include", "functional",
|
||||
@@ -976,7 +1172,7 @@ def main():
|
||||
"-DIMGUI_DISABLE_WIN32_FUNCTIONS", "-DIMGUI_DISABLE_DEMO_WINDOWS", "-DIMGUI_DISABLE_DEBUG_TOOLS"]
|
||||
imgui_deps = b"".join(open(os.path.join(d, f), "rb").read() for d in (imgui, os.path.join(imgui, "backends"), zstd)
|
||||
for f in sorted(os.listdir(d)) if f.endswith(".h"))
|
||||
imgui_users = ("vk_imgui.cpp", "vk_menu.cpp")
|
||||
imgui_users = ("vk_imgui.cpp", "vk_menu.cpp", "vk_salesman.cpp") # (vk_salesman.cpp: the game's files through vk_game_font.h)
|
||||
for f in ("imgui.cpp", "imgui_draw.cpp", "imgui_tables.cpp", "imgui_widgets.cpp", "backends/imgui_impl_vulkan.cpp"):
|
||||
vk_jobs.append((cxx, os.path.join(imgui, *f.split("/")),
|
||||
os.path.join(obj_dir, "vk_imgui_" + os.path.basename(f)[:-4] + ".o"), vflags + imgui_flags, imgui_deps))
|
||||
@@ -997,7 +1193,7 @@ def main():
|
||||
name = os.path.relpath(src, vk).replace(os.sep, "_")[:-4]
|
||||
lib = os.path.relpath(src, vk).split(os.sep)[0] in ("eden", "sirit")
|
||||
vk_jobs.append((cxx, src, os.path.join(obj_dir, "vk_" + name + ".o"),
|
||||
vflags + ([shader_stamp] if os.path.basename(src) == "vk_shader.cpp" else []) +
|
||||
vflags + ([shader_stamp, translation_def] if os.path.basename(src) == "vk_shader.cpp" else []) +
|
||||
([stamp_def] if os.path.basename(src) == "vk_menu.cpp" else []) +
|
||||
(ngx_flags if os.path.basename(src) == "vk_dlss.cpp" else []) +
|
||||
(sl_flags if os.path.basename(src) == "vk_dlssg.cpp" else []) +
|
||||
@@ -1024,6 +1220,8 @@ def main():
|
||||
vk_only = bool(os.environ.get("RX_VK_ONLY")) # (the renderer alone, the game's objects as they are)
|
||||
if vk_only:
|
||||
jobs = list(vk_jobs)
|
||||
if EARLY: # (by their objects' names: libopus, the WebGPU renderer, the Vulkan back end; never the runtime's host_*.o)
|
||||
jobs = [j for j in jobs if os.path.basename(j[2]).startswith(("opus_", "gpu_", "vk_"))]
|
||||
if a.dry_run: # (the compile starts here: what it would read, said and checked, and nothing compiled)
|
||||
missing = sorted({j[1] for j in jobs if not os.path.isfile(j[1])} |
|
||||
({vk_prebuilt} if vk_prebuilt and not os.path.isfile(vk_prebuilt) else set()))
|
||||
@@ -1087,6 +1285,8 @@ def main():
|
||||
print("compiled in %.0fs, %d failed" % (time.time() - t0, still), flush=True)
|
||||
if still:
|
||||
return 1
|
||||
if EARLY: # (nothing to link yet: the game's own objects come with the build after the lift)
|
||||
return 0
|
||||
vk_objs = {j[2] for j in vk_jobs}
|
||||
objs = [j[2] for j in jobs if j[2] not in vk_objs]
|
||||
rsp = os.path.join(out, "objs.rsp")
|
||||
@@ -1171,9 +1371,9 @@ def main():
|
||||
"-fuse-ld=lld", "-shared", "-o", vdll, "-Wl,-Map=" + vdll + ".map", "-Wl,-Bsymbolic", "-Wl,--gc-sections",
|
||||
"-static-libstdc++", "-static-libgcc", "@" + vrsp] + \
|
||||
([ngx_library(NGX_SDK, True)] if NGX_SDK else []) + ["-ldl", "-lpthread", "-lm"]
|
||||
else:
|
||||
else: # (Triforce: advapi32 for RivaTuner Statistics Server's folder in the registry, gpu/vk/rtss_sys_win.cpp)
|
||||
link = [cc] + toolchain.native_target() + toolchain.native_link() + \
|
||||
["-shared", "-o", vdll, toolchain.link_opt("/map:" + vdll[:-4] + ".map"), "@" + vrsp, "-luser32"] + (["-lc++", "-lunwind"] if toolchain.mingw() else []) + ngx_link(NGX_SDK, toolchain.mingw())
|
||||
["-shared", "-o", vdll, toolchain.link_opt("/map:" + vdll[:-4] + ".map"), "@" + vrsp, "-luser32", "-ladvapi32"] + (["-lc++", "-lunwind"] if toolchain.mingw() else []) + ngx_link(NGX_SDK, toolchain.mingw())
|
||||
r = subprocess.run(link, capture_output=True, text=True)
|
||||
if r.returncode:
|
||||
undefined = sorted({l.strip() for l in (r.stdout + r.stderr).splitlines() if "undefined symbol" in l})
|
||||
|
||||
264
nx2/chunks.py
264
nx2/chunks.py
@@ -13,6 +13,11 @@ The functions' C is made by several processes (fork, so Linux and macOS; one pro
|
||||
consecutive functions from the lifter as it stands, and write() takes the texts in its own order, as if it had made
|
||||
them. The lifter's counters (stats, unhandled, ijump, and any passed in `counters`, such as poll.stats) get each
|
||||
process's additions. The output is the one-process output, byte for byte (tests/test_parallel_emit.py).
|
||||
|
||||
(Triforce) Where there is no fork (Windows) the processes are spawned and sent what they need (nx2/emit_pool.py), and the same
|
||||
processes then cut the big chunks (split_big) and share the page marks (edit_c), a file each: 46 s, 6 s and 19 s of the lift's
|
||||
127 s in one process on a 16-thread PC, 13 s, 1.4 s and 5 s of its 69 s with 8 processes. HOOKSHOT_LIFT_JOBS=1 is the one
|
||||
process, as before, on every system.
|
||||
"""
|
||||
import bisect
|
||||
import collections
|
||||
@@ -20,9 +25,11 @@ import glob
|
||||
import multiprocessing
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
import whole_write # (rewrites written beside and moved over: issue 1197)
|
||||
|
||||
import a64_lift
|
||||
import emit_pool # ((Triforce) the emit's processes where there is no fork)
|
||||
import lifted_c
|
||||
|
||||
FILE = "chunks.txt"
|
||||
@@ -101,7 +108,8 @@ def _emit_run(es):
|
||||
|
||||
|
||||
def _available_gb():
|
||||
"""MemAvailable from /proc/meminfo in GB (None where there is none)"""
|
||||
"""MemAvailable from /proc/meminfo in GB; (Triforce) on Windows the physical memory GlobalMemoryStatusEx calls available
|
||||
(None where there is neither)"""
|
||||
try:
|
||||
with open("/proc/meminfo", encoding="ascii") as f:
|
||||
for line in f:
|
||||
@@ -109,22 +117,51 @@ def _available_gb():
|
||||
return int(line.split()[1]) / (1024 * 1024)
|
||||
except (OSError, ValueError, IndexError):
|
||||
pass
|
||||
if os.name == "nt":
|
||||
try:
|
||||
import ctypes
|
||||
|
||||
class Status(ctypes.Structure):
|
||||
_fields_ = [("dwLength", ctypes.c_ulong), ("dwMemoryLoad", ctypes.c_ulong)] + [
|
||||
(name, ctypes.c_ulonglong) for name in ("ullTotalPhys", "ullAvailPhys", "ullTotalPageFile", "ullAvailPageFile",
|
||||
"ullTotalVirtual", "ullAvailVirtual", "ullAvailExtendedVirtual")]
|
||||
s = Status()
|
||||
s.dwLength = ctypes.sizeof(s)
|
||||
if ctypes.WinDLL("kernel32").GlobalMemoryStatusEx(ctypes.byref(s)):
|
||||
return s.ullAvailPhys / (1 << 30)
|
||||
except (OSError, AttributeError, ValueError):
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
FORK = "fork" in multiprocessing.get_all_start_methods()
|
||||
SPAWN_MOST = 8 # ((Triforce) spawned emit processes at most: the lift's own process cannot keep more busy, emit_pool.py)
|
||||
SPAWN_GB = 1 # (and one per GB of available memory: one holds its run or a file of lifted C, 480 MB at the most seen)
|
||||
SPAWN_LEAST = 2 * emit_pool.RUN_FUNCS # (functions below which the lift stays one process: starting the others costs more)
|
||||
|
||||
|
||||
def emit_jobs():
|
||||
"""Processes for the emit: HOOKSHOT_LIFT_JOBS, else the cores this process may use (at most 16; each holds its run's
|
||||
texts and the pages of the lift it touches), and at most one per 2 GB of available memory: a Steam Deck (16 GB) ran
|
||||
out with 8, and the system's out-of-memory handling stopped the lift ("lift.py stopped (signal 15): BrokenPipeError")
|
||||
or one worker (the update then sat at "jump tables" for good). 1 where there is no fork"""
|
||||
if "fork" not in multiprocessing.get_all_start_methods():
|
||||
return 1
|
||||
or one worker (the update then sat at "jump tables" for good).
|
||||
(Triforce) Where there is no fork (Windows; before, always 1 there): half the processors, at most SPAWN_MOST, and at most
|
||||
one per SPAWN_GB of available memory. Half, because the files that need no translated code are compiled beside the lift
|
||||
there (make_game's EarlyCompile, one compiler a processor less one) and because the lift's own process, which sends the
|
||||
runs and writes the C, is what more of them would wait for (emit_pool.py has the measure). 1 is the one process on
|
||||
every system"""
|
||||
if os.environ.get("HOOKSHOT_LIFT_JOBS"):
|
||||
return max(1, int(os.environ["HOOKSHOT_LIFT_JOBS"]))
|
||||
try:
|
||||
n = len(os.sched_getaffinity(0))
|
||||
except AttributeError:
|
||||
n = os.cpu_count() or 1
|
||||
if not FORK:
|
||||
n = max(1, min(SPAWN_MOST, n // 2))
|
||||
gb = _available_gb()
|
||||
if gb is not None:
|
||||
n = max(1, min(n, int(gb // SPAWN_GB)))
|
||||
return n
|
||||
n = max(1, min(16, n))
|
||||
gb = _available_gb()
|
||||
if gb is not None:
|
||||
@@ -169,23 +206,65 @@ def parallel_emit(emit, order, texts, counters, jobs, run=2000):
|
||||
yield (e,) + tuple(emit(e, texts))
|
||||
|
||||
|
||||
def write(L, gen_dir, img_dir, rechunk=False, counters=(), jobs=None):
|
||||
_workers = None # ((Triforce) the spawned processes of this lift, from write() until edit_c()'s marks end, emit_pool.Workers)
|
||||
_marks = None # (the page marks in flight after edit_c(wait=False): the thread waiting for them, what ended it, the folder)
|
||||
|
||||
|
||||
def stop_workers(kill=False):
|
||||
"""(Triforce) The spawned processes ended, when there are any: after what they are doing, or (kill) now"""
|
||||
global _workers
|
||||
w, _workers = _workers, None
|
||||
if w is not None:
|
||||
w.stop(kill)
|
||||
|
||||
|
||||
def drop_lifted(gen_dir):
|
||||
"""(Triforce) The lifted C taken away after a lift that stopped while its processes were writing it: the chunks, their
|
||||
parts, the hot file and its pieces, whole or cut short. What is left of such a lift looks like a lift (every file that
|
||||
is there may end as a function does, and an earlier lift's a64_table.c still names every function), and the next lift
|
||||
would take its chunk boundaries from it (seed_from_gen) and a build its files. The same files make_game takes away
|
||||
before a lift"""
|
||||
for f in os.listdir(gen_dir) if os.path.isdir(gen_dir) else ():
|
||||
if f.startswith("code_") and f.endswith((".c", ".c.whole", ".part")):
|
||||
try:
|
||||
os.remove(os.path.join(gen_dir, f))
|
||||
except OSError:
|
||||
pass # (held by something else: the lift's own failure is what is said)
|
||||
|
||||
|
||||
def gave_up(gen_dir):
|
||||
"""(Triforce) A lift with spawned processes that failed: they are ended now and its C goes"""
|
||||
stop_workers(kill=True)
|
||||
drop_lifted(gen_dir)
|
||||
|
||||
|
||||
def write(L, gen_dir, img_dir, rechunk=False, counters=(), jobs=None, setup=()):
|
||||
"""L.write(gen_dir) with stable boundaries; returns its files. counters: dicts the emit adds to besides the
|
||||
lifter's own (poll.stats); jobs: emit processes (emit_jobs())"""
|
||||
lifter's own (poll.stats); jobs: emit processes (emit_jobs()); (Triforce) setup: what set the lifter's class up for
|
||||
the emit in this process (poll.install), module-level functions the spawned processes call before they emit"""
|
||||
global _workers
|
||||
starts = [] if rechunk else load(img_dir)
|
||||
if not starts and not rechunk:
|
||||
starts = seed_from_gen(gen_dir)
|
||||
b = Boundaries(a64_lift.CHUNK_BYTES, starts, L.funcs)
|
||||
base_emit = L.emit_function
|
||||
jobs = emit_jobs() if jobs is None else jobs
|
||||
spawned = jobs > 1 and not FORK and len(L.funcs) >= SPAWN_LEAST
|
||||
made = None
|
||||
|
||||
def emit_function(e, texts):
|
||||
nonlocal made
|
||||
global _workers
|
||||
b.current = e
|
||||
if made is None: # (the first call: write() has its texts now, and asks for the functions in sorted order)
|
||||
made = parallel_emit(base_emit, sorted(L.funcs), texts,
|
||||
[L.stats, L.unhandled, L.ijump] + list(counters), jobs)
|
||||
every = [L.stats, L.unhandled, L.ijump] + list(counters)
|
||||
if spawned:
|
||||
stop_workers(kill=True) # (an earlier write()'s, which no edit_c() ended)
|
||||
_workers = emit_pool.Workers(jobs, L, every, setup)
|
||||
print("emit: %d processes (HOOKSHOT_LIFT_JOBS=<n> for another count, 1 for one process)" % _workers.jobs, flush=True)
|
||||
made = emit_pool.emit(_workers, L, sorted(L.funcs), texts, every)
|
||||
else:
|
||||
made = parallel_emit(base_emit, sorted(L.funcs), texts, every, jobs)
|
||||
got = next(made, None)
|
||||
if got is None or got[0] != e:
|
||||
raise SystemExit("chunks: write() asked for f_%08x, the emit made %s" % (e, "f_%08x" % got[0] if got else "nothing"))
|
||||
@@ -193,6 +272,10 @@ def write(L, gen_dir, img_dir, rechunk=False, counters=(), jobs=None):
|
||||
L.emit_function = emit_function
|
||||
try:
|
||||
files = L.write(gen_dir, chunk_bytes=b)
|
||||
except BaseException:
|
||||
if spawned: # ((Triforce) a process failed, died or hung, or this one was stopped: no part of a lift is left behind)
|
||||
gave_up(gen_dir)
|
||||
raise
|
||||
finally:
|
||||
del L.emit_function
|
||||
if made is not None:
|
||||
@@ -210,7 +293,12 @@ def write(L, gen_dir, img_dir, rechunk=False, counters=(), jobs=None):
|
||||
for f in os.listdir(gen_dir):
|
||||
if f == "code_hot.c.whole" or HOT_PIECE.fullmatch(f):
|
||||
os.remove(os.path.join(gen_dir, f))
|
||||
files += split_big(gen_dir, files)
|
||||
try:
|
||||
files += split_big(gen_dir, files, workers=_workers if spawned else None)
|
||||
except BaseException:
|
||||
if spawned:
|
||||
gave_up(gen_dir)
|
||||
raise
|
||||
save(img_dir, starts or b.recorded)
|
||||
print("chunks: %d, boundaries %s" % (len(files), "kept" if starts else "from sizes (saved)"))
|
||||
return files
|
||||
@@ -251,43 +339,57 @@ def write_costs(gen_dir, costs):
|
||||
f.write("".join("%s %d\n" % kv for kv in sorted(known.items())))
|
||||
|
||||
|
||||
def split_big(gen_dir, files, budget=None):
|
||||
def split_one(gen_dir, path, budget):
|
||||
"""split_big for one chunk (a spawned process's share of it too): the new parts' paths, {file name: cost} of the chunk or of
|
||||
each of its parts, and what to say of a chunk that was cut (None of one left whole)"""
|
||||
m = re.fullmatch(r"code_(\d{3})\.c", os.path.basename(path))
|
||||
made, costs = [], {}
|
||||
with open(path, encoding="utf-8", newline="") as f:
|
||||
text = f.read()
|
||||
marks = [h.start() for h in HEAD.finditer(text)]
|
||||
ends = marks[1:] + [len(text)]
|
||||
cost = [fn_cost(b - a) for a, b in zip(marks, ends)]
|
||||
total = sum(cost)
|
||||
if total <= budget * 3 // 2 or len(marks) < 2:
|
||||
costs[os.path.basename(path)] = marks[0] + total if marks else len(text)
|
||||
return made, costs, None
|
||||
parts = -(-total // budget)
|
||||
cuts, run, k = [0], 0, 1 # (indexes into marks where a part starts)
|
||||
for i, c in enumerate(cost):
|
||||
if i and run >= total * k // parts:
|
||||
cuts.append(i)
|
||||
k = run * parts // total + 1 # (past a giant function: the next share, not a sliver after it)
|
||||
run += c
|
||||
bounds = [marks[i] for i in cuts] + [len(text)]
|
||||
preamble = text[:marks[0]]
|
||||
for j in range(len(cuts)):
|
||||
out = path if j == 0 else os.path.join(gen_dir, "code_%s_%d.c" % (m.group(1), j))
|
||||
piece = text[:bounds[1]] if j == 0 else preamble + text[bounds[j]:bounds[j + 1]]
|
||||
whole_write.write_whole(out, piece)
|
||||
costs[os.path.basename(out)] = len(preamble) + sum(cost[cuts[j]:cuts[j + 1] if j + 1 < len(cuts) else len(cost)])
|
||||
if j:
|
||||
made.append(out)
|
||||
return made, costs, "chunks: %s (%d MB, cost %d MB) in %d parts" % (os.path.basename(path), len(text) >> 20, total >> 20, len(cuts))
|
||||
|
||||
|
||||
def split_big(gen_dir, files, budget=None, workers=None):
|
||||
"""Chunks whose estimated compile time (fn_cost) is well past a plain chunk's, cut at function starts into parts of about equal
|
||||
cost: code_NNN.c, code_NNN_1.c ... A chunk of the giant functions took 350 s, alone at the end of the build, where a chunk of
|
||||
ordinary functions takes 45; the boundaries kept in chunks.txt also let chunks grow (44 MB beside 25). The first part keeps the
|
||||
name, so seed_from_gen and the boundaries are as before. Writes chunk_costs.txt for every chunk; returns the new parts' paths"""
|
||||
name, so seed_from_gen and the boundaries are as before. Writes chunk_costs.txt for every chunk; returns the new parts' paths.
|
||||
(Triforce) workers: the spawned processes that take a chunk each (emit_pool.Workers; None: this process takes them all)"""
|
||||
budget = budget or a64_lift.CHUNK_BYTES
|
||||
made, costs = [], {}
|
||||
for path in files:
|
||||
m = re.fullmatch(r"code_(\d{3})\.c", os.path.basename(path))
|
||||
if not m:
|
||||
continue
|
||||
with open(path, encoding="utf-8", newline="") as f:
|
||||
text = f.read()
|
||||
marks = [h.start() for h in HEAD.finditer(text)]
|
||||
ends = marks[1:] + [len(text)]
|
||||
cost = [fn_cost(b - a) for a, b in zip(marks, ends)]
|
||||
total = sum(cost)
|
||||
if total <= budget * 3 // 2 or len(marks) < 2:
|
||||
costs[os.path.basename(path)] = marks[0] + total if marks else len(text)
|
||||
continue
|
||||
parts = -(-total // budget)
|
||||
cuts, run, k = [0], 0, 1 # (indexes into marks where a part starts)
|
||||
for i, c in enumerate(cost):
|
||||
if i and run >= total * k // parts:
|
||||
cuts.append(i)
|
||||
k = run * parts // total + 1 # (past a giant function: the next share, not a sliver after it)
|
||||
run += c
|
||||
bounds = [marks[i] for i in cuts] + [len(text)]
|
||||
preamble = text[:marks[0]]
|
||||
for j in range(len(cuts)):
|
||||
out = path if j == 0 else os.path.join(gen_dir, "code_%s_%d.c" % (m.group(1), j))
|
||||
piece = text[:bounds[1]] if j == 0 else preamble + text[bounds[j]:bounds[j + 1]]
|
||||
whole_write.write_whole(out, piece)
|
||||
costs[os.path.basename(out)] = len(preamble) + sum(cost[cuts[j]:cuts[j + 1] if j + 1 < len(cuts) else len(cost)])
|
||||
if j:
|
||||
made.append(out)
|
||||
print("chunks: %s (%d MB, cost %d MB) in %d parts" % (os.path.basename(path), len(text) >> 20, total >> 20, len(cuts)))
|
||||
paths = [path for path in files if re.fullmatch(r"code_(\d{3})\.c", os.path.basename(path))]
|
||||
if workers is None:
|
||||
answers = (split_one(gen_dir, path, budget) for path in paths)
|
||||
else:
|
||||
answers = workers.each(split_one, (("the split of %s" % os.path.basename(path), (gen_dir, path, budget)) for path in paths))
|
||||
for new, cost, said in answers: # (in the chunks' order, whoever cut them)
|
||||
made += new
|
||||
costs.update(cost)
|
||||
if said:
|
||||
print(said)
|
||||
for f in os.listdir(gen_dir): # (parts of an earlier lift that this one did not make)
|
||||
if PART.fullmatch(f) and os.path.join(gen_dir, f) not in made:
|
||||
os.remove(os.path.join(gen_dir, f))
|
||||
@@ -295,10 +397,82 @@ def split_big(gen_dir, files, budget=None):
|
||||
return made
|
||||
|
||||
|
||||
def edit_c(gen, stats, supported, whole=True):
|
||||
def marks_one(path):
|
||||
"""(Triforce) a64_lift.group_marks for one file, in a spawned process: whether it changed the file, and its counts"""
|
||||
st = collections.Counter()
|
||||
return a64_lift.group_marks([path], st), list(st.items())
|
||||
|
||||
|
||||
def ask_marks(paths, workers):
|
||||
"""(Triforce) a64_lift.group_marks asked of the spawned processes, a file each, the largest first (the hot file, 82 MB,
|
||||
is the last by name and would end alone). Every file is asked for here, at once: a lift that ends on an error of its
|
||||
own while they are at it asks for nothing more, and the processes finish the files, all of them. What take_marks() takes"""
|
||||
by_size = sorted(paths, key=lambda p: (-os.path.getsize(p), p))
|
||||
return by_size, workers.ask(marks_one, [("the page marks of %s" % os.path.basename(p), (p,)) for p in by_size])
|
||||
|
||||
|
||||
def take_marks(paths, stats, workers, asked):
|
||||
"""(Triforce) Waits for what ask_marks() asked: the counts go into stats in the paths' order, as one process adds them
|
||||
when it groups the files itself. The files changed"""
|
||||
by_size, calls = asked
|
||||
answers = dict(zip(by_size, workers.answers(calls)))
|
||||
changed = 0
|
||||
for p in paths:
|
||||
n, counts = answers[p]
|
||||
changed += n
|
||||
for k, v in counts:
|
||||
stats[k] = stats.get(k, 0) + v
|
||||
return changed
|
||||
|
||||
|
||||
def edit_c(gen, stats, supported, whole=True, wait=True):
|
||||
"""(nx2/lift.py, after the lift writes its C; here, beside the writing, so it imports without the lift's tools) The edits of the written C, in this order: the game's patches (nx2/oot_patches.py, which find the lift's exact text; the
|
||||
supported build's whole standalone lift fails when an output-size edit finds no place), then, last, the page marks of each
|
||||
group of stores shared (a64_lift.group_marks), which changes that text. stats: the lift's, for its report"""
|
||||
group of stores shared (a64_lift.group_marks), which changes that text. stats: the lift's, for its report.
|
||||
(Triforce) The marks are made by write()'s spawned processes when it left any, which end with them. wait=False: this
|
||||
returns while they are at it, and marks_done() waits for them: the lift writes its last files meanwhile (hos_image.c takes
|
||||
as long as the marks, and reads nothing of the C); stats is theirs until then"""
|
||||
global _marks
|
||||
import oot_patches
|
||||
oot_patches.apply(gen, protect_hearts=supported, required=supported and whole)
|
||||
a64_lift.group_marks(sorted(glob.glob(os.path.join(glob.escape(gen), "code_*.c"))), stats)
|
||||
try:
|
||||
oot_patches.apply(gen, protect_hearts=supported, required=supported and whole)
|
||||
except BaseException:
|
||||
stop_workers(kill=True) # (the patches' own failure, said as before: the C stays as it is, for whoever looks into it)
|
||||
raise
|
||||
paths = sorted(glob.glob(os.path.join(glob.escape(gen), "code_*.c")))
|
||||
workers = _workers
|
||||
if workers is None:
|
||||
a64_lift.group_marks(paths, stats)
|
||||
return
|
||||
try:
|
||||
asked = ask_marks(paths, workers)
|
||||
except BaseException:
|
||||
gave_up(gen)
|
||||
raise
|
||||
ended = []
|
||||
|
||||
def marks(): # (this thread only waits for the processes and adds their counts up)
|
||||
try:
|
||||
take_marks(paths, stats, workers, asked)
|
||||
except BaseException as e:
|
||||
ended.append(e)
|
||||
thread = threading.Thread(target=marks)
|
||||
thread.start()
|
||||
_marks = (thread, ended, gen)
|
||||
if wait:
|
||||
marks_done()
|
||||
|
||||
|
||||
def marks_done():
|
||||
"""(Triforce) After edit_c(wait=False): waits for the page marks the spawned processes are making, and ends the
|
||||
processes. Nothing to do when edit_c made the marks itself (one process, or wait)"""
|
||||
global _marks
|
||||
m, _marks = _marks, None
|
||||
if m is None:
|
||||
return
|
||||
thread, ended, gen = m
|
||||
thread.join()
|
||||
if ended: # (a process failed in the middle of the files: some marked, some not, one perhaps cut short)
|
||||
gave_up(gen)
|
||||
raise ended[0]
|
||||
stop_workers()
|
||||
|
||||
480
nx2/code_mods.py
Normal file
480
nx2/code_mods.py
Normal file
@@ -0,0 +1,480 @@
|
||||
"""(Triforce) Code mods: the mods' patches of the game's code, applied before the game is translated.
|
||||
|
||||
The game's code is compiled ahead of time, so a patch of it cannot be applied while the game runs (host/mods.c loads the
|
||||
RomFS mods then). A build applies them instead: nx2/make_game.py calls build_step() on its own copy of the game's code
|
||||
(<work>/exefs, which the lift reads), never on the player's files and never on the copy the game folder keeps for updates
|
||||
(.triforce_exefs). The patched bytes are in the modules the lift key hashes (make_game.py lift_key), so a patch added,
|
||||
taken away, switched off or edited makes the next build translate the game again, and nothing else does.
|
||||
|
||||
Where the patches are, as emulators keep them and as host/mods.c finds its mods:
|
||||
|
||||
<game folder>/mods/<name>/exefs/<build id>.ips IPS ("PATCH" ... "EOF") or IPS32 ("IPS32" ... "EEOF")
|
||||
<game folder>/mods/<name>/exefs/<any name>.pchtxt IPSwitch text, its build id in the @nsobid- line
|
||||
|
||||
A mod may sit deeper (mods/contents/<title id>/<name>/exefs): any folder up to four levels down that holds a romfs, exefs
|
||||
or cheats folder is a mod, and nothing below it is looked at. A folder whose name starts with '-' or '_' is off, one that
|
||||
starts with '.' is skipped. Mods are taken in the order of their names (ASCII letters' case ignored), a mod's patch files
|
||||
in the order of theirs, and where two write the same byte the later one wins; the build says so.
|
||||
|
||||
A patch names the build id of the module it is for (main, a subsdk, sdk or rtld): the .ips file's name, the .pchtxt
|
||||
file's @nsobid- line, hex, trailing zeros left out or not. One for another build is refused with a line that says so.
|
||||
Offsets are as emulators and Atmosphere read them: they count the NSO's 0x100-byte header before the module's image (a
|
||||
.pchtxt offset after its "@flag offset_shift" is added). A file is applied whole or not at all: one that is cut short,
|
||||
that cannot be read, or that writes outside the module's three segments (.text, .rodata, .data) is refused. That
|
||||
leaves out code caves, patches that put new code into the empty space after .text: the translator has not been tried on
|
||||
them.
|
||||
|
||||
What a build did is kept in the game folder, triforce_code_mods.txt (RECORD, write_record): the next build, an update
|
||||
(--update) among them, reads the mods folder again, builds the same patches in again and says which of the last build's
|
||||
are gone or changed; the game compares the file with the mods folder at its start (host/mods.c, host/mod_code.h) and
|
||||
says in its log which patch files wait for a build.
|
||||
|
||||
Standard library only (lz4 too for a module that is LZ4-compressed, as the translator's own loader needs it). Not
|
||||
imported by the lift: this file is not part of the lift key (make_game.py LIFT_KEY_NOT), what it does to the code is.
|
||||
"""
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import struct
|
||||
|
||||
NSO_HEADER = 0x100
|
||||
SEGMENTS = (".text", ".rodata", ".data")
|
||||
PATCH_SUFFIXES = (".ips", ".pchtxt")
|
||||
PARTS = ("romfs", "exefs", "cheats") # (a folder holding one of these is a mod: host/mods.c find_mods)
|
||||
MOD_DEPTH = 4
|
||||
RECORD = "triforce_code_mods.txt"
|
||||
RECORD_HEAD = ("# The code patches (.ips, .pchtxt) this game was built with: nx2/code_mods.py writes it at every build, and\n"
|
||||
"# the game reads it at its start. Tab separated: applied or refused, the file's FNV-1a 64 and size, its\n"
|
||||
"# path under the mods folder, then the module and the bytes changed, or why it was refused.\n")
|
||||
|
||||
|
||||
class Refused(ValueError):
|
||||
"""A patch file that is not applied; the text says why"""
|
||||
|
||||
|
||||
# ---- the formats: [(offset in the module's image, bytes)] in the file's order
|
||||
|
||||
def parse_ips(data):
|
||||
"""The writes of an IPS or IPS32 file. Refused when it is neither, or cut short"""
|
||||
if data[:5] == b"PATCH":
|
||||
width, end = 3, b"EOF"
|
||||
elif data[:5] == b"IPS32":
|
||||
width, end = 4, b"EEOF"
|
||||
else:
|
||||
raise Refused("it is not an IPS file (no PATCH or IPS32 at its start)")
|
||||
writes, at = [], 5
|
||||
while True:
|
||||
# (the end's letters could be a record's offset too: they are the end only where the file ends, a truncation
|
||||
# length of up to three bytes after them)
|
||||
if data[at:at + len(end)] == end and len(data) - at - len(end) <= 3:
|
||||
return writes
|
||||
if at + width + 2 > len(data):
|
||||
raise Refused("it is cut short (no %s at its end)" % end.decode())
|
||||
offset = int.from_bytes(data[at:at + width], "big")
|
||||
size = int.from_bytes(data[at + width:at + width + 2], "big")
|
||||
at += width + 2
|
||||
if size:
|
||||
body = data[at:at + size]
|
||||
at += size
|
||||
if len(body) != size:
|
||||
raise Refused("it is cut short (a record of %d bytes at file offset %#x has %d)" % (size, at - size, len(body)))
|
||||
else: # (a run: its length, then the one byte it repeats)
|
||||
if at + 3 > len(data):
|
||||
raise Refused("it is cut short (in a run at file offset %#x)" % at)
|
||||
body = data[at + 2:at + 3] * int.from_bytes(data[at:at + 2], "big")
|
||||
at += 3
|
||||
if offset < NSO_HEADER:
|
||||
raise Refused("it writes the module's header (offset %#x: an offset counts the 0x100-byte NSO header)" % offset)
|
||||
if body:
|
||||
writes.append((offset - NSO_HEADER, bytes(body)))
|
||||
|
||||
|
||||
ESCAPES = {"a": 7, "b": 8, "f": 12, "n": 10, "r": 13, "t": 9, "v": 11, "\\": 92, "'": 39, '"': 34, "?": 63, "0": 0}
|
||||
PATCH_LINE = re.compile(r"([0-9A-Fa-f]{8})[ \t]+(\S.*)$")
|
||||
|
||||
|
||||
def pchtxt_value(text, little_endian, where):
|
||||
"""The bytes of a patch line's value: hex digits in byte order (reversed after @little-endian), or a string in double
|
||||
quotes with C's one-letter escapes, as UTF-8 and without a zero after it"""
|
||||
if text.startswith('"'):
|
||||
out, i = bytearray(), 1
|
||||
while i < len(text) and text[i] != '"':
|
||||
ch = text[i]
|
||||
if ch == "\\":
|
||||
i += 1
|
||||
if i >= len(text) or text[i] not in ESCAPES:
|
||||
raise Refused("%s: an escape it does not know in a string" % where)
|
||||
out.append(ESCAPES[text[i]])
|
||||
else:
|
||||
out += ch.encode("utf-8")
|
||||
i += 1
|
||||
if i >= len(text):
|
||||
raise Refused("%s: a string without its closing quote" % where)
|
||||
return bytes(out)
|
||||
digits = re.split(r"[ \t/]", text, maxsplit=1)[0]
|
||||
if len(digits) % 2 or not re.fullmatch(r"[0-9A-Fa-f]+", digits):
|
||||
raise Refused("%s: %r is not a value (hex digits, two a byte)" % (where, digits[:40]))
|
||||
value = bytes.fromhex(digits)
|
||||
return value[::-1] if little_endian else value
|
||||
|
||||
|
||||
def parse_pchtxt(text):
|
||||
"""(build id or None, writes) of an IPSwitch file. Patch lines count between @enabled and @disabled, @stop ends the
|
||||
file; a line that is neither a directive nor "<8 hex digits> <value>" is a title or a note and is passed over. A
|
||||
patch line whose value cannot be read refuses the file"""
|
||||
build, shift, enabled, little, writes = None, 0, False, False, []
|
||||
for number, raw in enumerate(text.splitlines(), 1):
|
||||
line = raw.strip()
|
||||
if not line or line.startswith("//") or line.startswith("#"):
|
||||
continue
|
||||
if line.startswith("@"):
|
||||
word = line.split()[0].lower()
|
||||
if word == "@stop":
|
||||
break
|
||||
if word.startswith("@nsobid-"):
|
||||
build = line.split()[0][len("@nsobid-"):]
|
||||
elif word == "@enabled":
|
||||
enabled = True
|
||||
elif word == "@disabled":
|
||||
enabled = False
|
||||
elif word == "@little-endian":
|
||||
little = True
|
||||
elif word == "@big-endian":
|
||||
little = False
|
||||
elif word == "@flag":
|
||||
f = line.split()
|
||||
if len(f) >= 3 and f[1] == "offset_shift":
|
||||
try:
|
||||
shift = int(f[2], 0)
|
||||
except ValueError:
|
||||
raise Refused("line %d: %r is not a number for offset_shift" % (number, f[2])) from None
|
||||
continue
|
||||
m = PATCH_LINE.match(line)
|
||||
if not m or not enabled:
|
||||
continue
|
||||
value = pchtxt_value(m.group(2), little, "line %d" % number)
|
||||
offset = int(m.group(1), 16) + shift
|
||||
if offset < NSO_HEADER:
|
||||
raise Refused("line %d: it writes the module's header (offset %#x after the shift: an offset counts the "
|
||||
"0x100-byte NSO header)" % (number, offset))
|
||||
if value:
|
||||
writes.append((offset - NSO_HEADER, value))
|
||||
return build, writes
|
||||
|
||||
|
||||
def same_build(named, build_id):
|
||||
"""Whether a patch's build id (hex, any case, trailing zeros left out or not) is a module's (hex of its 32 bytes)"""
|
||||
named = named.lower()
|
||||
return bool(re.fullmatch(r"[0-9a-f]{1,64}", named)) and named.ljust(64, "0") == build_id.lower().ljust(64, "0")
|
||||
|
||||
|
||||
# ---- the mods folder, as host/mods.c walks it
|
||||
|
||||
def name_key(name):
|
||||
"""A name as host/mods.c orders it (_stricmp: bytes, ASCII letters' case ignored)"""
|
||||
return bytes(b + 32 if 65 <= b <= 90 else b for b in name.encode("utf-8", "surrogateescape"))
|
||||
|
||||
|
||||
def find_mods(root):
|
||||
"""[(name, folder)] of the mods under root, by name: a mod's name is its path under root, with /"""
|
||||
out = []
|
||||
|
||||
def walk(folder, name, depth):
|
||||
if depth > 0 and any(os.path.exists(os.path.join(folder, part)) for part in PARTS):
|
||||
out.append((name, folder))
|
||||
return
|
||||
if depth >= MOD_DEPTH:
|
||||
return
|
||||
try:
|
||||
entries = sorted(os.listdir(folder))
|
||||
except OSError:
|
||||
return
|
||||
for entry in entries:
|
||||
sub = os.path.join(folder, entry)
|
||||
if entry[:1] in (".", "-", "_") or not os.path.isdir(sub):
|
||||
continue
|
||||
walk(sub, name + "/" + entry if name else entry, depth + 1)
|
||||
|
||||
if os.path.isdir(root):
|
||||
walk(root, "", 0)
|
||||
return sorted(out, key=lambda mod: name_key(mod[0]))
|
||||
|
||||
|
||||
def patch_files(root):
|
||||
"""[(path under the mods folder, file)] of the mods' code patches, in the order they are applied"""
|
||||
out = []
|
||||
for name, folder in find_mods(root):
|
||||
exefs = os.path.join(folder, "exefs")
|
||||
try:
|
||||
entries = sorted(os.listdir(exefs), key=name_key) if os.path.isdir(exefs) else []
|
||||
except OSError:
|
||||
entries = []
|
||||
for entry in entries:
|
||||
path = os.path.join(exefs, entry)
|
||||
if entry.lower().endswith(PATCH_SUFFIXES) and os.path.isfile(path):
|
||||
out.append(("%s/exefs/%s" % (name, entry), path))
|
||||
return out
|
||||
|
||||
|
||||
def fnv64(data):
|
||||
"""FNV-1a 64 of a file's bytes, 16 hex digits: what the record names a patch file's contents by (host/mod_code.h
|
||||
makes the same number)"""
|
||||
h = 0xCBF29CE484222325
|
||||
for b in data:
|
||||
h = ((h ^ b) * 0x100000001B3) & 0xFFFFFFFFFFFFFFFF
|
||||
return "%016x" % h
|
||||
|
||||
|
||||
# ---- the modules of the copy that is translated
|
||||
|
||||
class Module:
|
||||
"""One NSO of the exefs: its build id and its three segments; the image only once a patch is for it"""
|
||||
|
||||
def __init__(self, path):
|
||||
self.path, self.name = path, os.path.basename(path)
|
||||
with open(path, "rb") as f:
|
||||
self.head = f.read(NSO_HEADER)
|
||||
if len(self.head) < NSO_HEADER or self.head[:4] != b"NSO0":
|
||||
raise ValueError("%s is not a game module" % path)
|
||||
self.build_id = self.head[0x40:0x60].hex()
|
||||
self.flags = struct.unpack_from("<I", self.head, 0xC)[0]
|
||||
# (each: file offset, memory offset, size)
|
||||
self.segments = [struct.unpack_from("<3I", self.head, 0x10 + 0x10 * i) for i in range(3)]
|
||||
self.file_sizes = struct.unpack_from("<3I", self.head, 0x60)
|
||||
self.raw = self.image = self.original = None
|
||||
|
||||
def load(self):
|
||||
if self.image is not None:
|
||||
return
|
||||
if self.flags & 0x80:
|
||||
raise ValueError("%s is still ZBIC-compressed (nx2/unpack.py makes it plain first)" % self.path)
|
||||
with open(self.path, "rb") as f:
|
||||
self.raw = f.read()
|
||||
self.image = bytearray(max(memory + size for _, memory, size in self.segments))
|
||||
for i, ((offset, memory, size), stored) in enumerate(zip(self.segments, self.file_sizes)):
|
||||
data = self.raw[offset:offset + stored]
|
||||
if self.flags & (1 << i):
|
||||
import lz4.block # (as third_party/a64-kit/tools/nso.py reads it)
|
||||
data = lz4.block.decompress(data, uncompressed_size=size)
|
||||
if len(data) != size:
|
||||
raise ValueError("%s: its %s has %d bytes, %d expected" % (self.path, SEGMENTS[i], len(data), size))
|
||||
self.image[memory:memory + size] = data
|
||||
self.original = bytes(self.image)
|
||||
|
||||
def segment_of(self, offset, size):
|
||||
"""The segment that holds all of [offset, offset + size), or None"""
|
||||
for i, (_, memory, length) in enumerate(self.segments):
|
||||
if memory <= offset and offset + size <= memory + length:
|
||||
return i
|
||||
return None
|
||||
|
||||
def write(self):
|
||||
"""The module written again as a plain NSO with the image as it is now: the header, what lies between it and
|
||||
the first segment (the module's name), the three segments one after the other, their hashes made again"""
|
||||
head = bytearray(self.head)
|
||||
body = bytearray(self.raw[NSO_HEADER:max(NSO_HEADER, min(offset for offset, _, _ in self.segments))])
|
||||
for i, (_, memory, size) in enumerate(self.segments):
|
||||
data = bytes(self.image[memory:memory + size])
|
||||
struct.pack_into("<I", head, 0x10 + 0x10 * i, NSO_HEADER + len(body))
|
||||
struct.pack_into("<I", head, 0x60 + 4 * i, size)
|
||||
head[0xA0 + 0x20 * i:0xC0 + 0x20 * i] = hashlib.sha256(data).digest()
|
||||
body += data
|
||||
struct.pack_into("<I", head, 0xC, self.flags & ~7) # (no segment compressed; the hash bits stay)
|
||||
tmp = self.path + ".part" # (make_game.py check_modules passes over a .part)
|
||||
with open(tmp, "wb") as f:
|
||||
f.write(head)
|
||||
f.write(body)
|
||||
os.replace(tmp, self.path)
|
||||
|
||||
|
||||
def load_modules(exefs):
|
||||
"""The exefs's modules by name: every file of it that is an NSO (main.npdm is none)"""
|
||||
out = {}
|
||||
for name in sorted(os.listdir(exefs)):
|
||||
path = os.path.join(exefs, name)
|
||||
if os.path.isfile(path) and not name.endswith(".part"):
|
||||
with open(path, "rb") as f:
|
||||
magic = f.read(4)
|
||||
if magic == b"NSO0":
|
||||
out[name] = Module(path)
|
||||
return out
|
||||
|
||||
|
||||
# ---- applying
|
||||
|
||||
def read_patch(rel, data):
|
||||
"""(the build id the file names, its writes). Refused when the file cannot be read as what its name says it is"""
|
||||
if rel.lower().endswith(".ips"):
|
||||
named = os.path.splitext(os.path.basename(rel))[0]
|
||||
if not re.fullmatch(r"[0-9A-Fa-f]{1,64}", named):
|
||||
raise Refused("its name is not a build id (an .ips patch is named after the build id of the code it is for)")
|
||||
return named, parse_ips(data)
|
||||
build, writes = parse_pchtxt(data.decode("utf-8-sig", errors="replace"))
|
||||
if build is None:
|
||||
raise Refused("it has no @nsobid- line (the build id of the code it is for)")
|
||||
if not re.fullmatch(r"[0-9A-Fa-f]{1,64}", build):
|
||||
raise Refused("its @nsobid- line is not a build id: %r" % build[:70])
|
||||
return build, writes
|
||||
|
||||
|
||||
def check_writes(module, writes):
|
||||
"""Refused when a write is not whole inside one of the module's segments"""
|
||||
for offset, data in writes:
|
||||
if module.segment_of(offset, len(data)) is None:
|
||||
end = max(memory + size for _, memory, size in module.segments)
|
||||
where = "past the end of its image (%#x)" % end if offset + len(data) > end else \
|
||||
"outside its .text, .rodata and .data (the empty space between them: a code cave is not supported)"
|
||||
raise Refused("it writes %d bytes at %s+%#x, %s" % (len(data), module.name, offset, where))
|
||||
|
||||
|
||||
def apply(exefs, mods_root, say=print):
|
||||
"""The mods' code patches applied to the modules in exefs, in place. Returns the record's entries: a dict per patch
|
||||
file, in the order they were taken: state ("applied" or "refused"), path (under the mods folder), fnv, size, and
|
||||
module and changed (the bytes it changed that no later patch wrote over) or why. Says a line per file"""
|
||||
files = patch_files(mods_root)
|
||||
if not files:
|
||||
return []
|
||||
modules = load_modules(exefs)
|
||||
game = ", ".join("%s %s" % (m.name, m.build_id.rstrip("0")[:40] or "0") for m in modules.values())
|
||||
entries, owner = [], {} # (owner: {(module, offset): the entry whose byte is there})
|
||||
for rel, path in files:
|
||||
entry = {"state": "refused", "path": rel, "fnv": "0" * 16, "size": 0, "module": "", "changed": 0, "why": ""}
|
||||
entries.append(entry)
|
||||
try:
|
||||
try:
|
||||
with open(path, "rb") as f:
|
||||
data = f.read()
|
||||
except OSError as e:
|
||||
raise Refused("it cannot be read (%s)" % (e.strerror or e)) from None
|
||||
entry["fnv"], entry["size"] = fnv64(data), len(data)
|
||||
named, writes = read_patch(rel, data)
|
||||
module = next((m for m in modules.values() if same_build(named, m.build_id)), None)
|
||||
if module is None:
|
||||
raise Refused("it is for build %s, which is not this game's code (%s)" % (named.lower()[:40], game))
|
||||
try:
|
||||
module.load()
|
||||
except (OSError, ValueError) as e:
|
||||
raise Refused("the module it is for cannot be read (%s)" % e) from None
|
||||
check_writes(module, writes)
|
||||
except Refused as e:
|
||||
entry["why"] = str(e)
|
||||
say("code mods: %s is not applied: %s" % (rel, e))
|
||||
continue
|
||||
entry["state"], entry["module"] = "applied", module.name
|
||||
over = {}
|
||||
for offset, data in writes:
|
||||
for i, b in enumerate(data):
|
||||
at = (module.name, offset + i)
|
||||
before = owner.get(at)
|
||||
if before is not None and before is not entry and module.image[offset + i] != b:
|
||||
over[before["path"]] = over.get(before["path"], 0) + 1
|
||||
module.image[offset + i] = b
|
||||
owner[at] = entry
|
||||
for earlier, count in over.items():
|
||||
say("code mods: %s writes over %d bytes of %s (it comes later by name: its bytes are the ones built in)"
|
||||
% (rel, count, earlier))
|
||||
for (name, offset), entry in owner.items(): # (what each changed in the end: a byte is the last patch's that wrote it)
|
||||
if modules[name].image[offset] != modules[name].original[offset]:
|
||||
entry["changed"] += 1
|
||||
for entry in entries:
|
||||
if entry["state"] == "applied":
|
||||
say("code mods: %s applied to %s: %d bytes changed%s" % (
|
||||
entry["path"], entry["module"], entry["changed"],
|
||||
"" if entry["changed"] else " (it writes what the game has there already, or a later patch wrote over it)"))
|
||||
written = []
|
||||
for module in modules.values():
|
||||
if module.image is not None and bytes(module.image) != module.original:
|
||||
module.write()
|
||||
written.append(module.name)
|
||||
if written:
|
||||
say("code mods: the game is translated with %s patched. If the translation stops or the game misbehaves, switch the "
|
||||
"mod off (a - before its folder's name) and build again." % ", ".join(written))
|
||||
return entries
|
||||
|
||||
|
||||
# ---- what the game folder remembers
|
||||
|
||||
def record_text(entries):
|
||||
lines = []
|
||||
for e in entries:
|
||||
tail = (e["module"], str(e["changed"])) if e["state"] == "applied" else (e["why"].replace("\t", " ").replace("\n", " "),)
|
||||
path = e["path"].replace("\t", "?").replace("\n", "?").replace("\r", "?") # (a line a path, whatever the name)
|
||||
lines.append("\t".join((e["state"], e["fnv"], str(e["size"]), path) + tail))
|
||||
return RECORD_HEAD + "".join(line + "\n" for line in lines)
|
||||
|
||||
|
||||
def read_record(out):
|
||||
"""The entries of the game folder's record, [] when it has none (a folder built before code mods, or with none)"""
|
||||
entries = []
|
||||
try:
|
||||
with open(os.path.join(out, RECORD), encoding="utf-8", errors="replace") as f:
|
||||
for line in f:
|
||||
f_ = line.rstrip("\r\n").split("\t")
|
||||
if line.startswith("#") or len(f_) < 5 or f_[0] not in ("applied", "refused"):
|
||||
continue
|
||||
entry = {"state": f_[0], "fnv": f_[1], "path": f_[3], "module": "", "changed": 0, "why": ""}
|
||||
try:
|
||||
entry["size"] = int(f_[2])
|
||||
if f_[0] == "applied":
|
||||
entry["module"], entry["changed"] = f_[4], int(f_[5]) if len(f_) > 5 else 0
|
||||
else:
|
||||
entry["why"] = f_[4]
|
||||
except ValueError:
|
||||
continue
|
||||
entries.append(entry)
|
||||
except OSError:
|
||||
pass
|
||||
return entries
|
||||
|
||||
|
||||
def write_record(out, entries):
|
||||
"""The record of a finished build, written whole beside it and put in its place; with no patch files there is no
|
||||
record (a game folder without code mods is as it was before them)"""
|
||||
path = os.path.join(out, RECORD)
|
||||
if not entries:
|
||||
try:
|
||||
os.remove(path)
|
||||
except OSError:
|
||||
pass
|
||||
return
|
||||
with open(path + ".new", "w", encoding="utf-8", newline="\n") as f:
|
||||
f.write(record_text(entries))
|
||||
os.replace(path + ".new", path)
|
||||
|
||||
|
||||
def since_last(before, now):
|
||||
"""Lines about what differs from the build before, for a build that takes the folder's patches again (an update):
|
||||
the patches of that build that are gone, changed or no longer applied (a new one has its own line already)"""
|
||||
lines = []
|
||||
was = {e["path"]: e for e in before if e["state"] == "applied"}
|
||||
have = {e["path"]: e for e in now}
|
||||
for path, old in was.items():
|
||||
new = have.get(path)
|
||||
if new is None:
|
||||
lines.append("code mods: %s was built into this game folder before and is gone from its mods folder (or "
|
||||
"switched off): this build is made without it" % path)
|
||||
elif new["state"] != "applied":
|
||||
lines.append("code mods: %s was built into this game folder before and is refused now: this build is made "
|
||||
"without it" % path)
|
||||
elif (new["fnv"], new["size"]) != (old["fnv"], old["size"]):
|
||||
lines.append("code mods: %s changed since the build before: this build has it as it is now" % path)
|
||||
return lines
|
||||
|
||||
|
||||
def mods_folder(out):
|
||||
"""The mods folder a build reads: the game folder's, or HOS_MODS=<folder> as the game itself takes it"""
|
||||
return os.environ.get("HOS_MODS") or os.path.join(out, "mods")
|
||||
|
||||
|
||||
def build_step(exefs, out, say=print):
|
||||
"""nx2/make_game.py, once its copy of the game's code is there and is the supported build: the patches of the game
|
||||
folder's mods applied to that copy, with a line for each and for what changed since the folder's last build. Returns
|
||||
the entries for write_record, which make_game.py calls when the build is done"""
|
||||
before = read_record(out)
|
||||
entries = apply(exefs, mods_folder(out), say)
|
||||
for line in since_last(before, entries):
|
||||
say(line)
|
||||
applied = [e for e in entries if e["state"] == "applied"]
|
||||
if entries:
|
||||
say("code mods: %d of %d patch files applied (%s)" % (len(applied), len(entries), mods_folder(out)))
|
||||
return entries
|
||||
308
nx2/emit_pool.py
Normal file
308
nx2/emit_pool.py
Normal file
@@ -0,0 +1,308 @@
|
||||
"""(Triforce) The lift's emit in several processes where there is no fork (Windows).
|
||||
|
||||
nx2/chunks.py's forked workers inherit the lifter as it stands: 5 GB of decoded functions, tables and texts, shared for nothing.
|
||||
A spawned process starts empty, and handing it that state whole costs more than it saves (425 MB of functions and 180 MB of texts
|
||||
pickled, read again and held by every worker). So the lifter stays in the lift's own process, and each worker is sent what it
|
||||
needs and no more:
|
||||
|
||||
- once, when it starts: every attribute of the lifter but NOT_SENT (the entries, the PLT stubs, the jump tables, the names, the
|
||||
patches after instructions: 4 MB pickled), the functions that set the lifter's class up as this process did (poll.install),
|
||||
and where the counters live;
|
||||
- with each run of consecutive functions: those functions' bodies as the discovery left them and the texts of their
|
||||
instructions as write() translated them (about 3 MB and 3.5 MB a run: 148 runs of 2000 functions were 425 and 513 MB).
|
||||
|
||||
A worker emits its run with the lifter's own emit_function and sends back the texts, the names each calls and what the run added
|
||||
to the counters. The lift's process takes the runs in order and hands write() the functions one by one, as if it had made them:
|
||||
the C is the one-process C, byte for byte (tests/test_spawn_emit.py on authored code; the game's own lift was compared file by
|
||||
file). Nothing a worker computes is taken from anywhere but these two messages, so discovery is done once and is not assumed to
|
||||
repeat itself.
|
||||
|
||||
Measured on a 16-thread PC (the game: 294,523 functions, 17.6 million instructions, 185 runs): write()'s emit loop takes 46 s in
|
||||
one process, 37 s of it in emit_function, and 13 s with 8 processes (21 s with 3, on a PC that was not idle). What is left is
|
||||
the lift's own process: about 7 s to send (3 s to pickle the bodies, 3 s to pick each run's texts, 1 s to pickle them) and what
|
||||
write() itself needs for a gigabyte of C. More processes than keep up with it only wait for it, so emit_jobs (chunks.py) asks
|
||||
for half the processors, at most 8, and the files compiled beside the lift keep the rest. A worker held 230 to 480 MB at its
|
||||
most.
|
||||
|
||||
A worker that raises, dies or does not answer ends the lift with what it said (SystemExit, never a wait forever), and the
|
||||
workers still running are ended. A worker whose parent is gone ends by itself: Windows leaves the children of a stopped process
|
||||
running, and one of them could be writing a lifted file into the next build's folder.
|
||||
"""
|
||||
import collections
|
||||
import concurrent.futures
|
||||
import gc
|
||||
import importlib
|
||||
import multiprocessing
|
||||
import os
|
||||
import pickle
|
||||
import sys
|
||||
import threading
|
||||
import traceback
|
||||
from concurrent.futures.process import BrokenProcessPool
|
||||
|
||||
import a64_lift
|
||||
|
||||
# (the lifter's attributes a worker is not sent: the image and the functions (each run brings its own), write()'s stand-in for
|
||||
# emit_function, which exists only in the lift's process, and what only the discovery reads)
|
||||
NOT_SENT = ("img", "funcs", "emit_function", "eh_starts", "_preds")
|
||||
RUN_FUNCS = 2000 # (a run: at most this many functions,
|
||||
RUN_INSNS = 120000 # and no more once it holds this many instructions: 2000 functions are 120,000 on average, a few are 20,000 each)
|
||||
AHEAD = 2 # (runs asked for beyond one a worker: none waits while the lift's process writes a chunk)
|
||||
WAIT = 600 # (seconds without an answer to a run, a split or a file's marks, which take a second: the worker is taken for hung)
|
||||
SLICE = 5.0
|
||||
MOST = 60 # (Windows waits on at most 64 handles at once, and the pool keeps a few of its own)
|
||||
# (a process that is gone: what it printed as it went, a Python error at its start for one, is in the lift's output above this)
|
||||
GONE = "%s: a process of the lift stopped before it answered (out of memory, ended from outside, or it could not start): %s"
|
||||
|
||||
|
||||
def plan(order, size, most_funcs=None, most_size=None):
|
||||
"""order cut into runs of consecutive functions: a run ends at most_funcs functions, or with the function that brings it to
|
||||
most_size (size(e): a function's instructions), so a run of giant functions is no longer than a run of small ones. Every
|
||||
function is in exactly one run and the runs, joined, are order"""
|
||||
most_funcs, most_size = most_funcs or RUN_FUNCS, most_size or RUN_INSNS
|
||||
runs, cur, held = [], [], 0
|
||||
for e in order:
|
||||
cur.append(e)
|
||||
held += size(e)
|
||||
if len(cur) >= most_funcs or held >= most_size:
|
||||
runs.append(cur)
|
||||
cur, held = [], 0
|
||||
if cur:
|
||||
runs.append(cur)
|
||||
return runs
|
||||
|
||||
|
||||
def texts_of(funcs, es, texts):
|
||||
"""The instruction texts the emit of the functions es reads, out of all of them: a64_lift's emit_function, dead_flags and
|
||||
dirty_regs look a plain instruction's text up by its word, or by the word and its address where the text depends on the
|
||||
address (pc_relative), and no other kind's"""
|
||||
relative = a64_lift.pc_relative
|
||||
need = {}
|
||||
for e in es:
|
||||
for pc, (k, w) in funcs[e][0].items():
|
||||
if k[0] == "op":
|
||||
key = (w, pc) if relative(w) else w
|
||||
if key not in need:
|
||||
need[key] = texts[key]
|
||||
return need
|
||||
|
||||
|
||||
def emit_print(cls):
|
||||
"""What the class's emit_function is made of: its own name and, through the closures, the names of the functions it wraps
|
||||
(poll.install keeps the one before it in its closure). A worker whose class has another emit_function than the lift's
|
||||
process (a wrapper installed there and not named in `setup`) would write other C, without a word: it compares this first"""
|
||||
out, todo, seen = [], [getattr(cls, "emit_function", None)], set()
|
||||
while todo:
|
||||
f = todo.pop()
|
||||
f = getattr(f, "__func__", f)
|
||||
if f is None or id(f) in seen:
|
||||
continue
|
||||
seen.add(id(f))
|
||||
module = getattr(f, "__module__", None) or "?"
|
||||
out.append("%s.%s" % ("__main__" if module == "__mp_main__" else module, getattr(f, "__qualname__", "?")))
|
||||
for cell in getattr(f, "__closure__", None) or ():
|
||||
try:
|
||||
v = cell.cell_contents
|
||||
except ValueError:
|
||||
continue
|
||||
if hasattr(v, "__code__") or hasattr(v, "__func__"):
|
||||
todo.append(v)
|
||||
return out
|
||||
|
||||
|
||||
def where(L, counter):
|
||||
"""How a worker finds a counter of the lift's process again: an attribute of the lifter (stats), or a module's global
|
||||
(poll.stats). ("L", name) or (module, name)"""
|
||||
for k, v in vars(L).items():
|
||||
if v is counter:
|
||||
return "L", k
|
||||
for name, module in list(sys.modules.items()):
|
||||
for k, v in list((getattr(module, "__dict__", None) or {}).items()):
|
||||
if v is counter:
|
||||
return name, k
|
||||
raise SystemExit("emit: a counter that neither the lifter nor a module holds by name cannot be counted in other processes")
|
||||
|
||||
|
||||
# ---- in a worker ------------------------------------------------------------------------------------------------------------
|
||||
_mine = None # (a worker's own: its lifter and its counters, or what kept it from having them, as text)
|
||||
|
||||
|
||||
def _no_throttle():
|
||||
"""Windows' power throttling off for this process, as the build does for every process it starts: it slows the processes
|
||||
of a window in the background to almost nothing on some PCs"""
|
||||
if os.name != "nt":
|
||||
return
|
||||
try:
|
||||
import ctypes
|
||||
|
||||
class State(ctypes.Structure):
|
||||
_fields_ = [("Version", ctypes.c_ulong), ("ControlMask", ctypes.c_ulong), ("StateMask", ctypes.c_ulong)]
|
||||
k = ctypes.WinDLL("kernel32")
|
||||
k.GetCurrentProcess.restype = ctypes.c_void_p
|
||||
k.SetProcessInformation.argtypes = [ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_ulong]
|
||||
s = State(1, 1, 0)
|
||||
k.SetProcessInformation(k.GetCurrentProcess(), 4, ctypes.byref(s), ctypes.sizeof(s))
|
||||
except (OSError, ValueError, TypeError, AttributeError):
|
||||
pass
|
||||
|
||||
|
||||
def _leave_with_parent():
|
||||
"""This process ends when the lift's does (stopped from outside: nothing tells its children)"""
|
||||
parent = multiprocessing.parent_process()
|
||||
if parent is None:
|
||||
return
|
||||
|
||||
def watch():
|
||||
parent.join()
|
||||
os._exit(3)
|
||||
threading.Thread(target=watch, daemon=True).start()
|
||||
|
||||
|
||||
def _start(blob):
|
||||
global _mine
|
||||
try:
|
||||
_no_throttle()
|
||||
_leave_with_parent()
|
||||
setup, cls, state, counters, want = pickle.loads(blob)
|
||||
for f in setup:
|
||||
f()
|
||||
got = emit_print(cls)
|
||||
if got != want:
|
||||
raise RuntimeError("the lifter's emit_function here is %s, the lift's process has %s: what set it up there must be "
|
||||
"named in `setup` (nx2/chunks.py write)" % (" over ".join(got), " over ".join(want)))
|
||||
L = cls.__new__(cls)
|
||||
vars(L).update(state)
|
||||
_mine = (L, [vars(L)[k] if m == "L" else getattr(importlib.import_module(m), k) for m, k in counters])
|
||||
except BaseException: # (said by the first run it is asked for: an initializer's own failure only breaks the pool)
|
||||
_mine = traceback.format_exc()
|
||||
|
||||
|
||||
def _emit_run(bodies, texts):
|
||||
"""A run's functions emitted: [(e, text, refs)], and each counter's keys with what this run added (every key the counter
|
||||
holds, in its order: the lift's process adds them in the runs' order, so its counters get their keys in the order one
|
||||
process gives them)"""
|
||||
if not isinstance(_mine, tuple):
|
||||
raise RuntimeError("a process of the emit could not be set up:\n%s" % _mine)
|
||||
L, counters = _mine
|
||||
L.funcs = bodies
|
||||
for c in counters:
|
||||
for k in c:
|
||||
c[k] = 0
|
||||
out = [(e,) + tuple(L.emit_function(e, texts)) for e in bodies]
|
||||
return out, [list(c.items()) for c in counters]
|
||||
|
||||
|
||||
# ---- in the lift's process ----------------------------------------------------------------------------------------------------
|
||||
class Workers:
|
||||
"""The spawned processes. jobs: how many; L, counters, setup: what each is given when it starts (the lifter's state, where
|
||||
its counters and the others in `counters` live, and the functions to call first)"""
|
||||
|
||||
def __init__(self, jobs, L, counters=(), setup=()):
|
||||
blob = pickle.dumps((list(setup), type(L), {k: v for k, v in vars(L).items() if k not in NOT_SENT},
|
||||
[where(L, c) for c in counters], emit_print(type(L))), pickle.HIGHEST_PROTOCOL)
|
||||
self.jobs = max(1, min(int(jobs), MOST))
|
||||
self.pool = concurrent.futures.ProcessPoolExecutor(max_workers=self.jobs, mp_context=multiprocessing.get_context("spawn"),
|
||||
initializer=_start, initargs=(blob,))
|
||||
|
||||
def _ask(self, fn, call):
|
||||
try:
|
||||
return call[0], self.pool.submit(fn, *call[1])
|
||||
except BrokenProcessPool as e: # (a process died since the last answer: the pool takes nothing more)
|
||||
raise SystemExit(GONE % (call[0], e)) from None
|
||||
|
||||
def each(self, fn, calls, wait=None):
|
||||
"""fn(*args) in the processes for every (what, args) of calls, the results in the calls' order. calls is read as the
|
||||
processes come free (AHEAD more than there are), so what it makes for a call is not held for all of them at once.
|
||||
A call that raises, a process that dies and one that gives no answer for `wait` seconds (WAIT) end it with SystemExit
|
||||
and what was said, and with the processes ended; so does leaving it before its end"""
|
||||
pending, calls, done = collections.deque(), iter(calls), False
|
||||
try:
|
||||
while True:
|
||||
while calls is not None and len(pending) < self.jobs + AHEAD:
|
||||
call = next(calls, None)
|
||||
if call is None:
|
||||
calls = None
|
||||
else:
|
||||
pending.append(self._ask(fn, call))
|
||||
if not pending:
|
||||
done = True
|
||||
return
|
||||
answer = self._answer(*pending.popleft(), wait or WAIT)
|
||||
done = calls is None and not pending # (the last: whoever takes it and asks for no more leaves the processes be)
|
||||
yield answer
|
||||
finally:
|
||||
if not done:
|
||||
self.stop(kill=True)
|
||||
|
||||
def ask(self, fn, calls):
|
||||
"""Every (what, args) of calls asked of the processes now: what answers() then takes. For work that must go to its
|
||||
end once it has begun (the page marks: a file each, rewritten in place): nothing is left to ask for later, so a lift
|
||||
that ends meanwhile for a reason of its own still has every file finished, the pool doing what it was asked before
|
||||
the interpreter leaves"""
|
||||
try:
|
||||
return [self._ask(fn, call) for call in calls]
|
||||
except BaseException:
|
||||
self.stop(kill=True)
|
||||
raise
|
||||
|
||||
def answers(self, asked, wait=None):
|
||||
"""The answers to ask()'s calls, in their order; it fails as each() does"""
|
||||
left = len(asked)
|
||||
try:
|
||||
for what, future in asked:
|
||||
answer = self._answer(what, future, wait or WAIT)
|
||||
left -= 1
|
||||
yield answer
|
||||
finally:
|
||||
if left:
|
||||
self.stop(kill=True)
|
||||
|
||||
def _answer(self, what, future, wait):
|
||||
left = wait
|
||||
while not concurrent.futures.wait([future], timeout=min(left, SLICE))[0]:
|
||||
left -= SLICE # (in slices: a PC that slept through the wait is not a worker that hung)
|
||||
if left <= 0:
|
||||
raise SystemExit("%s: its process gave no answer for %d seconds and was stopped" % (what, wait))
|
||||
try:
|
||||
return future.result()
|
||||
except KeyboardInterrupt:
|
||||
raise
|
||||
except BrokenProcessPool as e:
|
||||
raise SystemExit(GONE % (what, e)) from None
|
||||
except BaseException as e: # (what the process raised, SystemExit too, with where it raised it; the summary last, the line a build names)
|
||||
raise SystemExit("%s%s: its process failed: %s: %s" % (
|
||||
"%s\n" % str(e.__cause__).strip() if e.__cause__ is not None else "", what, type(e).__name__, e)) from None
|
||||
|
||||
def stop(self, kill=False):
|
||||
"""The processes ended: asked to leave after what they are doing, or (kill) ended now"""
|
||||
pool, self.pool = self.pool, None
|
||||
if pool is None:
|
||||
return
|
||||
if kill:
|
||||
for p in list((getattr(pool, "_processes", None) or {}).values()):
|
||||
try:
|
||||
p.kill()
|
||||
except (OSError, ValueError, AttributeError):
|
||||
pass
|
||||
pool.shutdown(wait=True, cancel_futures=True)
|
||||
|
||||
|
||||
def emit(workers, L, order, texts, counters, need=texts_of, runs=None):
|
||||
"""(e, text, refs) for every function of order, in order, emitted by the workers; counters (the dicts Workers was given)
|
||||
get each run's additions. need(funcs, es, texts): the texts a run is sent; runs: the runs, when not plan()'s"""
|
||||
funcs = L.funcs
|
||||
if runs is None:
|
||||
runs = plan(order, lambda e: len(funcs[e][0]))
|
||||
|
||||
def calls():
|
||||
for es in runs:
|
||||
yield ("the emit of f_%08x to f_%08x" % (es[0], es[-1]), ({e: funcs[e] for e in es}, need(funcs, es, texts)))
|
||||
gc.freeze() # (the lift's millions of objects out of the collector's sight while the runs' short-lived ones come and go)
|
||||
try:
|
||||
for out, counts in workers.each(_emit_run, calls()):
|
||||
for c, added in zip(counters, counts):
|
||||
for k, v in added:
|
||||
c[k] = c.get(k, 0) + v
|
||||
yield from out
|
||||
finally:
|
||||
gc.unfreeze()
|
||||
@@ -181,11 +181,14 @@ def main():
|
||||
if not a.no_polls:
|
||||
poll.install()
|
||||
gen = os.path.join(out_dir, "gen")
|
||||
# ((Triforce) setup: what the emit's spawned processes do to the lifter's class before they emit, as this one did above)
|
||||
files = chunks.write(L, gen, os.path.join(a.work, "image"), a.rechunk and not a.standalone,
|
||||
counters=[] if a.no_polls else [poll.stats])
|
||||
counters=[] if a.no_polls else [poll.stats], setup=[] if a.no_polls else [poll.install])
|
||||
import oot_patches
|
||||
main_build = next((m["build_id"] for m in meta["modules"] if m["name"] == "main"), "")
|
||||
chunks.edit_c(gen, L.stats, a.standalone and main_build == oot_patches.LIFE_BUILD_ID, whole=not a.max_func)
|
||||
# ((Triforce) wait=False: where the emit's processes were spawned they share the page marks while the files below are
|
||||
# written, which read none of the C; marks_done() waits for them before the report says their counts)
|
||||
chunks.edit_c(gen, L.stats, a.standalone and main_build == oot_patches.LIFE_BUILD_ID, whole=not a.max_func, wait=False)
|
||||
a64_rtgen.write(gen)
|
||||
if a.standalone:
|
||||
start, exit_ = standalone.write_image_c(gen, img, meta, a.exefs)
|
||||
@@ -194,6 +197,7 @@ def main():
|
||||
nvn = standalone.nvn_exports(nso.load_exefs(a.exefs))
|
||||
standalone.write_nvn_exports(gen, nvn)
|
||||
print("standalone: nvn2_exports.c (%d sdk NVN functions routed to host/nvn2.c)" % len(nvn))
|
||||
chunks.marks_done()
|
||||
L.report()
|
||||
print("interrupt polls: %d loop headers in %d functions" % (poll.stats["polls"], poll.stats["functions"]))
|
||||
print("wrote %d code files to %s (%.0fs)" % (len(files), gen, time.time() - t0))
|
||||
|
||||
320
nx2/make_game.py
320
nx2/make_game.py
@@ -25,6 +25,7 @@ and one plain line the builders also read (older ones only show it): "the shader
|
||||
build's last step, the shaders' preparation, was skipped or failed and is left to the game's first start (NOT_PREPARED)
|
||||
"""
|
||||
import argparse
|
||||
import atexit
|
||||
import contextlib
|
||||
import filecmp
|
||||
import hashlib
|
||||
@@ -49,6 +50,7 @@ STARTED = time.time() # (the build's total time, kept in build_times.json and
|
||||
sys.path[:0] = [HERE, os.path.join(ROOT, "third_party", "a64-kit", "tools")] # (python -I leaves the script folder off)
|
||||
import toolchain as _toolchain
|
||||
import disk as _disk
|
||||
import code_mods as _code_mods # (Triforce: the mods' patches of the game's code, applied before the lift)
|
||||
# (hookshot) RX_TARGET=windows on Linux: the Windows game, with llvm-mingw's Linux release (HOOKSHOT_MINGW, or llvm-mingw
|
||||
# beside the builder), to run under Proton or copy to a Windows PC
|
||||
LINUX, CROSS = not _toolchain.windows(), _toolchain.windows() and os.name != "nt"
|
||||
@@ -97,7 +99,23 @@ def say_prepare(words):
|
||||
print("progress 1.000 Done", flush=True)
|
||||
|
||||
|
||||
STAGE_TIMES = [] # ((stage, when it began)): what each step of this build took, said at its end (stage_times)
|
||||
|
||||
|
||||
def stage_times(now):
|
||||
""" "before the game files 0:41, read 1:32, lift 2:10, compile 7:42, finish 1:17": each stage's time up to the next
|
||||
one's start, the last one's up to now, and what came before the first (the tools' checks and what was fetched)"""
|
||||
if not STAGE_TIMES:
|
||||
return ""
|
||||
parts = [("before the game files", STAGE_TIMES[0][1] - STARTED)]
|
||||
for (name, began), (_, ended) in zip(STAGE_TIMES, STAGE_TIMES[1:] + [("", now)]):
|
||||
parts.append((name, ended - began))
|
||||
return ", ".join("%s %s" % (name, clock(seconds)) for name, seconds in parts)
|
||||
|
||||
|
||||
def say(*words):
|
||||
if words and words[0] == "stage" and len(words) > 1:
|
||||
STAGE_TIMES.append((str(words[1]), time.time()))
|
||||
if PREPARE:
|
||||
return say_prepare([str(w) for w in words])
|
||||
print("@@" + " ".join(str(w) for w in words), flush=True)
|
||||
@@ -357,10 +375,36 @@ def file_progress(verb, clock=time.monotonic):
|
||||
return progress
|
||||
|
||||
|
||||
def stand_in_damaged_files(out, romfs):
|
||||
def unpack_seen(game):
|
||||
"""What the unpack itself saw of the romfs .zs files it wrote (nx2/nup.py ZS_CHECKED, in the folder it extracted to):
|
||||
{"files": how many, "damaged": [paths under romfs]}, or None when it left no such note or one that cannot be right
|
||||
(then every file is opened and looked at, as before)"""
|
||||
import json
|
||||
import nup
|
||||
try:
|
||||
with open(os.path.join(game, nup.ZS_CHECKED), encoding="utf-8") as f:
|
||||
seen = json.load(f)
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
if not isinstance(seen, dict) or not isinstance(seen.get("files"), int) or not isinstance(seen.get("damaged"), list):
|
||||
return None
|
||||
for rel in seen["damaged"]:
|
||||
if not isinstance(rel, str):
|
||||
return None
|
||||
parts = rel.split("/")
|
||||
if not rel.endswith(".zs") or "\\" in rel or ":" in rel or any(p in ("", ".", "..") for p in parts):
|
||||
return None
|
||||
return seen
|
||||
|
||||
|
||||
def stand_in_damaged_files(out, romfs, seen=None):
|
||||
"""An incomplete dump's damaged textures replaced by whole ones of the same kind, and said: without them the game
|
||||
leaves out the walls and floors that use them (the Water Temple, #1002). Only in a romfs this builder unpacked (its
|
||||
.triforce_romfs stamp), and once per unpack: .triforce_romfs_checked holds the stamp it was checked under"""
|
||||
.triforce_romfs stamp), and once per unpack: .triforce_romfs_checked holds the stamp it was checked under.
|
||||
seen: what this unpack saw of its own .zs files as it wrote them (unpack_seen): the damaged ones are then known and
|
||||
no file is opened to find them. Opening all 102,185 took 292 s of an 18 minute build on a PC whose virus scanner
|
||||
looks at each file opened; a folder unpacked by an earlier builder, or read from this .nup before without the
|
||||
check's note, is still gone through file by file"""
|
||||
good = os.path.join(out, ".triforce_romfs")
|
||||
checked = os.path.join(out, ".triforce_romfs_checked")
|
||||
if not os.path.isfile(good) or not os.path.isdir(romfs):
|
||||
@@ -373,9 +417,14 @@ def stand_in_damaged_files(out, romfs):
|
||||
if f.read() == stamp:
|
||||
return
|
||||
import nup
|
||||
print("checking your game files for damaged textures (once after an unpack): every .zs file is opened, which can take "
|
||||
"minutes", flush=True)
|
||||
done = nup.stand_in_damaged(romfs, file_progress("checked"))
|
||||
if seen is not None:
|
||||
print("your game files were checked for damaged textures as they were unpacked: %s .zs files, %d damaged"
|
||||
% (format(seen["files"], ","), len(seen["damaged"])), flush=True)
|
||||
done = nup.stand_in_damaged(romfs, known=seen["damaged"])
|
||||
else:
|
||||
print("checking your game files for damaged textures (once after an unpack): every .zs file is opened, which can take "
|
||||
"minutes", flush=True)
|
||||
done = nup.stand_in_damaged(romfs, file_progress("checked"))
|
||||
with open(checked, "w") as f:
|
||||
f.write(stamp)
|
||||
if not done:
|
||||
@@ -975,6 +1024,22 @@ def ship_state_list(out):
|
||||
print("the shipped pipeline state list: %d pipelines" % ((os.path.getsize(src) - 12) // 528), flush=True)
|
||||
|
||||
|
||||
def ship_triforce_assets(out):
|
||||
"""(Triforce, v0.5.0) Triforce's own files the game reads beside it (assets/triforce_assets: the Mods screen's mask, CC BY 4.0,
|
||||
its credits in CREDITS.txt), as <out>/triforce_assets. None is a game file. Without them the game runs as before: the Mask
|
||||
Salesman stands without the mask (gpu/vk/vk_salesman.cpp)"""
|
||||
src = os.path.join(ROOT, "assets", "triforce_assets")
|
||||
if not os.path.isdir(src):
|
||||
return
|
||||
dst = os.path.join(out, "triforce_assets")
|
||||
os.makedirs(dst, exist_ok=True)
|
||||
for name in sorted(os.listdir(src)):
|
||||
s, d = os.path.join(src, name), os.path.join(dst, name)
|
||||
if os.path.isfile(s) and not (os.path.isfile(d) and filecmp.cmp(s, d, shallow=False)):
|
||||
copy_writable(s, d)
|
||||
print("Triforce's own files beside the game: %s" % ", ".join(sorted(os.listdir(src))), flush=True)
|
||||
|
||||
|
||||
# (the builders read this line's start: the step was skipped or failed, and the shaders' row says so instead of a check mark)
|
||||
NOT_PREPARED = "the shaders were not prepared now"
|
||||
# (a run that prints nothing this long is stopped. Not its whole length: that is 25 minutes on some AMD GPUs, and the driver's own
|
||||
@@ -1140,11 +1205,189 @@ def shader_census_cmd(romfs, out):
|
||||
LIFT_KEY = ".lift_key" # (in <work>/sa/gen: what the lifted C there was made from)
|
||||
|
||||
|
||||
def compile_alone_failed(lines):
|
||||
"""Whether a build that failed in nx2/build.py failed in its compile and never came to a link: build.py's own lines say
|
||||
so ("compiled in 412s, 2 failed", and no "link FAILED" after it). The objects that compile did make are then whole,
|
||||
each stamped and cached under the hash of its source and flags only once its compiler ended well, and are worth
|
||||
keeping: a compiler that ran out of memory, or a disk that filled, used to cost the next build every one of the ~600
|
||||
files again (make_game deleted them all), an hour and more on the PCs where a compile fails in the first place. A
|
||||
build that got as far as its link and failed there, or that ended without saying how its compile went, is not
|
||||
trusted: its objects go, as before (a stale object linked "compiled in 0s" at every try, issue 1197)"""
|
||||
failed = None
|
||||
for line in lines:
|
||||
m = re.match(r"compiled in \d+s, (\d+) failed", line)
|
||||
if m:
|
||||
failed = int(m.group(1))
|
||||
elif "link FAILED" in line:
|
||||
return False
|
||||
return bool(failed)
|
||||
PROMPT_KEY = ".made_from" # (in <out>/prompts: prompts_key of what that folder was made from)
|
||||
|
||||
|
||||
def prompts_key(out):
|
||||
"""What the button prompts of other controllers are made from, as one hash: the tools that draw them (nx2/prompt_*.py,
|
||||
the faces' drawings and parts in nx2/prompt_faces, the icon fonts in third_party/kenney_input), the packages they are
|
||||
drawn with (nx2/prompt_requirements.txt) and the game's files, by the unpack's stamp (.triforce_romfs: the .nup's name
|
||||
and size). None when the romfs has no such stamp (the player's own folders, linked in): its prompts are made at every
|
||||
build, as before. (Triforce) The keyboard and mouse face's too: nx2/prompt_keys.py, which draws it, and the font its
|
||||
labels are set in (third_party/fonts/libertinus: that one file, the menus' other fonts are not the prompts')"""
|
||||
stamp = os.path.join(out, ".triforce_romfs")
|
||||
if not os.path.isfile(stamp):
|
||||
return None
|
||||
# (each file under a name that does not depend on where the tools are installed)
|
||||
files = [("nx2/" + n, os.path.join(HERE, n))
|
||||
for n in ("prompt_assets.py", "prompt_draw.py", "prompt_faces.py", "prompt_requirements.txt", "prompt_keys.py")]
|
||||
labels = "third_party/fonts/libertinus/LibertinusSans-Bold.ttf"
|
||||
files.append((labels, os.path.join(ROOT, *labels.split("/"))))
|
||||
for name, root in (("nx2/prompt_faces", os.path.join(HERE, "prompt_faces")),
|
||||
("third_party/kenney_input", os.path.join(ROOT, "third_party", "kenney_input"))):
|
||||
for folder, dirs, names in os.walk(root):
|
||||
dirs.sort()
|
||||
files += [(name + "/" + os.path.relpath(os.path.join(folder, n), root).replace(os.sep, "/"), os.path.join(folder, n))
|
||||
for n in sorted(names)]
|
||||
h = hashlib.sha1()
|
||||
with open(stamp, "rb") as f:
|
||||
h.update(f.read())
|
||||
for name, path in files:
|
||||
if os.path.isfile(path):
|
||||
h.update(name.encode())
|
||||
with open(path, "rb") as f:
|
||||
h.update(f.read())
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
class EarlyCompile:
|
||||
"""(Windows) The files that need nothing of the translated game, compiled while the game is being translated
|
||||
(nx2/build.py RX_EARLY): libopus and the two renderers with their libraries, ~520 of the build's ~630 files. The lift was
|
||||
one process there and left every other processor idle for its two minutes, and then the compile had those files to do
|
||||
on top of the game's. (The lift's emit has processes of its own there now, nx2/emit_pool.py: at most half the
|
||||
processors, and its first part, the discovery, is still one process.) finish() waits for it before the build proper starts, which then finds their objects made.
|
||||
A failure here costs nothing: the build compiles whatever is missing, as before. HOOKSHOT_EARLY_COMPILE=0: not done"""
|
||||
|
||||
def __init__(self, work, env, jobs):
|
||||
self.proc = None
|
||||
if os.name != "nt" or os.environ.get("HOOKSHOT_EARLY_COMPILE") == "0":
|
||||
return
|
||||
self.log = os.path.join(work, "early_compile.log")
|
||||
e = dict(os.environ, PYTHONIOENCODING="utf-8")
|
||||
e.update(env)
|
||||
e["RX_EARLY"] = "1"
|
||||
e.pop("HOOKSHOT_TIMES", None) # (no countdown lines: nobody reads them here)
|
||||
with open(self.log, "w", encoding="utf-8") as log:
|
||||
self.proc = subprocess.Popen(PY + ["nx2/build.py", "--standalone", "--jobs", str(max(1, jobs - 1))], cwd=ROOT, env=e,
|
||||
stdout=log, stderr=subprocess.STDOUT)
|
||||
no_throttle(getattr(self.proc, "_handle", None))
|
||||
|
||||
def stop(self):
|
||||
if self.proc is not None and self.proc.poll() is None:
|
||||
self.proc.kill()
|
||||
self.proc.wait()
|
||||
|
||||
def finish(self):
|
||||
if self.proc is None:
|
||||
return
|
||||
waited = time.time()
|
||||
code = self.proc.wait()
|
||||
said = ""
|
||||
try:
|
||||
with open(self.log, encoding="utf-8", errors="replace") as f:
|
||||
said = next((line.strip() for line in reversed(f.read().splitlines()) if line.startswith("compiled in ")), "")
|
||||
except OSError:
|
||||
pass
|
||||
print("the files that need no translated code were compiled beside the lift: %s%s (%s more after the lift ended)%s"
|
||||
% (said or "no result", "" if not code else ", stopped with status %d" % code, clock(time.time() - waited),
|
||||
"" if not code else ": the build compiles what is missing"), flush=True)
|
||||
self.proc = None
|
||||
|
||||
|
||||
class Prompts:
|
||||
"""The button prompts of other controllers (host/prompts.c): the game's icon font and textures redrawn per controller by
|
||||
nx2/prompt_assets.py, with fontTools. It took 87 s at the end of every build and every update, with nothing else
|
||||
running. It reads only the game's files, so it now starts as soon as they are read, beside the lift and the compile,
|
||||
into <out>/prompts.new, and finish() puts the folder in place; and it is not run at all when the folder there was made
|
||||
from the same tools and the same game files (prompts_key). Without it, or when it fails, the game shows its own icons,
|
||||
as before"""
|
||||
|
||||
def __init__(self, romfs, out, work):
|
||||
self.out, self.folder, self.new = out, os.path.join(out, "prompts"), os.path.join(out, "prompts.new")
|
||||
self.key = prompts_key(out)
|
||||
self.proc = self.log = None
|
||||
self.state = "no fontTools" if importlib.util.find_spec("fontTools") is None else None
|
||||
if self.state:
|
||||
return
|
||||
try:
|
||||
with open(os.path.join(self.folder, PROMPT_KEY), encoding="utf-8") as f:
|
||||
if self.key and f.read().strip() == self.key:
|
||||
self.state = "kept"
|
||||
return
|
||||
except OSError:
|
||||
pass
|
||||
shutil.rmtree(self.new, ignore_errors=True)
|
||||
self.log = os.path.join(work, "prompts.log")
|
||||
with open(self.log, "w", encoding="utf-8") as log:
|
||||
self.proc = subprocess.Popen(PY + [os.path.join(HERE, "prompt_assets.py"), "--romfs", romfs, "--out", self.new],
|
||||
cwd=ROOT, env=dict(os.environ, PYTHONIOENCODING="utf-8"), stdout=log,
|
||||
stderr=subprocess.STDOUT)
|
||||
no_throttle(getattr(self.proc, "_handle", None))
|
||||
|
||||
def stop(self):
|
||||
"""(the build failed or was stopped) the drawing ended and what it wrote taken away: the folder in place stays"""
|
||||
if self.proc is not None and self.proc.poll() is None:
|
||||
self.proc.kill()
|
||||
self.proc.wait()
|
||||
shutil.rmtree(self.new, ignore_errors=True)
|
||||
|
||||
def finish(self):
|
||||
"""Waits for the drawing, says what it said, and puts the new folder in the old one's place"""
|
||||
if self.state == "no fontTools":
|
||||
shutil.rmtree(self.folder, ignore_errors=True)
|
||||
print("the button prompts of other controllers were not made (no fontTools): the game shows its own", flush=True)
|
||||
return
|
||||
if self.state == "kept":
|
||||
print("the button prompts of other controllers are as the last build made them (the same tools and game files): kept",
|
||||
flush=True)
|
||||
return
|
||||
code = self.proc.wait()
|
||||
try:
|
||||
with open(self.log, encoding="utf-8", errors="replace") as f:
|
||||
sys.stderr.write(f.read())
|
||||
sys.stderr.flush()
|
||||
except OSError:
|
||||
pass
|
||||
shutil.rmtree(self.folder, ignore_errors=True)
|
||||
if code:
|
||||
print("the button prompts of other controllers were not made: the game shows its own", flush=True)
|
||||
shutil.rmtree(self.new, ignore_errors=True)
|
||||
return
|
||||
os.replace(self.new, self.folder)
|
||||
if self.key:
|
||||
with open(os.path.join(self.folder, PROMPT_KEY), "w", encoding="utf-8") as f:
|
||||
f.write(self.key + chr(10))
|
||||
|
||||
|
||||
# (Triforce) Files of nx2/ that are not the translator's and that the lift (nx2/lift.py, nx2/split_hot.py and what they
|
||||
# import) cannot read: this driver, the steps before and after the lift, the version, the button prompts, and the tests
|
||||
# kept in nx2/. They were part of the lift key, so an update that changed one of them, and every release changes the
|
||||
# version and most change this file, translated the whole game again on every player's PC: 2:48 of a 3:05 update build
|
||||
# on a 16-thread PC, for C that came out the same byte for byte. tests/test_lift_key_files.py holds the list to what
|
||||
# the lift can import and to what its files name.
|
||||
LIFT_KEY_NOT = ("make_game.py", "build.py", "nup.py", "version.py", "shader_census.py", "prompt_assets.py", "prompt_draw.py",
|
||||
"prompt_faces.py", "prompt_requirements.txt", "prompt_keys.py",
|
||||
"code_mods.py") # (the mods' code patches: what it does to the game's code is in the key, the exefs's files)
|
||||
|
||||
|
||||
def lift_key_file(rel):
|
||||
"""Whether a file of nx2/ (rel: its path under nx2/, with /) is part of the lift key: all but LIFT_KEY_NOT, the
|
||||
prompt_faces folder and the tests (test_*.py)"""
|
||||
name = rel.rsplit("/", 1)[-1]
|
||||
return not (rel in LIFT_KEY_NOT or rel.startswith("prompt_faces/") or (name.startswith("test_") and name.endswith(".py")))
|
||||
|
||||
|
||||
def lift_key(exefs, work, env):
|
||||
"""What the lift's output depends on: the game's code, the translator (nx2/, the vendored a64-kit), the HLE functions host/ defines
|
||||
(hooked in the lift; the rest of host/ is compiled after it, so a change there alone does not lift again), the hot list and chunk
|
||||
boundaries, the lift's environment and the Python running it. A file the lift never reads may be in it (a new build then lifts
|
||||
again); one it reads must be"""
|
||||
"""What the lift's output depends on: the game's code, the translator (nx2/ but for LIFT_KEY_NOT, the vendored a64-kit), the HLE
|
||||
functions host/ defines (hooked in the lift; the rest of host/ is compiled after it, so a change there alone does not lift again),
|
||||
the hot list and chunk boundaries, the lift's environment and the Python running it. A file the lift never reads may be in it (a
|
||||
new build then lifts again); one it reads must be"""
|
||||
import kit # noqa: F401 (a64-kit on the path, for standalone)
|
||||
import standalone
|
||||
h = hashlib.sha1(sys.version.encode(), usedforsecurity=False)
|
||||
@@ -1154,7 +1397,8 @@ def lift_key(exefs, work, env):
|
||||
for top in (HERE, os.path.join(ROOT, "third_party", "a64-kit")):
|
||||
for d, dirs, fs in os.walk(top):
|
||||
dirs[:] = sorted(x for x in dirs if x != "__pycache__")
|
||||
files += [(ROOT, os.path.join(d, f)) for f in sorted(fs) if f.endswith((".py", ".c", ".h", ".cpp", ".hpp", ".txt", ".json"))]
|
||||
files += [(ROOT, os.path.join(d, f)) for f in sorted(fs) if f.endswith((".py", ".c", ".h", ".cpp", ".hpp", ".txt", ".json"))
|
||||
and (top != HERE or lift_key_file(os.path.relpath(os.path.join(d, f), HERE).replace(os.sep, "/")))]
|
||||
files += [(work, os.path.join(work, "hot_funcs.txt")), (work, os.path.join(work, "image", "chunks.txt"))]
|
||||
for base, f in files:
|
||||
if os.path.isfile(f):
|
||||
@@ -1274,7 +1518,9 @@ def fsr_runtime_dir(work):
|
||||
|
||||
def fetch_fsr_runtime(work):
|
||||
"""(--fsr, Windows) AMD's two signed FSR 4 libraries into fsr_runtime_dir, only what is missing. A download that fails
|
||||
is said and the build goes on: the game then does not offer AMD FSR 4 until a build gets them"""
|
||||
is said and the build goes on: the game then does not offer AMD FSR 4 until a build gets them. (Triforce) And its
|
||||
frame generation library with them (scripts/fsr_sdk.py OPTIONAL): without that one the game has FSR 4 as before and
|
||||
the Frame generation row no AMD FSR 4"""
|
||||
fsr = fsr_script()
|
||||
if LINUX or fsr is None:
|
||||
return
|
||||
@@ -1288,7 +1534,8 @@ def fetch_fsr_runtime(work):
|
||||
|
||||
def place_fsr_runtime(out, work, wanted):
|
||||
"""(AMD FSR 4) a game built with it gets AMD's two libraries and their license text beside its exe, where
|
||||
gpu/vk/vk_fsr4.cpp looks; a folder built with them before and without FSR 4 now loses them. Returns what was placed"""
|
||||
gpu/vk/vk_fsr4.cpp looks, and (Triforce) AMD's frame generation library when it was fetched; a folder built with them
|
||||
before and without FSR 4 now loses them, all three. Returns what was placed"""
|
||||
fsr = fsr_script()
|
||||
if LINUX or fsr is None:
|
||||
return []
|
||||
@@ -1447,8 +1694,8 @@ def parse_args():
|
||||
p.add_argument("--fetch-ngx-sdk", action="store_true", help="download NVIDIA's DLSS SDK into HOOKSHOT_NGX_SDK before the build")
|
||||
p.add_argument("--accept-nvidia-license", action="store_true", help="NVIDIA's DLSS SDK license, accepted (with --fetch-ngx-sdk)")
|
||||
# (AMD FSR 4, off unless asked: nothing is downloaded. gpu/vk/vk_fsr4.cpp)
|
||||
p.add_argument("--fsr", action="store_true", help="build with AMD FSR 4; on Windows AMD's two libraries are downloaded "
|
||||
"from AMD's repository and put beside the game (scripts/fsr_sdk.py)")
|
||||
p.add_argument("--fsr", action="store_true", help="build with AMD FSR 4; on Windows AMD's two libraries, and the one for "
|
||||
"its frame generation, are downloaded from AMD's repository and put beside the game (scripts/fsr_sdk.py)")
|
||||
p.add_argument("--no-fsr", action="store_true", help="with --update: build without AMD FSR 4 even if the folder had it")
|
||||
# (RenoDX's HDR, in every build since 11-13: its shaders and the Slang compiler are downloaded and compiled here.
|
||||
# scripts/renodx_shaders.py, gpu/vk/vk_renodx.cpp)
|
||||
@@ -1582,6 +1829,7 @@ def main(a=None):
|
||||
"or choose the file again in Hookshot Builder." % (a.nup, e))
|
||||
stamp = "unpacked by this builder: %s %d" % (os.path.basename(a.nup), st.st_size)
|
||||
same = False
|
||||
unpacked_seen = None
|
||||
if os.path.exists(good) and os.path.isdir(romfs) and os.path.isdir(kept):
|
||||
with open(good) as f:
|
||||
same = f.read().strip() == stamp
|
||||
@@ -1610,6 +1858,7 @@ def main(a=None):
|
||||
raise
|
||||
if os.path.exists(romfs):
|
||||
shutil.rmtree(romfs)
|
||||
unpacked_seen = unpack_seen(game) # (the .zs files as the unpack saw them: stand_in_damaged_files below)
|
||||
shutil.move(os.path.join(game, "romfs"), romfs)
|
||||
shutil.move(staged_exefs, exefs)
|
||||
shutil.rmtree(game, ignore_errors=True)
|
||||
@@ -1619,7 +1868,7 @@ def main(a=None):
|
||||
f.write(stamp + chr(10))
|
||||
check_modules(exefs)
|
||||
# (also when the .nup was read before: a folder built from a 23.3 GB dump by an earlier builder gets them too)
|
||||
stand_in_damaged_files(out, romfs)
|
||||
stand_in_damaged_files(out, romfs, unpacked_seen)
|
||||
else:
|
||||
check_cache_paths(out, (kept, romfs))
|
||||
refuse_dump_links(a.exefs)
|
||||
@@ -1630,6 +1879,10 @@ def main(a=None):
|
||||
shutil.rmtree(kept, ignore_errors=True)
|
||||
shutil.copytree(exefs, kept)
|
||||
check_supported_build(exefs)
|
||||
# (Triforce) code mods: the .ips and .pchtxt patches of the game folder's mods, applied to this copy of the game's code, which
|
||||
# the lift reads and whose files its key hashes: never to the player's files nor to the copy kept for updates (kept, above).
|
||||
# A patch added, taken away, switched off or edited so changes the key and the game is translated again (nx2/code_mods.py)
|
||||
code_patches = _code_mods.build_step(exefs, out, say=lambda line: print(line, flush=True))
|
||||
|
||||
# (made once on the development PC: lists of addresses and counts, no game code)
|
||||
for name in ("hot_funcs.txt", "hot_order.txt", "hookshot_pieces.prof", "hookshot.res"):
|
||||
@@ -1648,6 +1901,9 @@ def main(a=None):
|
||||
if os.path.exists(os.path.join(work, "hookshot_pieces.prof")) and profile_readable(os.path.join(work, "hookshot_pieces.prof")):
|
||||
env["RX_PGO"] = os.path.join(work, "hookshot_pieces.prof")
|
||||
|
||||
prompts = Prompts(romfs, out, work) # (drawn beside the lift and the compile: finish() below)
|
||||
atexit.register(prompts.stop) # (a build that stops for any reason takes its drawing with it)
|
||||
|
||||
say("stage", "lift", "Translating the game to PC code")
|
||||
# (the same game, tools and settings as the lift before: its C is what this lift would write, and nothing recompiles from it. A
|
||||
# rebuild after a change to host/ alone skips the minutes of the lift)
|
||||
@@ -1676,9 +1932,12 @@ def main(a=None):
|
||||
except OSError as e:
|
||||
fail("Could not remove earlier translated file %s: %s. The new translation has not started." %
|
||||
(os.path.join(gen_dir, f), e))
|
||||
early = EarlyCompile(work, env, a.jobs) # (what needs no lifted code is compiled meanwhile)
|
||||
atexit.register(early.stop)
|
||||
run(PY + ["nx2/lift.py", "--standalone", "--write", "--exefs", exefs, "--work", work], env)
|
||||
if os.path.isfile(os.path.join(work, "sa", "gen", "code_hot.c")):
|
||||
run(PY + ["nx2/split_hot.py", os.path.join(work, "sa", "gen")], env) # (eight files side by side: ~6 minutes, not 79)
|
||||
early.finish()
|
||||
write_lift_key(work, key)
|
||||
cut = lifted_cut_short(gen_dir)
|
||||
if not cut:
|
||||
@@ -1714,7 +1973,11 @@ def main(a=None):
|
||||
mb = 0
|
||||
return 23.5 * mb if name.startswith("code_hot") else 3.4 * mb + 1 if name.startswith("code_") else 1.3
|
||||
|
||||
said = [] # (build.py's lines about how its compile and its links went: compile_alone_failed)
|
||||
|
||||
def compiled(line):
|
||||
if line.startswith("compiled in ") or "link FAILED" in line:
|
||||
said.append(line)
|
||||
m = re.match(r"(\d+) files, ", line)
|
||||
if m:
|
||||
try:
|
||||
@@ -1751,9 +2014,15 @@ def main(a=None):
|
||||
os.remove(key_file)
|
||||
# (and its compiled objects go too: a cached runtime object from another version linked with every object "compiled in 0s"
|
||||
# and a64_dirty, a64_icall, a64_lookup, recomp_cntpct undefined, at every try (issue 1197, Linux, v2.0.10))
|
||||
for sub in (os.path.join(work, "objcache"), os.path.join(work, "sa", "build", "obj")):
|
||||
shutil.rmtree(sub, ignore_errors=True)
|
||||
print("the build failed: the next one translates and compiles the game again (nothing of this one is reused)", flush=True)
|
||||
# (Triforce: but not when the compile alone failed, before any link: compile_alone_failed)
|
||||
if compile_alone_failed(said):
|
||||
print("the build failed in its compile: the files that did compile are kept, and the next build translates the game "
|
||||
"again and compiles only the rest", flush=True)
|
||||
else:
|
||||
for sub in (os.path.join(work, "objcache"), os.path.join(work, "sa", "build", "obj")):
|
||||
shutil.rmtree(sub, ignore_errors=True)
|
||||
print("the build failed: the next one translates and compiles the game again (nothing of this one is reused)", flush=True)
|
||||
prompts.stop()
|
||||
raise
|
||||
compile_seconds = time.time() - compile_started
|
||||
|
||||
@@ -1816,19 +2085,14 @@ def main(a=None):
|
||||
if run(shader_census_cmd(romfs, os.path.join(out, "triforce_shaders.bin")), check=False, output=sys.stderr):
|
||||
print("the shader list was not made: the game will compile its shaders as it plays", flush=True)
|
||||
# (hookshot) the button prompts of other controllers (host/prompts.c): the game's icon font redrawn per controller, with fontTools.
|
||||
# Without it the game shows its own icons
|
||||
prompts = os.path.join(out, "prompts")
|
||||
shutil.rmtree(prompts, ignore_errors=True)
|
||||
if importlib.util.find_spec("fontTools") is None:
|
||||
print("the button prompts of other controllers were not made (no fontTools): the game shows its own", flush=True)
|
||||
elif run(PY + [os.path.join(HERE, "prompt_assets.py"), "--romfs", romfs, "--out", prompts], check=False, output=sys.stderr):
|
||||
print("the button prompts of other controllers were not made: the game shows its own", flush=True)
|
||||
shutil.rmtree(prompts, ignore_errors=True)
|
||||
# Without it the game shows its own icons. (Triforce: drawn since the game files were read, or kept: class Prompts)
|
||||
prompts.finish()
|
||||
icon = os.path.join(ROOT, "assets", "triforce.png")
|
||||
if LINUX and os.path.exists(icon): # (the window's icon: host/posix/display_sdl.c)
|
||||
copy_writable(icon, os.path.join(out, "triforce.png")) # (a second build overwrites the first's)
|
||||
seed_pipelines(out)
|
||||
ship_state_list(out)
|
||||
ship_triforce_assets(out)
|
||||
language = os.path.join(out, "language.txt") # (the same text as host/nn_applet.c writes)
|
||||
with open(language, "w", encoding="utf-8", newline="\n") as f:
|
||||
f.write("# The game's language. Change the line below to one of the codes listed here, save, and start the game again.\n\n%s\n\n" % a.language)
|
||||
@@ -1850,6 +2114,8 @@ def main(a=None):
|
||||
# (the Python builder: builder is its pythonw.exe, and these the arguments before --update, host/hos_update.c)
|
||||
if os.environ.get("HOOKSHOT_BUILDER_ARGS"):
|
||||
f.write("builder_args=%s\n" % os.environ["HOOKSHOT_BUILDER_ARGS"])
|
||||
# (Triforce) what code patches this game is built with, for the next build and for the game's start (host/mods.c)
|
||||
_code_mods.write_record(out, code_patches)
|
||||
# (the game's code as read: written again by every build, so not left behind in the work folder. The translated code (sa\gen)
|
||||
# and the compiled objects stay: the next build skips the lift when nothing it reads changed (#428) and compiles only what
|
||||
# changed; the builder says how much stays and lets it go. Kept in a development checkout)
|
||||
@@ -1872,6 +2138,8 @@ def main(a=None):
|
||||
print("the build took %s, compiled in %s%s" % (clock(total_seconds), clock(compile_seconds),
|
||||
"" if previous is None else " (the last one %s)" % clock(previous)), flush=True)
|
||||
say("times", "%.0f" % total_seconds, "%.0f" % compile_seconds, "-1" if previous is None else "%.0f" % previous)
|
||||
# (where the time went, for a report of a slow build and for measuring a change to the build itself)
|
||||
print("the build's steps: %s" % stage_times(time.time()), flush=True)
|
||||
# (after the times: they stay comparable from build to build, without a GPU's minutes that can be switched off)
|
||||
skipped = prepare_skipped(a)
|
||||
if skipped:
|
||||
|
||||
57
nx2/nup.py
57
nx2/nup.py
@@ -438,6 +438,25 @@ class ProgressSink:
|
||||
return count
|
||||
|
||||
|
||||
class HeadSink:
|
||||
"""The first four bytes written, kept: extract's look at each romfs .zs file as it is unpacked (ZS_CHECKED)."""
|
||||
def __init__(self, output):
|
||||
self.output = output
|
||||
self.head = b""
|
||||
|
||||
def write(self, data):
|
||||
if len(self.head) < 4:
|
||||
self.head += bytes(data[:4 - len(self.head)])
|
||||
return self.output.write(data)
|
||||
|
||||
|
||||
# (written into the folder a romfs was extracted to: how many of its .zs files were looked at as they were written, and
|
||||
# which of them do not start as zstd data, by their path under romfs. nx2/make_game.py reads it in place of opening every
|
||||
# .zs file again after the unpack, damaged_game_files: 102,185 files, 292 s of an 18 minute build where a virus scanner
|
||||
# looks at each file that is opened)
|
||||
ZS_CHECKED = "zs_checked.json"
|
||||
|
||||
|
||||
def file_system(path):
|
||||
"""The file system of the volume path is on (NTFS, exFAT, FAT32, ReFS): Windows only, None elsewhere"""
|
||||
if os.name != "nt":
|
||||
@@ -465,8 +484,13 @@ def out_of_space(error, dst, part, rel, size):
|
||||
part, rel, size, free, ", " + kind if kind else ""))
|
||||
|
||||
|
||||
def extract(nup, t, out, only=None, quiet=False):
|
||||
def extract(nup, t, out, only=None, quiet=False, zs=None):
|
||||
"""zs: a dict that is given "files", the romfs .zs files there are, and "damaged", those of them that do not start as
|
||||
zstd data (their paths under romfs, "/" between the parts): each one looked at as it is written, and one that was
|
||||
there already read back"""
|
||||
total = n = written = 0
|
||||
if zs is not None:
|
||||
zs.update(files=0, damaged=[])
|
||||
interval = 2
|
||||
next_report = 0 if quiet else monotonic() + interval
|
||||
|
||||
@@ -483,15 +507,26 @@ def extract(nup, t, out, only=None, quiet=False):
|
||||
continue
|
||||
for rel, r in nup.walk(t[part]):
|
||||
dst = extraction_path(out, part, rel)
|
||||
looked = zs is not None and part == "romfs" and rel.endswith(".zs")
|
||||
if os.path.exists(dst) and os.path.getsize(dst) == r[2]:
|
||||
if looked:
|
||||
with open(dst, "rb") as f:
|
||||
zs["files"] += 1
|
||||
if not zstd_start(f.read(4)):
|
||||
zs["damaged"].append(rel)
|
||||
continue # (re-runs pick up where they stopped)
|
||||
os.makedirs(os.path.dirname(dst), exist_ok=True)
|
||||
temporary = dst + ".part"
|
||||
g = open(temporary, "wb")
|
||||
try:
|
||||
with g:
|
||||
nup.copy_to(r, g if quiet else ProgressSink(g, progress), rel)
|
||||
sink = HeadSink(g) if looked else g
|
||||
nup.copy_to(r, sink if quiet else ProgressSink(sink, progress), rel)
|
||||
os.replace(temporary, dst)
|
||||
if looked:
|
||||
zs["files"] += 1
|
||||
if not zstd_start(sink.head):
|
||||
zs["damaged"].append(rel)
|
||||
except BaseException as error:
|
||||
full = out_of_space(error, dst, part, rel, r[2]) # (measured before the member's space is freed)
|
||||
# Free the failed member's space without hiding the extraction error.
|
||||
@@ -583,11 +618,16 @@ def stand_in_for(path, damaged=()):
|
||||
return best and best[1]
|
||||
|
||||
|
||||
def stand_in_damaged(romfs, progress=None):
|
||||
def stand_in_damaged(romfs, progress=None, known=None):
|
||||
"""Each damaged .zs file in an unpacked romfs replaced by a copy of its stand-in: (damaged, stand-in or None) each.
|
||||
Only for a romfs this tool unpacked: the player's own folder is never written to. progress: damaged_game_files'"""
|
||||
Only for a romfs this tool unpacked: the player's own folder is never written to. progress: damaged_game_files'.
|
||||
known: the damaged files as the unpack itself found them (extract's zs, paths under romfs with "/"): they are taken
|
||||
as they are and no file is opened to look for them"""
|
||||
done = []
|
||||
damaged = damaged_game_files(romfs, progress)
|
||||
if known is None:
|
||||
damaged = damaged_game_files(romfs, progress)
|
||||
else:
|
||||
damaged = [os.path.normpath(os.path.join(romfs, *rel.split("/"))) for rel in known]
|
||||
for path in damaged:
|
||||
other = stand_in_for(path, set(damaged))
|
||||
if other:
|
||||
@@ -642,8 +682,13 @@ def main():
|
||||
folder = parent
|
||||
free = shutil.disk_usage(folder).free
|
||||
print("extracting title %d (v%d) to %s (%.0f GB free)" % (t["index"], t["version"], a.extract, free / 1e9))
|
||||
n, total = extract(nup, t, a.extract, a.only)
|
||||
zs = {}
|
||||
n, total = extract(nup, t, a.extract, a.only, zs=zs)
|
||||
print("done: %d files written, %.2f GB" % (n, total / 1e9))
|
||||
if a.only != "exefs": # (for nx2/make_game.py: the romfs .zs files as they were seen while they were written)
|
||||
import json
|
||||
with open(os.path.join(long_path(os.path.realpath(os.path.abspath(a.extract))), ZS_CHECKED), "w", encoding="utf-8") as f:
|
||||
json.dump(zs, f)
|
||||
if a.only != "romfs": # (main is ZBIC-compressed on firmware 22+: made a plain NSO for the tools)
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import unpack
|
||||
|
||||
@@ -19,6 +19,7 @@ Each is a runtime switch where it can be, so the game's own path stays reachable
|
||||
"""
|
||||
import glob
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import whole_write # (rewrites written beside and moved over: issue 1197)
|
||||
|
||||
@@ -176,6 +177,27 @@ def patch_menu_tick(source):
|
||||
'#include "a64.h"\n', '#include "a64.h"\nextern void hos_oot_menu_tick(Ctx *c, uint64_t screen);\n', 1)
|
||||
|
||||
|
||||
MOD_SIZE_CALL = "c->x[30] = 0x801b64e4ULL; f_836e3d30(c);\n"
|
||||
MOD_SIZE_HOOK = (" c->x[0] = hos_mods_table_size(c->x[0], recomp_load64(c, c->x[29] + 24));"
|
||||
" /* (nx2/oot_patches.py: room for the mods' files, host/mods.c) */\n")
|
||||
|
||||
|
||||
def patch_mod_sizes(source):
|
||||
"""((Triforce) 11-17) f_801b6340 asks the ResourceSizeTable (RESTBL, f_836e3d30: the CRC of the path, a binary search) how
|
||||
much memory a file's load needs; the file's resource heap is made that size (f_801b6840). A mod's file served instead of the
|
||||
game's (host/mods.c) can need more than the table gives it: a model with more materials and shapes than the original fails
|
||||
its gsys::ModelResource allocations, which the game does not check (the Dark Link mod's Deku Shield and Slingshot: a write
|
||||
to address 0 on loading a child save). The host raises the size for a mod's file to its real size and room beside it
|
||||
(hos_mods_table_size); the game's own files keep the table's"""
|
||||
if MOD_SIZE_CALL not in source:
|
||||
return source, 0
|
||||
if "hos_mods_table_size(" in source: # (made before: a lift patched again)
|
||||
return source, 1
|
||||
out = source.replace(MOD_SIZE_CALL, MOD_SIZE_CALL + MOD_SIZE_HOOK, 1)
|
||||
out = out.replace('#include "a64.h"\n', '#include "a64.h"\nextern uint64_t hos_mods_table_size(uint64_t size, uint64_t path);\n', 1)
|
||||
return out, 1
|
||||
|
||||
|
||||
TEXT_HEAD = "void f_80b1e140(Ctx *__restrict A64_NOESC c) {"
|
||||
# (the letters' block: past the early returns, the auto-advance timer and the hidden / no-text checks; reached once a call)
|
||||
TEXT_OLD = ("L_80b1e1b4:;\n"
|
||||
@@ -558,6 +580,78 @@ def patch_sprint(s):
|
||||
return s, done
|
||||
|
||||
|
||||
# (Triforce) The host's zstd decoder (host/host_zstd.h has what the two functions are and the rules; HOS_HOST_ZSTD, off unless
|
||||
# set). The game's own zstd, linked into main: f_8010b380 (main+0xeb380) is ZSTD_decompressMultiFrame, f_80115c80 (main+0xf5c80)
|
||||
# the stream feed its loading threads decode through. At the head of each the host is asked first: it answers the call (x0 the
|
||||
# result, the function returns) or says 0 and the game's code runs as it is. hos_host_zstd is 0 once the switch was read as off,
|
||||
# and the call is no longer made. A switch at run time, so that it takes no new translation to turn on or off.
|
||||
# Only the supported build's: the heads are found by address, and by lines only these two functions have (their shape below), so
|
||||
# a function of that name with other code (another game version, a changed lift) is left as it is.
|
||||
# (Triforce) The shapes are matched with the store marks' spellings taken out (_unmarked, below). The page marks of a group of
|
||||
# stores (a64_lift.mark_groups) are made after these patches (nx2/chunks.py edit_c): a store of a group becomes
|
||||
# recomp_storeN_sp( and an A64_MARK line follows the group. The lift as it stands on disk has them, these patches see the text
|
||||
# before them. The stream feed's begin stores (+48 and +0 through x19) are such a group: its shape, read from the lift on disk,
|
||||
# was never in the text the patch saw, and every build said "found at 1 of 2 places" (the stream feed left out). The lines are
|
||||
# written as the text is before the marks, the text the patch sees (tests/test_host_zstd.py holds them to that).
|
||||
ZSTD_SITES = (
|
||||
("void f_8010b380(Ctx *__restrict A64_NOESC c) {\n", "hos_zstd_multiframe",
|
||||
# (the eighth argument kept and tested, and the dictionary read from a DDict's +8 and +16)
|
||||
(" recomp_store64_sp(c,c->x[31] + 104,c->x[7]);\nif (c->x[7] != 0) { goto L_8010fd78; }\n",
|
||||
"L_8010fd78:;\n c->x[8]=recomp_load64(c,c->x[31] + 104);\n"
|
||||
" { uint64_t _b=c->x[8]; int64_t _o=8; { uint64_t _p0,_p1; recomp_ldp64(c,(_b+_o),&_p0,&_p1); c->x[5]=_p0; c->x[6]=_p1; } }\n")),
|
||||
("void f_80115c80(Ctx *__restrict A64_NOESC c) {\n", "hos_zstd_stream",
|
||||
# (the phase read from the state, the "done" result, the error code asked of zstd, and the stores of a begin. (Triforce)
|
||||
# And every field host/host_zstd.h reads or writes, at its offset: the output and its room (+8, +16) read in phase 2, the
|
||||
# DCtx's "bytes expected next" (HZ_DCTX_EXPECTED, 28776) read first there, the workspace (+24) read at the begin before
|
||||
# the content size goes to +20; the DCtx to +48 and the phase to +0 at its end)
|
||||
(" c->x[8]=recomp_load8(c,c->x[0] + 0);\n",
|
||||
" c->x[0] = 0xfffffffdULL;\n",
|
||||
"c->x[30] = 0x8011629cULL; f_80d5ba00(c);\n",
|
||||
" c->x[22]=recomp_load64(c,c->x[21] + 28776);\n",
|
||||
" c->x[23]=recomp_load64(c,c->x[19] + 8);\n"
|
||||
" { uint64_t _r = (uint64_t)(int64_t)(int32_t)recomp_load32(c,c->x[19] + 16); c->x[2] = _r; }\n",
|
||||
" c->x[21]=recomp_load64(c,c->x[19] + 24);\n recomp_store32(c,c->x[19] + 20,c->x[8]);\n",
|
||||
" recomp_store64(c,c->x[19] + 48,c->x[21]);\n recomp_store8(c,c->x[19] + 0,c->x[8]);\n")),
|
||||
)
|
||||
ZSTD_HOOK = " if (hos_host_zstd && %s(c)) return; /* (nx2/oot_patches.py: the host's zstd decoder, HOS_HOST_ZSTD) */\n"
|
||||
ZSTD_DECL = ("extern volatile int hos_host_zstd;\nextern int hos_zstd_multiframe(Ctx *c);\n"
|
||||
"extern int hos_zstd_stream(Ctx *c);\n")
|
||||
# (a64_lift.MG_MARK's line, and the _sp spelling of a marked store: mark_groups_lines, stack_stores, frame_stores)
|
||||
MARK_LINE = re.compile(r"^ A64_MARK\(\(c->x\[\d+\] \+ \(uint64_t\)\(int64_t\)\(-?\d+\)\), \d+\);\n", re.M)
|
||||
MARK_SP = re.compile(r"\brecomp_(store8|store16|store32|store64|stp32|stp64)_sp\(")
|
||||
|
||||
|
||||
def _unmarked(text):
|
||||
"""(Triforce) text without the store marks' spellings: a group's A64_MARK lines out, recomp_storeN_sp( as recomp_storeN(.
|
||||
The code's shape alone, the same before a64_lift.mark_groups and after it (a second run over a finished lift), and with
|
||||
A64_MARK_EACH, A64_MARK_STACK or A64_MARK_FRAME at lift time. They change only where a store's page is marked, never what
|
||||
the guest's code does, so a shape can count on everything else"""
|
||||
return MARK_SP.sub(r"recomp_\1(", MARK_LINE.sub("", text))
|
||||
|
||||
|
||||
def patch_host_zstd(s):
|
||||
"""The host decoder's two calls: (s, how many of the two heads are in s with their call, made now or before). A function
|
||||
whose code is not the decoder's (another game version) is left as it is."""
|
||||
done = 0
|
||||
for head_text, name, shape in ZSTD_SITES:
|
||||
at = _function(s, head_text)
|
||||
if at is None:
|
||||
continue
|
||||
head, end = at
|
||||
body = s[head:end]
|
||||
hook = ZSTD_HOOK % name
|
||||
if body.startswith(head_text + hook):
|
||||
done += 1
|
||||
elif hook not in body:
|
||||
plain = _unmarked(body)
|
||||
if all(plain.count(_unmarked(line)) == 1 for line in shape):
|
||||
s = s[:head] + head_text + hook + body[len(head_text):] + s[end:]
|
||||
done += 1
|
||||
if done and ZSTD_DECL not in s:
|
||||
s = s.replace('#include "a64.h"\n', '#include "a64.h"\n' + ZSTD_DECL, 1)
|
||||
return s, done
|
||||
|
||||
|
||||
def patch_text_speed(s):
|
||||
"""The scale at the head of the letters' block of f_80b1e140; None when the file has no such function, s unchanged when the
|
||||
function is there but its code differs (another game version)."""
|
||||
@@ -616,6 +710,8 @@ def apply(gen, *, protect_hearts=False, required=False):
|
||||
sound_jobs = solver_sites = 0 # (patch_sound_job, patch_solver_steps: the places found with their hooks)
|
||||
floor_sites = sim_sites = record_sites = 0 # (patch_floor_carry, patch_sim_steps, patch_record_clock: the same)
|
||||
sprint_sites = 0 # (patch_sprint: its two calls)
|
||||
zstd_sites = 0 # (patch_host_zstd: its two calls, the supported build's alone)
|
||||
mod_size_sites = 0 # ((Triforce) patch_mod_sizes: its one call, the supported build's alone)
|
||||
for path in sorted(glob.glob(os.path.join(glob.escape(gen), "code_*.c"))):
|
||||
with open(path, encoding="utf-8", newline="") as f:
|
||||
s = f.read()
|
||||
@@ -721,6 +817,17 @@ def apply(gen, *, protect_hearts=False, required=False):
|
||||
if sprinted != out:
|
||||
print("oot_patches: the sprint button in", os.path.basename(path))
|
||||
out = sprinted
|
||||
if protect_hearts: # (found by address: the supported build's functions)
|
||||
zstd, found = patch_host_zstd(out)
|
||||
zstd_sites += found
|
||||
if zstd != out:
|
||||
print("oot_patches: the host's zstd decoder's call(s) in", os.path.basename(path))
|
||||
out = zstd
|
||||
sized, found = patch_mod_sizes(out)
|
||||
mod_size_sites += found
|
||||
if sized != out:
|
||||
print("oot_patches: room for the mods' files in", os.path.basename(path))
|
||||
out = sized
|
||||
if out != s:
|
||||
whole_write.write_whole(path, out, newline="")
|
||||
if sound_jobs != 1: # (without it, above 60 fps sounds and then the music can stop after a few loading zones)
|
||||
@@ -736,6 +843,10 @@ def apply(gen, *, protect_hearts=False, required=False):
|
||||
% (record_sites, sum(n for _, n in RECORD_SITES)))
|
||||
if sprint_sites != 2: # (without them the Sprint button row does nothing)
|
||||
print("oot_patches: WARNING: the sprint button's calls were found at %d of 2 places" % sprint_sites)
|
||||
if protect_hearts and mod_size_sites != 1: # (without it a mod's bigger model can crash the game: host/mods.c)
|
||||
print("oot_patches: WARNING: the resource size table's lookup was found at %d of 1 place" % mod_size_sites)
|
||||
if protect_hearts and zstd_sites != len(ZSTD_SITES): # (without them HOS_HOST_ZSTD does nothing, or half: host/hos_zstd.c)
|
||||
print("oot_patches: WARNING: the host's zstd decoder's calls were found at %d of %d places" % (zstd_sites, len(ZSTD_SITES)))
|
||||
for what, n in done.items():
|
||||
if n != 1:
|
||||
print("oot_patches: WARNING: the %s was found %d times" % (what, n))
|
||||
|
||||
@@ -18,10 +18,11 @@ MAIN = 0x80020000 # (main's base in this image: nx2/eden_image.py)
|
||||
UI_M00, UI_M11 = "(2.0f / 3840.0f)", "(2.0f / 2160.0f)" # (the UI's full-screen orthographic projection, 3840x2160 grid)
|
||||
# (the screens whose UI keeps the whole picture: the item-wheel and pause / system menu backgrounds redraw a capture of
|
||||
# the whole picture; the event and loading fades cover it. Bit k of host/hos_main.c's hos_wide_keep_mask keeps screen k
|
||||
# whole; the Insert menu's Ultrawide menus row clears the menus' bits 1 and 3 (vk_framegen.cpp): their layout then
|
||||
# whole; the Insert menu's Ultrawide menus row clears the menus' bit 1 (vk_framegen.cpp): their layout then
|
||||
# keeps its 16:9 shape in the middle, as the HUD's does, instead of being stretched over the whole picture. Bit 0, the
|
||||
# item wheel's background (the last frame of play, blurred, HUD and all), and bit 2, the fades, keep covering it all: bit 0
|
||||
# cleared, that capture went into the middle at twice its width)
|
||||
# item wheel's background (the last frame of play, blurred, HUD and all), and bits 2 and 3, the two fades, keep covering
|
||||
# it all: bit 0 cleared, that capture went into the middle at twice its width, and a fade in the middle alone left the
|
||||
# picture showing at its sides while a place loaded)
|
||||
KEEP_VTABLES = (MAIN + 0x4bed098, MAIN + 0x4be4fb0, MAIN + 0x4be61c8, MAIN + 0x4be7b30)
|
||||
|
||||
|
||||
|
||||
@@ -3,12 +3,14 @@ and Kenney's Input Prompts (CC0, third_party/kenney_input).
|
||||
|
||||
python -I nx2/prompt_assets.py --romfs <dir> --out <game folder>/prompts
|
||||
|
||||
Writes <out>/<face>/External.bfotf for each face (switch, xbox, ps4, ps5, steamdeck). Every text in the game draws its button icons
|
||||
Writes <out>/<face>/External.bfotf for each face (switch, xbox, ps4, ps5, steamdeck, keyboard). Every text in the game draws its button icons
|
||||
from Font2/Scalable/External.*.bfotf (dialogue, the control hints, menus): A B X Y at U+E040..E043, L R ZL ZR at E044..E047, Plus
|
||||
and Minus at E04B and E04C, the stick presses at E056 and E057, the d-pad at E060..E063. Each written font has one icon per pad
|
||||
input at U+F700 + HOS_PAD_* (gpu/vk/vk_api.h), and each of those Switch buttons' code points in a cmap segment of its own: the
|
||||
game (host/prompts.c) points each one at the icon of the pad input bound to that button when it serves the font, so the icons
|
||||
follow the controls, changed ones too.
|
||||
follow the controls, changed ones too. (Triforce) The keyboard face (the keyboard and mouse; nx2/prompt_keys.py draws it, from
|
||||
nothing but a font of the repository's) has an icon per key that can be bound instead, at U+F700 + its key code, and what is
|
||||
said of a pad input below is a key code's there.
|
||||
|
||||
The textures with button icons (the ocarina's notes, the letters on black, the controller in the pause menu) the same way, with
|
||||
Pillow, astc_encoder and texture2ddecoder: <out>/Common.bntx is the game's UI/Layout2Archive/Common bntx decompressed, and
|
||||
@@ -78,10 +80,19 @@ def zs_dictionary(romfs, which=b"zs.zsdic"):
|
||||
|
||||
|
||||
def read_zs(romfs, rel, dictionary):
|
||||
"""(Triforce, as NX2navi's tools/pad_prompts.py unpack) The file goes to the decompressor a piece at a time and the
|
||||
pieces' outputs are joined once. Given the whole file in one call the library grows one bytes object by copying as the
|
||||
output comes, and the textures (69 MB packed, 244 MB unpacked) took 20 s of this tool's 80 that way"""
|
||||
import zstandard
|
||||
d = zstandard.ZstdDecompressor(dict_data=zstandard.ZstdCompressionDict(dictionary))
|
||||
d = zstandard.ZstdDecompressor(dict_data=zstandard.ZstdCompressionDict(dictionary)).decompressobj()
|
||||
with open(os.path.join(romfs, rel), "rb") as f:
|
||||
return d.decompressobj().decompress(f.read())
|
||||
raw = f.read()
|
||||
parts = []
|
||||
for at in range(0, len(raw), 1 << 16):
|
||||
parts.append(d.decompress(raw[at:at + (1 << 16)]))
|
||||
if d.eof:
|
||||
break
|
||||
return b"".join(parts)
|
||||
|
||||
|
||||
def size_table(romfs, dictionary, game_size, font_size):
|
||||
@@ -415,43 +426,56 @@ def face_layers(b, tex, face, otf):
|
||||
"""this face's images, as layers (in textures.bin's order), each a hand-made one if nx2/prompt_faces/<face>/ has it"""
|
||||
from PIL import Image, ImageDraw
|
||||
import prompt_draw
|
||||
icons = Icons(face, otf)
|
||||
keys = bool(FACES.get(face, {}).get("keys")) # ((Triforce) the keyboard and mouse: an icon per key code, no controller)
|
||||
if keys:
|
||||
import prompt_keys
|
||||
icons = prompt_keys.Icons() if keys else Icons(face, otf)
|
||||
# (what a Switch button can be on: its name in icons, its number in textures.bin, its name in a layer's)
|
||||
inputs = [(vk, vk, "key%d" % vk) for vk in sorted(icons.code)] if keys else [(p, i, p) for i, p in enumerate(PADS)]
|
||||
layers = []
|
||||
made = {} # ((Triforce) an input's image is the same on every texture of one size: made once, and encoded once)
|
||||
|
||||
def keep(sw, pad): # (the Switch face's default controls: the game's own textures)
|
||||
return pad in icons.code and not (face == "switch" and DEFAULT[SW[sw]] == pad)
|
||||
|
||||
for name, sw in NOTES.items():
|
||||
t = tex[name]
|
||||
w, h = t[2], t[3]
|
||||
for pad in PADS:
|
||||
if not keep(sw, pad):
|
||||
continue
|
||||
def note(pad, w, h):
|
||||
if ("note", pad, w, h) not in made:
|
||||
icon, shape, holes = icons.masks(pad, int(min(w, h) * 0.9))
|
||||
im = Image.new("RGBA", (w, h), (0, 0, 0, 0))
|
||||
o = ((w - icon.width) // 2, (h - icon.height) // 2)
|
||||
im.paste(Image.new("RGBA", icon.size, (255, 255, 255, 255)), o, shape)
|
||||
im.paste(Image.new("RGBA", icon.size, (0, 0, 0, 255)), o, holes)
|
||||
layers.append(Layer(f"{name}.{pad}.png", name, SW[sw], PADS.index(pad), im))
|
||||
for name, sw in LETTERS.items():
|
||||
t = tex[name]
|
||||
w, h = t[2], t[3]
|
||||
for pad in PADS:
|
||||
if not keep(sw, pad):
|
||||
continue
|
||||
made["note", pad, w, h] = im
|
||||
return made["note", pad, w, h]
|
||||
|
||||
def letter_of(pad, w, h):
|
||||
if ("letter", pad, w, h) not in made:
|
||||
icon, shape, holes = icons.masks(pad, int(min(w, h) * 0.9))
|
||||
letter = holes if holes.getbbox() else icon # (an icon with no letter in it: its shape)
|
||||
# (an icon with no letter in it: its shape; a mouse's: the mouse, not what its outline leaves)
|
||||
letter = holes if holes.getbbox() and pad not in getattr(icons, "plain", ()) else icon
|
||||
im = Image.new("L", (w, h), 0)
|
||||
im.paste(letter, ((w - letter.width) // 2, (h - letter.height) // 2))
|
||||
layers.append(Layer(f"{name}.{pad}.png", name, SW[sw], PADS.index(pad), im))
|
||||
made["letter", pad, w, h] = im
|
||||
return made["letter", pad, w, h]
|
||||
|
||||
for name, sw in NOTES.items():
|
||||
t = tex[name]
|
||||
for pad, at, label in inputs:
|
||||
if keep(sw, pad):
|
||||
layers.append(Layer(f"{name}.{label}.png", name, SW[sw], at, note(pad, t[2], t[3])))
|
||||
for name, sw in LETTERS.items():
|
||||
t = tex[name]
|
||||
for pad, at, label in inputs:
|
||||
if keep(sw, pad):
|
||||
layers.append(Layer(f"{name}.{label}.png", name, SW[sw], at, letter_of(pad, t[2], t[3])))
|
||||
own = None
|
||||
if face != "switch":
|
||||
if face != "switch" and not keys: # ((Triforce) the keyboard has no picture: the page keeps the game's controller)
|
||||
hand = override(face, PICTURES[0] + ".png")
|
||||
own = Image.open(hand).convert("RGBA") if hand else prompt_draw.picture(
|
||||
face, decode(b, tex[PICTURES[0]]).convert("RGBA"), Icons("switch", otf))
|
||||
if own:
|
||||
layers += picture_layers(b, tex, own)
|
||||
for name in PICTURES if face != "switch" and not own else ():
|
||||
for name in PICTURES if face != "switch" and not own and not keys else ():
|
||||
t = tex[name]
|
||||
im = decode(b, t)
|
||||
px = im.load()
|
||||
@@ -480,7 +504,7 @@ def face_layers(b, tex, face, otf):
|
||||
def face_textures(b, tex, layers):
|
||||
"""textures.bin's entries for these layers: [(Switch button, pad input, runs)], Switch button 0xff for those always written"""
|
||||
from PIL import ImageChops
|
||||
entries, made = [], {}
|
||||
entries, made, encoded = [], {}, {}
|
||||
for layer in layers:
|
||||
t, im = tex[layer.texture], layer.im
|
||||
made[layer.name] = im
|
||||
@@ -493,7 +517,13 @@ def face_textures(b, tex, layers):
|
||||
x0, y0, x1, y1 = min(x0, bx0), min(y0, by0), max(x1, bx1), max(y1, by1)
|
||||
entries.append((layer.sw, layer.pad, runs(t, im, (x0, y0, x1, y1))))
|
||||
else:
|
||||
entries.append((layer.sw, layer.pad, runs(t, im)))
|
||||
# ((Triforce) the same image on another texture of that size and format, a note or a letter on another Switch
|
||||
# button: the same blocks, at that texture's place. The notes were encoded six times over, the letters four)
|
||||
same = (id(im), t[1], t[2], t[3], t[4])
|
||||
if same not in encoded:
|
||||
encoded[same] = (t[5], runs(t, im))
|
||||
data, rs = encoded[same]
|
||||
entries.append((layer.sw, layer.pad, [[off - data + t[5], piece] for off, piece in rs]))
|
||||
return entries
|
||||
|
||||
|
||||
@@ -874,12 +904,15 @@ def hint_panes(blarc):
|
||||
return out or None
|
||||
|
||||
|
||||
def write_hint(path, otf, panes):
|
||||
def write_hint(path, otf, panes, names=None):
|
||||
"""(names: the glyph of each input's icon, in the inputs' order, None for a number that is no input: the pad inputs',
|
||||
or the keyboard face's 256 key codes')"""
|
||||
from fontTools.ttLib import TTFont
|
||||
hmtx = TTFont(io.BytesIO(otf))["hmtx"]
|
||||
names = names or [f"pad_{p}" for p in PADS]
|
||||
with open(path, "wb") as f:
|
||||
f.write(b"HPHN" + struct.pack("<I", len(PADS)))
|
||||
f.write(b"".join(struct.pack("<f", round(max(0, hmtx[f"pad_{p}"][0] - HINT_FITS) * HINT_SCALE)) for p in PADS))
|
||||
f.write(b"HPHN" + struct.pack("<I", len(names)))
|
||||
f.write(b"".join(struct.pack("<f", round(max(0, hmtx[n][0] - HINT_FITS) * HINT_SCALE) if n else 0.0) for n in names))
|
||||
f.write(struct.pack("<I", len(panes)) + b"".join(struct.pack("<IB3xf", at, slot, x) for at, slot, x in panes))
|
||||
|
||||
|
||||
@@ -1074,10 +1107,23 @@ def main():
|
||||
for face in a.faces.split(","):
|
||||
d = os.path.join(a.out, face)
|
||||
os.makedirs(d, exist_ok=True)
|
||||
plain = face_font(otf, face)
|
||||
names = None
|
||||
if FACES.get(face, {}).get("keys"): # ((Triforce) the keyboard and mouse: its icons are drawn, nx2/prompt_keys.py)
|
||||
try:
|
||||
import prompt_keys
|
||||
plain, binds = prompt_keys.game_font(otf)
|
||||
except (Exception, SystemExit) as e: # (tools without the labels' font, a game font it cannot take: the
|
||||
# controllers' faces are made all the same, and the game shows its own icons for the keyboard)
|
||||
print(f"prompts: {face} was not made ({e})", file=sys.stderr)
|
||||
os.rmdir(d)
|
||||
continue
|
||||
prompt_keys.write_binds(os.path.join(d, "keys.bin"), binds)
|
||||
names = prompt_keys.hint_names()
|
||||
else:
|
||||
plain = face_font(otf, face)
|
||||
font = wrap(key, plain)
|
||||
if hint:
|
||||
write_hint(os.path.join(d, "hint.bin"), plain, hint)
|
||||
write_hint(os.path.join(d, "hint.bin"), plain, hint, names)
|
||||
largest = max(largest, len(font))
|
||||
with open(os.path.join(d, "External.bfotf"), "wb") as f:
|
||||
f.write(font)
|
||||
|
||||
@@ -21,6 +21,9 @@ Each face:
|
||||
line ends on it), a button's centre and radius (its line ends just off the edge it comes from), a shoulder's top
|
||||
edge (r 0). The four face buttons are framed. Not a pad input's: boxes the lines keep off (x0, y0, x1, y1: a
|
||||
screen, a touchpad). SWITCH_PARTS: the game's Pro Controller's.
|
||||
keys (Triforce) True for the keyboard and mouse, which is no controller and has none of the above: its icons are one
|
||||
per key that can be bound, drawn by nx2/prompt_keys.py (the keys, their labels and what each code point of the
|
||||
font shows are there), and the host picks them by the keys bound instead of the pad inputs.
|
||||
"""
|
||||
import os
|
||||
|
||||
@@ -96,6 +99,10 @@ FACES = {
|
||||
icons=dict(_DPAD, south="button_a", east="button_b", west="button_x", north="button_y", back="button_view",
|
||||
start="button_options", lshoulder="button_l1", rshoulder="button_r1", ltrigger="button_l2",
|
||||
rtrigger="button_r2")),
|
||||
# (Triforce) the keyboard and mouse: no controller, so no font of Kenney's, no drawing and no parts. keys: its icons are
|
||||
# the keys' and the mouse's buttons', one per key that can be bound, drawn by nx2/prompt_keys.py; the controls page keeps
|
||||
# the game's own controller picture
|
||||
"keyboard": dict(keys=True),
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -23,6 +23,13 @@ The Switch face keeps the game's own images for its default controls; only the o
|
||||
Pad inputs: `south east west north back guide start lstick rstick lshoulder rshoulder up down left right ltrigger rtrigger`.
|
||||
Switch buttons: `a b x y ls rs l r zl zr plus minus left up right down`.
|
||||
|
||||
The `keyboard` face (the keyboard and mouse) is drawn by `nx2/prompt_keys.py` from nothing but Libertinus Sans Bold
|
||||
(`third_party/fonts`): a cap with its label for every key that can be bound, a mouse for each mouse button. Its notes and
|
||||
letters are named by the key's code where the others have a pad input, `OcarinaButtonA_00.key69.png` for E, and a PNG of
|
||||
that name in `nx2/prompt_faces/keyboard/` takes the image's place as for any face. It has no controller picture and no
|
||||
leader lines: the controls page keeps the game's. `python -I nx2/prompt_keys.py --out <dir>` writes every icon into one
|
||||
picture, to look at.
|
||||
|
||||
## Drawing one by hand
|
||||
|
||||
1. Write out what the build makes (and the game's own textures, to draw over):
|
||||
|
||||
500
nx2/prompt_keys.py
Normal file
500
nx2/prompt_keys.py
Normal file
@@ -0,0 +1,500 @@
|
||||
"""(Triforce) The keyboard and mouse face of the button prompts: the game's button icons showing the key or the mouse button
|
||||
each control is bound to (NX2navi's prompts = pc; nx2/prompt_assets.py builds it with the controllers' faces, host/prompts.c
|
||||
serves it).
|
||||
|
||||
Nothing is read for it but a font the repository carries: a key is a cap drawn here, a rounded square (wider for a word) with
|
||||
its label cut out of it in Libertinus Sans Bold (third_party/fonts/libertinus, SIL OFL: the font the menus fall back on); an
|
||||
arrow key's label is an arrow; a mouse button is a mouse with that button filled; the camera is a mouse between arrows.
|
||||
|
||||
The controls can be changed (the Controls page), so the font has an icon for every key that can be bound, at U+F700 + its
|
||||
Windows key code (KEYS: the keys of gpu/vk/vk_menu.cpp bindable_keys), and the host points each button's code point at the
|
||||
icon of the first key bound to it when it serves the font, as it does with a pad's inputs. What the font's own code points
|
||||
show:
|
||||
|
||||
the Switch's buttons (prompt_assets.SWITCH) the key of each (DEFAULT: host/nn_hid.c key_default's first keys)
|
||||
the left stick's four directions (MOVES) the movement keys
|
||||
the left stick, its two axes, the d-pad and its those keys side by side, as they lie on the keyboard (CLUSTERS)
|
||||
two axes
|
||||
the right stick and its directions (CAMERA) the mouse, with arrows for the way it moves
|
||||
|
||||
An icon of several keys cannot be one glyph per choice of keys. It is a glyph that calls a subroutine per key, and each key's
|
||||
small cap is such a subroutine (a local one, as the game's font already has: mini). <face>/keys.bin lists where those calls
|
||||
are in the font and each key's call, and the host writes the calls of the keys bound: "HPKB", u32 count, then count times
|
||||
u32 offset (in the font, without its 8-byte header), u8 control (HOS_KEY_*), u8 0 x3; then u32 256 and for each key code the
|
||||
two bytes of its call (0 0: no such key, the blank cap's).
|
||||
|
||||
python -I nx2/prompt_keys.py --out <dir> (every icon as it is drawn, in one PNG, to look at; needs Pillow)
|
||||
"""
|
||||
import io
|
||||
import math
|
||||
import os
|
||||
import struct
|
||||
import sys
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, HERE)
|
||||
import prompt_assets as P # noqa: E402
|
||||
|
||||
LETTERS = os.path.join(os.path.dirname(HERE), "third_party", "fonts", "libertinus", "LibertinusSans-Bold.ttf")
|
||||
BASE = P.PAD_BASE # (a key's icon is at BASE + its key code, as a pad input's is at BASE + HOS_PAD_*)
|
||||
|
||||
# every key a control can be bound to, by its Windows key code, and its label: a line, two ("Num" small over the key), or a
|
||||
# drawing ("@up" an arrow, "@mouse_l" the mouse with its left button filled). F1 is the menu's.
|
||||
KEYS = {ord(c): c for c in "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"}
|
||||
KEYS.update({0x70 + n - 1: "F%d" % n for n in range(2, 13)})
|
||||
KEYS.update({0x60 + n: "Num\n%d" % n for n in range(10)})
|
||||
KEYS.update({
|
||||
0x20: "Space", 0x10: "Shift", 0x11: "Ctrl", 0x12: "Alt", 0x09: "Tab", 0x0D: "Enter", 0x08: "Bksp", 0x14: "Caps",
|
||||
0x26: "@up", 0x28: "@down", 0x25: "@left", 0x27: "@right", 0x24: "Home", 0x23: "End", 0x21: "PgUp", 0x22: "PgDn",
|
||||
0x6B: "Num\n+", 0x6D: "Num\n-", 0x6A: "Num\n*", 0x6F: "Num\n/", 0x6E: "Num\n.",
|
||||
0xBA: ";", 0xBB: "=", 0xBC: ",", 0xBD: "-", 0xBE: ".", 0xBF: "/", 0xC0: "`", 0xDB: "[", 0xDC: "\\", 0xDD: "]", 0xDE: "'",
|
||||
0x01: "@mouse_l", 0x02: "@mouse_r", 0x04: "@mouse_m", 0x05: "@mouse_4", 0x06: "@mouse_5"})
|
||||
# the first key of each control out of the box, in HOS_KEY_* order (host/nn_hid.c key_default): the Switch's buttons A B X Y
|
||||
# LS RS L R ZL ZR Plus Minus Left Up Right Down, then move up, down, left, right and camera up, down, left, right
|
||||
DEFAULT = [0x45, 0x10, 0x20, 0x01, 0x11, 0x43, 0x51, 0x52, 0x5A, 0x58, 0x09, 0x4D, 0x31, 0x32, 0x33, 0x34, 0x57, 0x53, 0x41,
|
||||
0x44, 0x26, 0x28, 0x25, 0x27]
|
||||
MOVE_UP, MOVE_DOWN, MOVE_LEFT, MOVE_RIGHT = 16, 17, 18, 19
|
||||
PAD_LEFT, PAD_UP, PAD_RIGHT, PAD_DOWN = 12, 13, 14, 15
|
||||
# the left stick pushed one way: that movement key (the font's code point: the control)
|
||||
MOVES = {0xE052: MOVE_UP, 0xE050: MOVE_DOWN, 0xE05A: MOVE_LEFT, 0xE058: MOVE_RIGHT}
|
||||
# the icons of several keys: each key's control and its place (column, row from the top)
|
||||
CLUSTERS = {
|
||||
0xE05E: [(MOVE_UP, 1, 0), (MOVE_LEFT, 0, 1), (MOVE_DOWN, 1, 1), (MOVE_RIGHT, 2, 1)], # (the left stick)
|
||||
0xE05C: [(MOVE_UP, 0, 0), (MOVE_DOWN, 0, 1)], 0xE054: [(MOVE_LEFT, 0, 0), (MOVE_RIGHT, 1, 0)],
|
||||
0xE049: [(PAD_UP, 1, 0), (PAD_LEFT, 0, 1), (PAD_DOWN, 1, 1), (PAD_RIGHT, 2, 1)], # (the d-pad)
|
||||
0xE06A: [(PAD_UP, 1, 0), (PAD_LEFT, 0, 1), (PAD_DOWN, 1, 1), (PAD_RIGHT, 2, 1)],
|
||||
0xE064: [(PAD_UP, 0, 0), (PAD_DOWN, 0, 1)], 0xE065: [(PAD_LEFT, 0, 0), (PAD_RIGHT, 1, 0)]}
|
||||
# the right stick is the camera, which the mouse turns: the mouse and the ways it moves (the camera's keys, a second way
|
||||
# to turn it, do not show)
|
||||
CAMERA = {0xE05F: "xy", 0xE068: "xy", 0xE053: "y", 0xE051: "y", 0xE05D: "y", 0xE059: "x", 0xE05B: "x", 0xE055: "x"}
|
||||
MICE = {vk for vk, label in KEYS.items() if label.startswith("@mouse")} # (their icons are no caps)
|
||||
|
||||
# ---- the drawings, in units of their own: a cap is H high, its corners R round, a word's as wide as the word up to
|
||||
# P.WIDE caps; a mouse MOUSE wide. A contour that is ink runs counter-clockwise; a label is cut out of its cap by running
|
||||
# the other way, which is the way Libertinus's outlines run (TrueType's)
|
||||
H, R, MOUSE, STROKE = 1000, 180, 640, 80
|
||||
KAPPA = 0.5523 # (a quarter circle's control points, of its radius)
|
||||
MINI, MINI_R, MINI_GAP = 290, 52, 20 # (a cap in an icon of several keys, the game's font's units: two rows fill an icon)
|
||||
MINI_MARK = [(16, 30), (30, 30), (30, 44), (16, 44)] # (a small cap's last contour, which ends at its last point: see mini)
|
||||
|
||||
|
||||
def rounded(pen, x0, y0, x1, y1, r):
|
||||
"""a rounded rectangle, counter-clockwise from (x0 + r, y0) (a side of no length, a round end's, is left out)"""
|
||||
k = r * (1 - KAPPA)
|
||||
at = (x0 + r, y0)
|
||||
pen.moveTo(at)
|
||||
for line, c1, c2, to in ((x1 - r, y0), (x1 - k, y0), (x1, y0 + k), (x1, y0 + r)), \
|
||||
((x1, y1 - r), (x1, y1 - k), (x1 - k, y1), (x1 - r, y1)), \
|
||||
((x0 + r, y1), (x0 + k, y1), (x0, y1 - k), (x0, y1 - r)), \
|
||||
((x0, y0 + r), (x0, y0 + k), (x0 + k, y0), (x0 + r, y0)):
|
||||
if line != at:
|
||||
pen.lineTo(line)
|
||||
pen.curveTo(c1, c2, to)
|
||||
at = to
|
||||
pen.closePath()
|
||||
|
||||
|
||||
def polygon(pen, points):
|
||||
pen.moveTo(points[0])
|
||||
for p in points[1:]:
|
||||
pen.lineTo(p)
|
||||
pen.closePath()
|
||||
|
||||
|
||||
def arc(pen, cx, cy, r, a0, a1):
|
||||
"""a circle's arc from angle a0 to a1 (degrees, counter-clockwise), from the pen's place at its start"""
|
||||
steps = max(1, math.ceil(abs(a1 - a0) / 90))
|
||||
for i in range(steps):
|
||||
t0, t1 = (math.radians(a0 + (a1 - a0) * j / steps) for j in (i, i + 1))
|
||||
k = 4 / 3 * math.tan((t1 - t0) / 4) * r
|
||||
pen.curveTo((cx + r * math.cos(t0) - k * math.sin(t0), cy + r * math.sin(t0) + k * math.cos(t0)),
|
||||
(cx + r * math.cos(t1) + k * math.sin(t1), cy + r * math.sin(t1) - k * math.cos(t1)),
|
||||
(cx + r * math.cos(t1), cy + r * math.sin(t1)))
|
||||
|
||||
|
||||
def arrow(pen, cx, cy, size, way):
|
||||
"""an arrow (counter-clockwise) pointing up, left, down or right, size long, about (cx, cy)"""
|
||||
a, b = size / 2, size / 6
|
||||
up = [(0, a), (-a, 0), (-b, 0), (-b, -a), (b, -a), (b, 0), (a, 0)]
|
||||
turn = {"up": (1, 0), "left": (0, 1), "down": (-1, 0), "right": (0, -1)}[way] # (cos, sin of a quarter turn each)
|
||||
polygon(pen, [(cx + x * turn[0] - y * turn[1], cy + x * turn[1] + y * turn[0]) for x, y in up])
|
||||
|
||||
|
||||
class Letters:
|
||||
"""Libertinus Sans Bold's outlines, for the labels"""
|
||||
|
||||
def __init__(self):
|
||||
from fontTools.ttLib import TTFont
|
||||
font = TTFont(LETTERS)
|
||||
self.glyphs, self.cmap, self.advance, self.cap = font.getGlyphSet(), font.getBestCmap(), font["hmtx"], font["OS/2"].sCapHeight
|
||||
|
||||
def ink(self, text):
|
||||
"""a line's ink, from its left to its right, in the font's units"""
|
||||
from fontTools.pens.boundsPen import BoundsPen
|
||||
x, lo, hi = 0, None, None
|
||||
for ch in text:
|
||||
g = self.cmap[ord(ch)]
|
||||
b = BoundsPen(self.glyphs)
|
||||
self.glyphs[g].draw(b)
|
||||
if b.bounds:
|
||||
lo = x + b.bounds[0] if lo is None else min(lo, x + b.bounds[0])
|
||||
hi = x + b.bounds[2] if hi is None else max(hi, x + b.bounds[2])
|
||||
x += self.advance[g][0]
|
||||
return lo or 0, hi or 0
|
||||
|
||||
def draw(self, pen, text, scale, x, y):
|
||||
"""a line from x on the baseline y, as the font's outlines run (clockwise); a glyph made of others as outlines too"""
|
||||
from fontTools.pens.recordingPen import DecomposingRecordingPen
|
||||
from fontTools.pens.transformPen import TransformPen
|
||||
for ch in text:
|
||||
g = self.cmap[ord(ch)]
|
||||
outline = DecomposingRecordingPen(self.glyphs)
|
||||
self.glyphs[g].draw(outline)
|
||||
outline.replay(TransformPen(pen, (scale, 0, 0, scale, x, y)))
|
||||
x += self.advance[g][0] * scale
|
||||
|
||||
|
||||
def lines_of(letters, label, h, room):
|
||||
"""a label's lines in a cap h high with room for the ink: [(text, scale, the ink's width, baseline from the cap's middle)].
|
||||
One letter is half the cap high, a word less, and what is wider than the room is made smaller (one letter has a square
|
||||
cap's room: a W's cap is no wider than an I's); of two lines the upper is the smaller ("Num")"""
|
||||
def line(text, cap_height, middle):
|
||||
s = cap_height / letters.cap
|
||||
lo, hi = letters.ink(text)
|
||||
fits = min(room, h * 0.64) if len(text) == 1 else room
|
||||
if (hi - lo) * s > fits:
|
||||
s = fits / (hi - lo)
|
||||
return text, s, (hi - lo) * s, middle - s * letters.cap / 2
|
||||
lines = label.split("\n")
|
||||
if len(lines) == 1:
|
||||
return [line(lines[0], h * (0.5 if len(lines[0]) == 1 else 0.4), 0)]
|
||||
return [line(lines[0], h * 0.2, h * 0.24), line(lines[1], h * 0.38, -h * 0.13)]
|
||||
|
||||
|
||||
def key_width(letters, vk):
|
||||
"""how wide a key's icon is (it is H high)"""
|
||||
label = KEYS.get(vk, "")
|
||||
if vk in MICE:
|
||||
return MOUSE
|
||||
if not label or label.startswith("@"):
|
||||
return H
|
||||
widest = max(w for _, _, w, _ in lines_of(letters, label, H, H * (P.WIDE - 0.3)))
|
||||
return int(min(H * P.WIDE, max(H, widest + H * 0.36)))
|
||||
|
||||
|
||||
def cap(pen, letters, label, w, h, r, room):
|
||||
"""a cap w by h from (0, 0), its label cut out of it"""
|
||||
from fontTools.pens.reverseContourPen import ReverseContourPen
|
||||
from fontTools.pens.transformPen import TransformPen
|
||||
rounded(pen, 0, 0, w, h, r)
|
||||
if label.startswith("@mouse"): # (in an icon of several keys: a small mouse cut out of the cap, its lines 30 thick)
|
||||
s = h * 0.72 / H
|
||||
mouse(TransformPen(ReverseContourPen(pen), (s, 0, 0, s, w / 2 - s * MOUSE / 2, h * 0.14)), label[7:], 30 / s)
|
||||
elif label.startswith("@"):
|
||||
arrow(ReverseContourPen(pen), w / 2, h / 2, h * 0.5, label[1:])
|
||||
elif label:
|
||||
for text, s, width, base in lines_of(letters, label, h, room):
|
||||
lo, _ = letters.ink(text)
|
||||
letters.draw(pen, text, s, w / 2 - width / 2 - lo * s, h / 2 + base)
|
||||
|
||||
|
||||
def mouse(pen, button, stroke=STROKE):
|
||||
"""a mouse MOUSE by H from (0, 0): its outline, and the button filled ("l", "r", "m"; "4", "5": the number on its palm;
|
||||
"": none). No part touches another: the same contours cut a small mouse out of a cap"""
|
||||
from fontTools.pens.reverseContourPen import ReverseContourPen
|
||||
from fontTools.pens.transformPen import TransformPen
|
||||
half, gap = MOUSE / 2, max(40, stroke / 2)
|
||||
rounded(pen, 0, 0, MOUSE, H, half)
|
||||
rounded(ReverseContourPen(pen), stroke, stroke, MOUSE - stroke, H - stroke, half - stroke)
|
||||
top, r = H - half, half - stroke - gap # (the buttons: the upper end's inside, a gap off the outline and each other)
|
||||
if button in ("l", "r"):
|
||||
side = pen if button == "l" else TransformPen(ReverseContourPen(pen), (-1, 0, 0, 1, MOUSE, 0))
|
||||
x = half - gap / 2
|
||||
a = math.degrees(math.acos((x - half) / r))
|
||||
side.moveTo((half - r, top - 120))
|
||||
side.lineTo((x, top - 120))
|
||||
side.lineTo((x, top + r * math.sin(math.radians(a))))
|
||||
arc(side, half, top, r, a, 180)
|
||||
side.closePath()
|
||||
elif button == "m": # (the wheel)
|
||||
wheel = min(55, 0.3 * r)
|
||||
rounded(pen, half - wheel, top - 0.35 * r, half + wheel, top + 0.6 * r, wheel)
|
||||
elif button: # (the side buttons have no place in this picture: their number)
|
||||
cap_height = (half - stroke - 20) / 0.625
|
||||
s = cap_height / letters_once().cap
|
||||
lo, hi = letters_once().ink(button)
|
||||
letters_once().draw(ReverseContourPen(pen), button, s, half - s * (lo + hi) / 2, half - cap_height / 2)
|
||||
|
||||
|
||||
def letters_once(_made=[]):
|
||||
if not _made:
|
||||
_made.append(Letters())
|
||||
return _made[0]
|
||||
|
||||
|
||||
def camera_size(axes):
|
||||
return {"xy": (820, 1000), "x": (820, 500), "y": (320, 1000)}[axes]
|
||||
|
||||
|
||||
def camera(pen, axes):
|
||||
"""the mouse moving: a small mouse, an arrowhead each way it moves (axes "xy", "x" or "y"), from (0, 0); camera_size"""
|
||||
from fontTools.pens.reverseContourPen import ReverseContourPen
|
||||
from fontTools.pens.transformPen import TransformPen
|
||||
w, h = camera_size(axes)
|
||||
cx, cy = w / 2, h / 2
|
||||
body = TransformPen(pen, (1, 0, 0, 1, cx - 160, cy - 250))
|
||||
rounded(body, 0, 0, 320, 500, 160)
|
||||
rounded(ReverseContourPen(body), 60, 60, 260, 440, 100)
|
||||
polygon(pen, [(cx - 30, cy + 60), (cx + 30, cy + 60), (cx + 30, cy + 150), (cx - 30, cy + 150)]) # (its wheel)
|
||||
if "y" in axes:
|
||||
polygon(pen, [(cx, cy + 500), (cx - 120, cy + 340), (cx + 120, cy + 340)])
|
||||
polygon(pen, [(cx, cy - 500), (cx + 120, cy - 340), (cx - 120, cy - 340)])
|
||||
if "x" in axes:
|
||||
polygon(pen, [(cx - 410, cy), (cx - 250, cy - 120), (cx - 250, cy + 120)])
|
||||
polygon(pen, [(cx + 410, cy), (cx + 250, cy + 120), (cx + 250, cy - 120)])
|
||||
|
||||
|
||||
def key(pen, letters, vk):
|
||||
"""a key's icon from (0, 0): key_width wide, H high (a key code that is no key: the blank cap)"""
|
||||
if vk in MICE:
|
||||
mouse(pen, KEYS[vk][7:])
|
||||
else:
|
||||
cap(pen, letters, KEYS.get(vk, ""), key_width(letters, vk), H, R, H * (P.WIDE - 0.3))
|
||||
|
||||
|
||||
def mini(pen, letters, vk):
|
||||
"""a key's small cap, MINI square, for an icon of several keys. Its outline begins at (MINI_R, 0) with the cap and ends
|
||||
at MINI_MARK's last point whatever the key: the last contour is a small square inside the cap's ink (it runs the cap's
|
||||
way: filled all the same, and no label reaches that corner), so that the glyph that calls it knows where the pen is
|
||||
before and after, and any key's cap can stand in for another's"""
|
||||
cap(pen, letters, KEYS.get(vk, ""), MINI, MINI, MINI_R, MINI * 0.74)
|
||||
polygon(pen, MINI_MARK)
|
||||
|
||||
|
||||
def standalone():
|
||||
"""the icons as a font of their own (OpenType, each at BASE + its key code, the camera's at BASE + 0x100 on), in the
|
||||
drawings' units, a cap half an em high (prompt_assets.Icons draws an icon on a sheet two ems wide): what the textures'
|
||||
images are drawn from"""
|
||||
from fontTools.fontBuilder import FontBuilder
|
||||
from fontTools.pens.t2CharStringPen import T2CharStringPen
|
||||
letters = letters_once()
|
||||
names, charstrings, metrics, cmap = [".notdef"], {}, {".notdef": (H, 0)}, {}
|
||||
pen = T2CharStringPen(H, None)
|
||||
charstrings[".notdef"] = pen.getCharString()
|
||||
for vk in sorted(KEYS):
|
||||
name, w = "key_%d" % vk, key_width(letters, vk)
|
||||
pen = T2CharStringPen(w + 100, None)
|
||||
key(pen, letters, vk)
|
||||
names.append(name)
|
||||
charstrings[name], metrics[name], cmap[BASE + vk] = pen.getCharString(), (w + 100, 0), name
|
||||
for i, axes in enumerate(("xy", "x", "y")):
|
||||
name, w = "camera_" + axes, camera_size(axes)[0]
|
||||
pen = T2CharStringPen(w + 100, None)
|
||||
camera(pen, axes)
|
||||
names.append(name)
|
||||
charstrings[name], metrics[name], cmap[BASE + 0x100 + i] = pen.getCharString(), (w + 100, 0), name
|
||||
fb = FontBuilder(2 * H, isTTF=False)
|
||||
fb.setupGlyphOrder(names)
|
||||
fb.setupCharacterMap(cmap)
|
||||
fb.setupCFF("TriforceKeys", {"FullName": "Triforce Keys"}, charstrings, {})
|
||||
fb.setupHorizontalMetrics(metrics)
|
||||
fb.setupHorizontalHeader(ascent=H, descent=0)
|
||||
fb.setupNameTable({"familyName": "Triforce Keys", "styleName": "Regular"})
|
||||
fb.setupOS2(sTypoAscender=H, sTypoDescender=0, usWinAscent=H, usWinDescent=0)
|
||||
fb.setupPost()
|
||||
out = io.BytesIO()
|
||||
fb.save(out)
|
||||
return out.getvalue()
|
||||
|
||||
|
||||
class Icons(P.Icons):
|
||||
"""the keys' icons for the textures (the ocarina's notes, the letters on black), by key code: prompt_assets.Icons drawn
|
||||
from standalone(). plain: the icons whose letter on black is the whole icon, not what is cut out of it (the mice)"""
|
||||
|
||||
def __init__(self):
|
||||
self.path, self.code, self.plain = io.BytesIO(standalone()), {vk: BASE + vk for vk in KEYS}, MICE
|
||||
|
||||
|
||||
def subr_bias(count):
|
||||
"""the number a charstring subtracts from a subroutine's index, by how many subroutines the font has (CFF)"""
|
||||
return 107 if count < 1240 else 1131 if count < 33900 else 32768
|
||||
|
||||
|
||||
def game_font(otf, default=DEFAULT):
|
||||
"""the game's font (decrypted) as the keyboard and mouse face, the controls' keys out of the box `default`: (the font,
|
||||
binds). binds: {"patches": [(offset in the font, control)], "calls": {key code: the two bytes of its small cap's call}},
|
||||
what the host writes for other keys (write_binds); no patches when the font cannot take the subroutines (none of its
|
||||
own, or too many): the icons of several keys then show the keys out of the box whatever is bound"""
|
||||
from fontTools.misc.psCharStrings import T2CharString
|
||||
from fontTools.pens.boundsPen import BoundsPen
|
||||
from fontTools.pens.t2CharStringPen import T2CharStringPen
|
||||
from fontTools.pens.transformPen import TransformPen
|
||||
from fontTools.ttLib import TTFont
|
||||
from fontTools.ttLib.tables.DefaultTable import DefaultTable
|
||||
|
||||
game = TTFont(io.BytesIO(otf))
|
||||
gset, cmap = game.getGlyphSet(), dict(game.getBestCmap())
|
||||
top = game["CFF "].cff.topDictIndex[0]
|
||||
chars, order, hmtx = top.CharStrings, list(game.getGlyphOrder()), game["hmtx"]
|
||||
own = chars[cmap[P.SWITCH[0]]] # (a charstring of the font: every new one shares its private dictionary and subroutines)
|
||||
letters = letters_once()
|
||||
|
||||
def bounds(name):
|
||||
p = BoundsPen(gset)
|
||||
gset[name].draw(p)
|
||||
return p.bounds
|
||||
|
||||
def fitted(box, w, h, advance):
|
||||
"""the transform of a drawing w by h into a glyph's box (as prompt_assets.face_font's draw: a square about the box's
|
||||
middle, as high as it and up to P.WIDE times as wide, the glyph widened), and the glyph's advance"""
|
||||
side = max(box[2] - box[0], box[3] - box[1])
|
||||
s = min(side * P.WIDE / w, side / h)
|
||||
over = max(0, s * w - side)
|
||||
return (s, 0, 0, s, (box[0] + box[2] + over) / 2 - s * w / 2, (box[1] + box[3]) / 2 - s * h / 2), advance + round(over)
|
||||
|
||||
def charstring(pen):
|
||||
return pen.getCharString(private=own.private, globalSubrs=own.globalSubrs)
|
||||
|
||||
# each key's icon, in the box of the game's A
|
||||
a_box, a_advance = bounds(cmap[P.SWITCH[0]]), hmtx[cmap[P.SWITCH[0]]][0]
|
||||
for vk in [0] + sorted(KEYS):
|
||||
t, width = fitted(a_box, key_width(letters, vk), H, a_advance)
|
||||
pen = T2CharStringPen(width, gset)
|
||||
key(TransformPen(pen, t), letters, vk)
|
||||
name = "key_%d" % vk
|
||||
chars.charStringsIndex.append(charstring(pen))
|
||||
chars.charStrings[name] = len(chars.charStringsIndex) - 1
|
||||
top.charset.append(name)
|
||||
order.append(name)
|
||||
hmtx.metrics[name] = (width, 0)
|
||||
# the camera's: the mouse moving, inside the right stick's glyphs' boxes (their advances stay)
|
||||
for code, axes in CAMERA.items():
|
||||
box = bounds(cmap[code]) if code in cmap else None
|
||||
if box:
|
||||
w, h = camera_size(axes)
|
||||
s = min((box[2] - box[0]) / w, (box[3] - box[1]) / h)
|
||||
pen = T2CharStringPen(hmtx[cmap[code]][0], gset)
|
||||
camera(TransformPen(pen, (s, 0, 0, s, (box[0] + box[2] - s * w) / 2, (box[1] + box[3] - s * h) / 2)), axes)
|
||||
chars[cmap[code]] = charstring(pen)
|
||||
# each key's small cap as a subroutine, then the icons of several keys calling them. The calls the host rewrites are
|
||||
# all two bytes long (a subroutine's number less the bias from 108 to 1131): empty subroutines fill up to the first
|
||||
subrs = getattr(own.private, "Subrs", None)
|
||||
codes = [0] + sorted(KEYS)
|
||||
bias = subr_bias(len(subrs)) if subrs is not None else 0
|
||||
first = max(len(subrs), bias + 108) if subrs is not None else 0
|
||||
live = subrs is not None and subr_bias(first + len(codes)) == bias and first + len(codes) - 1 - bias <= 1131
|
||||
index = {}
|
||||
if live:
|
||||
while len(subrs) < first:
|
||||
subrs.append(T2CharString(program=["return"], private=own.private, globalSubrs=own.globalSubrs))
|
||||
for vk in codes:
|
||||
pen = T2CharStringPen(None, gset)
|
||||
mini(pen, letters, vk)
|
||||
program = charstring(pen).program
|
||||
at = next(i for i, token in enumerate(program) if isinstance(token, str))
|
||||
assert program[at] in ("rmoveto", "hmoveto", "vmoveto") and program[-1] == "endchar", program[:4]
|
||||
index[vk] = len(subrs)
|
||||
subrs.append(T2CharString(program=program[at + 1:-1] + ["return"], private=own.private,
|
||||
globalSubrs=own.globalSubrs))
|
||||
clusters = [] # ((the glyph's program, [(where in it a call's number is, its control)]))
|
||||
for code, cells in CLUSTERS.items():
|
||||
if code not in cmap or not bounds(cmap[code]):
|
||||
continue
|
||||
name, step = cmap[code], MINI + MINI_GAP
|
||||
box, advance = bounds(name), hmtx[name][0]
|
||||
cols, rows = max(c for _, c, _ in cells) + 1, max(r for _, _, r in cells) + 1
|
||||
w, h = cols * step - MINI_GAP, rows * step - MINI_GAP
|
||||
over = max(0, w - (box[2] - box[0]))
|
||||
x0, y0 = round((box[0] + box[2] + over) / 2 - w / 2), round((box[1] + box[3]) / 2 - h / 2)
|
||||
width = advance + round(over)
|
||||
program, calls, at = [width], [], (0, 0)
|
||||
pen = T2CharStringPen(width, gset)
|
||||
for control, col, row in cells:
|
||||
ox, oy = x0 + col * step, y0 + (rows - 1 - row) * step
|
||||
if live:
|
||||
program += [ox + MINI_R - at[0], oy - at[1], "rmoveto"]
|
||||
calls.append((len(program), control))
|
||||
program += [index[default[control]] - bias, "callsubr"]
|
||||
at = (ox + MINI_MARK[-1][0], oy + MINI_MARK[-1][1])
|
||||
else:
|
||||
mini(TransformPen(pen, (1, 0, 0, 1, ox, oy)), letters, default[control])
|
||||
if live:
|
||||
chars[name] = T2CharString(program=program + ["endchar"], private=own.private, globalSubrs=own.globalSubrs)
|
||||
clusters.append((program + ["endchar"], calls))
|
||||
else:
|
||||
chars[name] = charstring(pen)
|
||||
hmtx.metrics[name] = (width, hmtx[name][1])
|
||||
|
||||
game.setGlyphOrder(order)
|
||||
gid = {n: i for i, n in enumerate(order)}
|
||||
mapping = {c: gid[n] for c, n in cmap.items() if c <= 0xFFFF}
|
||||
for vk in codes:
|
||||
mapping[BASE + vk] = gid["key_%d" % vk]
|
||||
single = {c: control for control, c in enumerate(P.SWITCH)}
|
||||
single.update(MOVES)
|
||||
for c, control in single.items():
|
||||
mapping[c] = gid["key_%d" % default[control]]
|
||||
table = DefaultTable("cmap")
|
||||
table.data = P.cmap4(mapping, set(single))
|
||||
game["cmap"] = table
|
||||
out = io.BytesIO()
|
||||
game.save(out)
|
||||
font = out.getvalue()
|
||||
|
||||
def compiled(program):
|
||||
cs = T2CharString(program=list(program))
|
||||
cs.compile()
|
||||
return bytes(cs.bytecode)
|
||||
|
||||
patches, seen = [], set()
|
||||
for program, calls in clusters: # (where each call's number is in the font as written: the glyph's bytes, found whole)
|
||||
code = compiled(program)
|
||||
if code in seen:
|
||||
continue
|
||||
seen.add(code)
|
||||
found, at = [], font.find(code)
|
||||
while at >= 0:
|
||||
found.append(at)
|
||||
at = font.find(code, at + 1)
|
||||
if len(found) != sum(1 for p, _ in clusters if compiled(p) == code):
|
||||
raise SystemExit("prompts: an icon of several keys is in the font %d times" % len(found))
|
||||
for at in found:
|
||||
for where, control in calls:
|
||||
off = at + len(compiled(program[:where] + ["endchar"])) - 1
|
||||
assert 247 <= font[off] <= 250, font[off]
|
||||
patches.append((off, control))
|
||||
calls = {vk: compiled([index[vk] - bias, "callsubr"])[:2] for vk in index}
|
||||
return font, {"patches": sorted(patches), "calls": calls}
|
||||
|
||||
|
||||
def write_binds(path, binds):
|
||||
with open(path, "wb") as f:
|
||||
f.write(b"HPKB" + struct.pack("<I", len(binds["patches"])))
|
||||
for off, control in binds["patches"]:
|
||||
f.write(struct.pack("<IB3x", off, control))
|
||||
f.write(struct.pack("<I", 256) + b"".join(binds["calls"].get(vk, b"\0\0") for vk in range(256)))
|
||||
|
||||
|
||||
def hint_names():
|
||||
"""the glyph of each key code's icon in game_font's font (None: no such key), for prompt_assets.write_hint"""
|
||||
return ["key_%d" % vk if vk == 0 or vk in KEYS else None for vk in range(256)]
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
from PIL import Image, ImageDraw, ImageFont
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--out", required=True)
|
||||
a = ap.parse_args()
|
||||
font = ImageFont.truetype(io.BytesIO(standalone()), 192)
|
||||
codes = [BASE + vk for vk in sorted(KEYS)] + [BASE + 0x100 + i for i in range(3)]
|
||||
sheet = Image.new("L", (12 * 180, (len(codes) + 11) // 12 * 130), 0)
|
||||
for i, code in enumerate(codes):
|
||||
ImageDraw.Draw(sheet).text((i % 12 * 180 + 10, i // 12 * 130 + 10), chr(code), font=font, fill=255)
|
||||
os.makedirs(a.out, exist_ok=True)
|
||||
sheet.save(os.path.join(a.out, "keys.png"))
|
||||
print("prompts: keys.png, %d icons" % len(codes), file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -194,6 +194,14 @@ def write_image_c(gen, img, meta, exefs):
|
||||
start_hook, exit_hook = thread_hooks(img, meta)
|
||||
lines.append("const uint64_t hos_thread_start_hook = %#xULL, hos_thread_exit_hook = %#xULL;"
|
||||
% (start_hook, exit_hook))
|
||||
# (Triforce) for the mods' cheat files (host/mod_cheats.c): where the module main begins, which a cheat's addresses
|
||||
# count from, and each module's code, which a cheat must not write (it is compiled ahead of time)
|
||||
code = [(m["segments"][0]["addr"], m["segments"][0]["addr"] + m["segments"][0]["size"]) for m in meta["modules"]]
|
||||
lines += ["const uint64_t hos_main_base = %#xULL; /* (the module main) */"
|
||||
% next((m["base"] for m in meta["modules"] if m["name"] == "main"), meta["base"]),
|
||||
"const int hos_code_count = %d;" % len(code),
|
||||
"const uint64_t hos_code[][2] = {%s}; /* (each module's .text: from, to) */"
|
||||
% ", ".join("{%#xULL, %#xULL}" % span for span in code)]
|
||||
text = "\n".join(lines) + "\n"
|
||||
path = os.path.join(gen, "hos_image.c")
|
||||
if not os.path.exists(path) or open(path).read() != text:
|
||||
|
||||
@@ -68,18 +68,18 @@ mg_spec.loader.exec_module(make_game)
|
||||
calls = []
|
||||
reported = [] # (whether each call was given a progress(done, total): the check says how far it is, #1764)
|
||||
sys.modules["nup"] = type(sys)("nup")
|
||||
sys.modules["nup"].stand_in_damaged = lambda romfs, progress=None: calls.append(romfs) or reported.append(callable(progress)) or []
|
||||
sys.modules["nup"].stand_in_damaged = lambda romfs, progress=None, known=None: calls.append(romfs) or reported.append(callable(progress)) or []
|
||||
with tempfile.TemporaryDirectory() as out:
|
||||
romfs = os.path.join(out, "romfs")
|
||||
os.makedirs(romfs)
|
||||
make_game.stand_in_damaged_files(out, romfs)
|
||||
check("no stamp: not the builder's romfs", calls, [])
|
||||
with open(os.path.join(out, ".hookshot_romfs"), "w") as f:
|
||||
with open(os.path.join(out, ".triforce_romfs"), "w") as f:
|
||||
f.write("unpacked by this builder: game.nup 1\n")
|
||||
make_game.stand_in_damaged_files(out, romfs)
|
||||
make_game.stand_in_damaged_files(out, romfs)
|
||||
check("checked once", calls, [romfs])
|
||||
with open(os.path.join(out, ".hookshot_romfs"), "w") as f:
|
||||
with open(os.path.join(out, ".triforce_romfs"), "w") as f:
|
||||
f.write("unpacked by this builder: other.nup 2\n")
|
||||
make_game.stand_in_damaged_files(out, romfs)
|
||||
check("checked again for another unpack", calls, [romfs, romfs])
|
||||
|
||||
@@ -7,6 +7,11 @@ image.bin nor its log's folder and closed with nothing on screen. activeCodePage
|
||||
every A function and the C runtime's narrow file functions UTF-8 for the whole process, its DLLs too (triforce_vk.dll,
|
||||
SDL2, FFmpeg). host/hos_main.c says so in a message box when the Windows is too old for it and the folder's name needs it.
|
||||
|
||||
(Triforce) longPathAware: where Windows' own LongPathsEnabled is on (Windows 10 1607 and newer, a setting of the PC's), the A and
|
||||
W functions and the C runtime's file functions take paths of MAX_PATH (260) and more, for the files the game finds in a deep
|
||||
folder (the game's own nn::fs paths add the long-path prefix themselves, host/nn_fs.c fs_wide; host/hos_paths.h's and gpu/vk/vk_paths.h's
|
||||
probe too). Without that setting it changes nothing.
|
||||
|
||||
python nx2/win_manifest.py [out.res]
|
||||
"""
|
||||
import os
|
||||
@@ -19,6 +24,7 @@ MANIFEST = b"""<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<application xmlns="urn:schemas-microsoft-com:asm.v3">
|
||||
<windowsSettings>
|
||||
<activeCodePage xmlns="http://schemas.microsoft.com/SMI/2019/WindowsSettings">UTF-8</activeCodePage>
|
||||
<longPathAware xmlns="http://schemas.microsoft.com/SMI/2016/WindowsSettings">true</longPathAware>
|
||||
</windowsSettings>
|
||||
</application>
|
||||
<trustInfo xmlns="urn:schemas-microsoft-com:asm.v3">
|
||||
|
||||
Reference in New Issue
Block a user