mirror of
https://git.triforceproject.com/triforce/source
synced 2026-11-19 16:20:01 -04:00
The new launcher (Windows and Linux: the game's own look, music and settings, saves and mods pages, smaller updates in parts), benchmark mode (Find my best settings: the game tours four places by itself and writes its results), Quit and Restart in the pause menu under Mods, Squadala! in the Mods menu, battle music above 60 fps, menu text, the controller in the menus with the game in the background, the whole GameBanana list, RX 580 and RX 5000/6000 shader fixes, the pipeline cache kept through an update, async present black flashes, and faster builds.
2230 lines
122 KiB
Python
2230 lines
122 KiB
Python
"""A game folder from the user's own dump: what the builder app (builder/) runs.
|
|
|
|
python -I nx2/make_game.py --nup <file.nup> --out <folder> [--work <folder>] [--jobs <n>]
|
|
python -I nx2/make_game.py --romfs <dir> --exefs <dir> --out <folder> [--work <folder>] [--jobs <n>]
|
|
python -I nx2/make_game.py --check --nup <file.nup> (or --exefs <dir>: is this the right game and version?)
|
|
python -I nx2/make_game.py --update <folder> (a folder built before, again with these tools: the game files it keeps)
|
|
python -I nx2/make_game.py --game <.nup or folder> --prepare (RecompLauncher: --out defaults to ~/Games/Hookshot; no window,
|
|
"progress <0..1> <text>" lines and an "error <text>" line on stdout,
|
|
exit 0 when the game is built; with --check, its "check ok|bad" line)
|
|
|
|
The folder holds the executable and all it needs, started with a double click: nothing goes back into the repository, and no game
|
|
data ever comes with the tools. Lines the app reads, each on its own line of stdout:
|
|
|
|
@@stage <id> <text> a step begins
|
|
@@progress <0..1> how far that step is
|
|
@@plan <changed> <total> how many files the compile rebuilds, of all of them
|
|
@@file <name> <seconds> <new|cached|again> one module's build
|
|
@@eta <seconds> the compile's time left, from what the earlier builds measured on this PC
|
|
@@times <total> <compile> <previous|-1> finished: this build's times beside the last one's
|
|
@@check <ok|bad> <text> (--check) the answer
|
|
@@done <folder> finished
|
|
@@error <text> stopped
|
|
|
|
and one plain line the builders also read (older ones only show it): "the shaders were not prepared now (<why>): ..." when the
|
|
build's last step, the shaders' preparation, was skipped or failed and is left to the game's first start (NOT_PREPARED)
|
|
"""
|
|
import argparse
|
|
import atexit
|
|
import contextlib
|
|
import filecmp
|
|
import hashlib
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
import queue
|
|
import re
|
|
import shutil
|
|
import stat
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
import sys
|
|
import tempfile
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
ROOT = os.path.dirname(HERE)
|
|
PY = [sys.executable, "-I", "-u", "-X", "utf8"] # (utf8: every step's output in UTF-8 whatever the console's code page: lift.py crashed on a letter, #180) # (-u: the steps' output as it comes, not in 8 KB blocks minutes apart)
|
|
BUILD_ID = "e0f3ccfa" # (the version the patches, the hot list and the profile were made from)
|
|
STARTED = time.time() # (the build's total time, kept in build_times.json and said as @@times)
|
|
sys.path[:0] = [HERE, os.path.join(ROOT, "third_party", "a64-kit", "tools")] # (python -I leaves the script folder off)
|
|
import toolchain as _toolchain
|
|
import disk as _disk
|
|
import code_mods as _code_mods # (Triforce: the mods' patches of the game's code, applied before the lift)
|
|
# (hookshot) RX_TARGET=windows on Linux: the Windows game, with llvm-mingw's Linux release (HOOKSHOT_MINGW, or llvm-mingw
|
|
# beside the builder), to run under Proton or copy to a Windows PC
|
|
LINUX, CROSS = not _toolchain.windows(), _toolchain.windows() and os.name != "nt"
|
|
# (SDL2.dll is sdl2-compat: it loads SDL3.dll from beside the exe. Left out of bin\ since v2.0.0, it said "Failed loading SDL3
|
|
# library" three times and no controller worked: Discord #bug-reporting, 11-08)
|
|
BESIDE = ["triforce", "triforce_vk.so", "libSDL2-2.0.so.0"] if LINUX else ["Triforce.exe", "triforce_vk.dll", "SDL2.dll", "SDL3.dll", "wgpu_native.dll"]
|
|
LANGUAGES = [("en-US", "English (US)"), ("en-GB", "English (UK)"), ("fr", "French (France)"), ("fr-CA", "French (Canada)"), ("de", "German"),
|
|
("it", "Italian"), ("es", "Spanish (Spain)"), ("es-419", "Spanish (Latin America)"), ("pt-BR", "Portuguese (Brazil)"), ("nl", "Dutch"),
|
|
("pl", "Polish"), ("ru", "Russian"), ("ja", "Japanese"), ("ko", "Korean"), ("zh-Hans", "Chinese (Simplified)"),
|
|
("zh-Hant", "Chinese (Traditional)")]
|
|
|
|
|
|
def no_throttle(handle=None):
|
|
"""(issue 87) Windows' power throttling (EcoQoS) slows a background window's processes to almost nothing on some machines: the
|
|
build sat at 0% CPU. Turned off for this process, or the one whose handle this is"""
|
|
if os.name != "nt":
|
|
return
|
|
import ctypes
|
|
class State(ctypes.Structure):
|
|
_fields_ = [("Version", ctypes.c_ulong), ("ControlMask", ctypes.c_ulong), ("StateMask", ctypes.c_ulong)]
|
|
k = ctypes.WinDLL("kernel32")
|
|
k.GetCurrentProcess.restype = ctypes.c_void_p
|
|
k.SetProcessInformation.argtypes = [ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_ulong]
|
|
s = State(1, 1, 0) # (PROCESS_POWER_THROTTLING_EXECUTION_SPEED in the mask, off in the state)
|
|
try:
|
|
k.SetProcessInformation(ctypes.c_void_p(int(handle)) if handle else k.GetCurrentProcess(), 4, ctypes.byref(s), ctypes.sizeof(s))
|
|
except (OSError, ValueError, TypeError):
|
|
pass
|
|
|
|
|
|
PREPARE = False # (--prepare) say() prints RecompLauncher's "progress <0..1> <text>" lines instead of the builder app's @@ lines
|
|
STAGE_SPAN = {"read": (0.0, 0.05), "lift": (0.05, 0.25), "compile": (0.25, 0.95), "finish": (0.95, 1.0)} # (a share of the whole each stage takes)
|
|
_stage = {"span": (0.0, 1.0), "text": ""}
|
|
|
|
|
|
def say_prepare(words):
|
|
if words[0] == "stage" and len(words) > 2:
|
|
_stage["span"], _stage["text"] = STAGE_SPAN.get(words[1], _stage["span"]), " ".join(words[2:])
|
|
print("progress %.3f %s" % (_stage["span"][0], _stage["text"]), flush=True)
|
|
elif words[0] == "progress" and len(words) > 1:
|
|
lo, hi = _stage["span"]
|
|
print("progress %.3f %s" % (lo + (hi - lo) * float(words[1]), _stage["text"]), flush=True)
|
|
elif words[0] in ("error", "check"): # (on stdout, the only stream a launcher reads; HookshotBuilder --cli also takes its "error" line)
|
|
print(" ".join(words), flush=True)
|
|
elif words[0] == "done":
|
|
print("progress 1.000 Done", flush=True)
|
|
|
|
|
|
STAGE_TIMES = [] # ((stage, when it began)): what each step of this build took, said at its end (stage_times)
|
|
|
|
|
|
def stage_times(now):
|
|
""" "before the game files 0:41, read 1:32, lift 2:10, compile 7:42, finish 1:17": each stage's time up to the next
|
|
one's start, the last one's up to now, and what came before the first (the tools' checks and what was fetched)"""
|
|
if not STAGE_TIMES:
|
|
return ""
|
|
parts = [("before the game files", STAGE_TIMES[0][1] - STARTED)]
|
|
for (name, began), (_, ended) in zip(STAGE_TIMES, STAGE_TIMES[1:] + [("", now)]):
|
|
parts.append((name, ended - began))
|
|
return ", ".join("%s %s" % (name, clock(seconds)) for name, seconds in parts)
|
|
|
|
|
|
def say(*words):
|
|
if words and words[0] == "stage" and len(words) > 1:
|
|
STAGE_TIMES.append((str(words[1]), time.time()))
|
|
if PREPARE:
|
|
return say_prepare([str(w) for w in words])
|
|
print("@@" + " ".join(str(w) for w in words), flush=True)
|
|
|
|
|
|
def fail(text):
|
|
say("error", text)
|
|
sys.exit(1)
|
|
|
|
|
|
def failure_reason(lines):
|
|
"""The line a stopped step is named with. Only the last 12 lines used to be kept, so a compiler's "error:" hundreds
|
|
of lines earlier was replaced by "compiled in 790s, 1 failed"."""
|
|
tail, seen = [], None
|
|
for line in lines:
|
|
tail = (tail + [line])[-12:]
|
|
if "error:" in line.lower() and "CalledProcessError" not in line:
|
|
seen = line
|
|
if seen is not None:
|
|
return seen
|
|
return tail[-1] if tail else "no output"
|
|
|
|
|
|
def killed_reason():
|
|
"""(issues 478, 768, 807) A step stopped by SIGKILL or SIGTERM from outside the build: the builder's own Cancel stops make_game
|
|
with it, so this is the system's, the kernel's or a desktop's low-memory killer most likely. Its last lines then said nothing
|
|
of that (the lift's emit processes failing on their result pipe, BrokenPipeError, or the lift's last progress line)"""
|
|
total = None
|
|
try:
|
|
with open("/proc/meminfo") as f:
|
|
total = next((round(int(l.split()[1]) / (1 << 20)) for l in f if l.startswith("MemTotal:")), None) # (kB, to the nearest GB)
|
|
except (OSError, ValueError, IndexError):
|
|
pass
|
|
return ("the system stopped it, most likely because the PC ran out of memory%s. Close other programs and build again"
|
|
% (" (%d GB here)" % total if total else ""))
|
|
|
|
|
|
def reported(line):
|
|
"""The sentence a builder shows. @@check is '@@check <ok|bad> <text>'; @@error is '@@error <text>'. None otherwise."""
|
|
if line.startswith("@@error "):
|
|
return line[len("@@error "):]
|
|
if line.startswith("@@check "):
|
|
parts = line.split(" ", 2)
|
|
return parts[2] if len(parts) == 3 else ""
|
|
return None
|
|
|
|
|
|
class LogStream:
|
|
"""Keep console output live while also saving it to the build log."""
|
|
def __init__(self, console, log):
|
|
self.console, self.log = console, log
|
|
|
|
def write(self, text):
|
|
result = self.console.write(text)
|
|
self.log.write(text)
|
|
self.flush()
|
|
return result
|
|
|
|
def flush(self):
|
|
self.console.flush()
|
|
self.log.flush()
|
|
|
|
def __getattr__(self, name):
|
|
return getattr(self.console, name)
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def build_log(path):
|
|
if path is None:
|
|
yield
|
|
return
|
|
try:
|
|
log = open(path, "w", encoding="utf-8", newline="\n")
|
|
except OSError as e:
|
|
fail("Could not open build log %s: %s" % (path, e))
|
|
with log:
|
|
with contextlib.redirect_stdout(LogStream(sys.stdout, log)), contextlib.redirect_stderr(LogStream(sys.stderr, log)):
|
|
yield
|
|
|
|
|
|
TIMES_FORMAT = 1
|
|
KEEP_BUILDS = 20
|
|
|
|
|
|
def times_path(work):
|
|
"""Where the builds' times are kept: beside the object cache, in the builder's own folder (LOCALAPPDATA\\HookshotBuilder
|
|
on Windows, ~/.cache/hookshot-builder on Linux). A builder update keeps it, so the next build still knows what this PC
|
|
took, and the one after that can compare."""
|
|
return os.path.join(work, "build_times.json")
|
|
|
|
|
|
def load_times(work):
|
|
"""The seconds each module took on this PC, and the last builds' totals; a fresh record when there is none to trust
|
|
(missing, half written, or written by another version of the tools)."""
|
|
data = {"format": TIMES_FORMAT, "builds": [], "seconds": {}}
|
|
try:
|
|
with open(times_path(work), encoding="utf-8") as f:
|
|
old = json.load(f)
|
|
except (OSError, ValueError):
|
|
return data
|
|
if not isinstance(old, dict) or old.get("format") != TIMES_FORMAT:
|
|
return data
|
|
builds = old.get("builds")
|
|
if isinstance(builds, list):
|
|
data["builds"] = [b for b in builds
|
|
if isinstance(b, dict) and isinstance(b.get("total_seconds"), (int, float))][-KEEP_BUILDS:]
|
|
seconds = old.get("seconds")
|
|
if isinstance(seconds, dict):
|
|
data["seconds"] = {str(k): float(v) for k, v in seconds.items()
|
|
if isinstance(v, (int, float)) and not isinstance(v, bool)}
|
|
return data
|
|
|
|
|
|
def save_times(work, data):
|
|
"""The whole file at once: a build stopped while writing it leaves the last record in place."""
|
|
path = times_path(work)
|
|
try:
|
|
os.makedirs(work, exist_ok=True)
|
|
with open(path + ".new", "w", encoding="utf-8") as f:
|
|
json.dump(data, f, indent=1, sort_keys=True)
|
|
f.write("\n")
|
|
os.replace(path + ".new", path)
|
|
except OSError as e:
|
|
print("the build times were not saved (%s)" % e, flush=True)
|
|
|
|
|
|
def record_times(work, files, compile_seconds, total_seconds, modules=0, kind="build"):
|
|
"""This build in the record: what each module it compiled took (a module a cache already held keeps the time it took
|
|
before), the compile's and the whole build's seconds. Returns the last build's total, to show beside this one."""
|
|
data = load_times(work)
|
|
previous = data["builds"][-1]["total_seconds"] if data["builds"] else None
|
|
data["seconds"].update({str(k): round(float(v), 1) for k, v in files.items() if v > 0})
|
|
data["builds"].append({"when": time.strftime("%Y-%m-%d %H:%M:%S"), "kind": kind, "modules": int(modules),
|
|
"compiled": len(files), "compile_seconds": round(compile_seconds, 1),
|
|
"total_seconds": round(total_seconds, 1)})
|
|
data["builds"] = data["builds"][-KEEP_BUILDS:]
|
|
save_times(work, data)
|
|
return previous
|
|
|
|
|
|
def clock(seconds):
|
|
"""A length of time the way the build's lines say it: '1:02:03'"""
|
|
s = int(round(seconds))
|
|
return "%d:%02d:%02d" % (s // 3600, s // 60 % 60, s % 60)
|
|
|
|
|
|
class Silent(Exception):
|
|
"""a step printed nothing for longer than run()'s silence"""
|
|
|
|
|
|
def output_lines(p, silence):
|
|
"""A step's lines as they come; with silence (seconds), read on a thread so that a step which goes quiet that long (a hung
|
|
driver, a crash dialog left open) raises Silent instead of waiting for ever"""
|
|
if silence is None:
|
|
yield from p.stdout
|
|
return
|
|
lines = queue.Queue()
|
|
|
|
def read():
|
|
try:
|
|
for line in p.stdout:
|
|
lines.put(line)
|
|
except (OSError, ValueError):
|
|
pass
|
|
finally:
|
|
with contextlib.suppress(OSError, ValueError):
|
|
p.stdout.close() # (here, not by run(): a stopped step's pipe may still be read by this thread)
|
|
lines.put(None)
|
|
|
|
threading.Thread(target=read, daemon=True).start()
|
|
while True:
|
|
try:
|
|
line = lines.get(timeout=silence)
|
|
except queue.Empty:
|
|
raise Silent() from None
|
|
if line is None:
|
|
return
|
|
yield line
|
|
|
|
|
|
def run(cmd, env=None, on_line=None, check=True, output=None, show=None, cwd=ROOT, base=None, silence=None):
|
|
"""A step, its output printed as it comes. show: what of each line is printed (its None: nothing; on_line and a failure's
|
|
reason still get the line itself). base: the environment it starts from (os.environ). silence: seconds without a line after
|
|
which the step is stopped; it returns None then (with check, the build fails)"""
|
|
e = dict(os.environ if base is None else base, PYTHONIOENCODING="utf-8")
|
|
e.update(env or {})
|
|
p = subprocess.Popen(cmd, cwd=cwd, env=e, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding="utf-8",
|
|
errors="replace", bufsize=1)
|
|
no_throttle(getattr(p, "_handle", None))
|
|
lines = []
|
|
try:
|
|
for line in output_lines(p, silence):
|
|
line = line.rstrip()
|
|
shown = show(line) if show else line
|
|
if shown is not None:
|
|
print(shown, file=output, flush=True)
|
|
lines.append(line)
|
|
if on_line:
|
|
on_line(line)
|
|
except Silent:
|
|
p.kill()
|
|
p.wait()
|
|
if check:
|
|
fail("%s printed nothing for %s and was stopped" % (os.path.basename(cmd[0]), span(silence)))
|
|
return None
|
|
p.stdout.close()
|
|
code = p.wait()
|
|
if code and check:
|
|
step = next((os.path.basename(c) for c in cmd[1:] if c.endswith(".py")), os.path.basename(cmd[-1])) # (issue 154: not "-u")
|
|
status = "signal %d" % -code if os.name == "posix" and code < 0 else "exit status %d" % code
|
|
fail("%s stopped (%s): %s" % (step, status, killed_reason() if os.name == "posix" and -code in (9, 15) else failure_reason(lines)))
|
|
return code
|
|
|
|
|
|
def build_id(main):
|
|
"""the module id of an NSO, plain or compressed: the header is the same"""
|
|
with open(main, "rb") as f:
|
|
head = f.read(0x60)
|
|
if head[:4] != b"NSO0":
|
|
fail("%s is not a game executable" % main)
|
|
return head[0x40:0x60].hex()
|
|
|
|
|
|
def check_modules(exefs):
|
|
"""Every module of the exefs is one the tools can read: said here, not as a Python error from the translator"""
|
|
for name in os.listdir(exefs):
|
|
if name != "main.npdm" and not name.endswith(".part"):
|
|
with open(os.path.join(exefs, name), "rb") as f:
|
|
head = f.read(16)
|
|
if head[:4] != b"NSO0":
|
|
fail("The file %s in this dump is not a game module (it starts %s). Try another dump of the game, or send this message to the "
|
|
"Triforce developers." % (name, head.hex()))
|
|
|
|
|
|
def check_supported_build(exefs):
|
|
bid = build_id(os.path.join(exefs, "main"))
|
|
if not bid.startswith(BUILD_ID):
|
|
fail("This is build %s. Triforce supports build %s." % (bid[:8], BUILD_ID))
|
|
|
|
|
|
def file_progress(verb, clock=time.monotonic):
|
|
"""A progress(done, total) for a long pass over many files: "@@progress" for the builder's bar of the step it is in, at most
|
|
4 times a second and at the end, and a plain line at each tenth ("checked 40,000 of 102,185 files (40%, 51 s)"), so a log or a
|
|
terminal shows it moving too"""
|
|
start = clock()
|
|
last = [None, 0] # (when the last @@progress went out; the last tenth said)
|
|
def progress(done, total):
|
|
now = clock()
|
|
if last[0] is None or now - last[0] >= 0.25 or done >= total:
|
|
last[0] = now
|
|
print("@@progress %.3f" % (done / total if total else 1.0), flush=True)
|
|
tenth = done * 10 // total if total else 10
|
|
if tenth > last[1]:
|
|
last[1] = tenth
|
|
print("%s %s of %s files (%d%%, %d s)" % (verb, format(done, ","), format(total, ","), tenth * 10, now - start),
|
|
flush=True)
|
|
return progress
|
|
|
|
|
|
def unpack_seen(game):
|
|
"""What the unpack itself saw of the romfs .zs files it wrote (nx2/nup.py ZS_CHECKED, in the folder it extracted to):
|
|
{"files": how many, "damaged": [paths under romfs]}, or None when it left no such note or one that cannot be right
|
|
(then every file is opened and looked at, as before)"""
|
|
import json
|
|
import nup
|
|
try:
|
|
with open(os.path.join(game, nup.ZS_CHECKED), encoding="utf-8") as f:
|
|
seen = json.load(f)
|
|
except (OSError, ValueError):
|
|
return None
|
|
if not isinstance(seen, dict) or not isinstance(seen.get("files"), int) or not isinstance(seen.get("damaged"), list):
|
|
return None
|
|
for rel in seen["damaged"]:
|
|
if not isinstance(rel, str):
|
|
return None
|
|
parts = rel.split("/")
|
|
if not rel.endswith(".zs") or "\\" in rel or ":" in rel or any(p in ("", ".", "..") for p in parts):
|
|
return None
|
|
return seen
|
|
|
|
|
|
def stand_in_damaged_files(out, romfs, seen=None):
|
|
"""An incomplete dump's damaged textures replaced by whole ones of the same kind, and said: without them the game
|
|
leaves out the walls and floors that use them (the Water Temple, #1002). Only in a romfs this builder unpacked (its
|
|
.triforce_romfs stamp), and once per unpack: .triforce_romfs_checked holds the stamp it was checked under.
|
|
seen: what this unpack saw of its own .zs files as it wrote them (unpack_seen): the damaged ones are then known and
|
|
no file is opened to find them. Opening all 102,185 took 292 s of an 18 minute build on a PC whose virus scanner
|
|
looks at each file opened; a folder unpacked by an earlier builder, or read from this .nup before without the
|
|
check's note, is still gone through file by file"""
|
|
good = os.path.join(out, ".triforce_romfs")
|
|
checked = os.path.join(out, ".triforce_romfs_checked")
|
|
if not os.path.isfile(good) or not os.path.isdir(romfs):
|
|
return
|
|
check_cache_paths(out, (good, checked, romfs))
|
|
with open(good) as f:
|
|
stamp = f.read()
|
|
if os.path.isfile(checked):
|
|
with open(checked) as f:
|
|
if f.read() == stamp:
|
|
return
|
|
import nup
|
|
if seen is not None:
|
|
print("your game files were checked for damaged textures as they were unpacked: %s .zs files, %d damaged"
|
|
% (format(seen["files"], ","), len(seen["damaged"])), flush=True)
|
|
done = nup.stand_in_damaged(romfs, known=seen["damaged"])
|
|
else:
|
|
print("checking your game files for damaged textures (once after an unpack): every .zs file is opened, which can take "
|
|
"minutes", flush=True)
|
|
done = nup.stand_in_damaged(romfs, file_progress("checked"))
|
|
with open(checked, "w") as f:
|
|
f.write(stamp)
|
|
if not done:
|
|
return
|
|
print("Your game dump is incomplete: %d game file%s in it %s damaged (a complete dump is about 23.8 GB, an incomplete "
|
|
"one 23.3 GB). Each is replaced by a similar file so that the places using it are still drawn, with a different "
|
|
"texture there. Dump the game again for the original look:" % (len(done), "" if len(done) == 1 else "s",
|
|
"is" if len(done) == 1 else "are"), flush=True)
|
|
for path, other in done:
|
|
print(" %s: %s" % (os.path.relpath(path, romfs),
|
|
"in its place " + os.path.basename(other) if other else "nothing similar found, left as it is"),
|
|
flush=True)
|
|
|
|
|
|
def check_cache_paths(out, paths):
|
|
"""Resolve generated paths and reject symbolic links/reparse points before replacement."""
|
|
root = os.path.realpath(out)
|
|
for path in paths:
|
|
try:
|
|
inside = os.path.commonpath((root, os.path.realpath(path))) == root
|
|
except ValueError:
|
|
inside = False
|
|
attrs = os.lstat(path) if os.path.lexists(path) else None
|
|
linked = os.path.islink(path) or bool(
|
|
getattr(attrs, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400))
|
|
if not inside or linked:
|
|
fail("The generated cache path %s is not a normal path inside the build folder (symbolic links and reparse points are not supported). "
|
|
"Choose a different output folder." % path)
|
|
|
|
|
|
def check_cpu():
|
|
"""(linux) The AppImage's Vulkan back end (HOOKSHOT_VK_PREBUILT) is built for x86-64-v3 elsewhere
|
|
(scripts/linux/appimage.sh), while the game follows this PC (nx2/host_cpu.py): on a CPU without AVX2 it would stop at
|
|
its start. Said now, before the game is read, not 40 minutes in (issue 363)"""
|
|
if not os.environ.get("HOOKSHOT_VK_PREBUILT") or os.environ.get("RX_MARCH"):
|
|
return
|
|
sys.path.insert(0, HERE)
|
|
from host_cpu import host_march
|
|
m = host_march()
|
|
if m != "x86-64-v3":
|
|
fail("Your processor has no AVX2 (it runs %s), and the AppImage's renderer needs it. Build Hookshot from source "
|
|
"instead (see the readme): a source build follows your processor." % m)
|
|
|
|
|
|
def check(a, work):
|
|
check_windows_sdk() # (first: said in plain words when the game is chosen, not after it has been read)
|
|
check_cpu()
|
|
if a.nup:
|
|
probe = os.path.join(work, "probe")
|
|
shutil.rmtree(probe, ignore_errors=True)
|
|
run(PY + ["nx2/nup.py", a.nup, "--extract", probe, "--only", "exefs"])
|
|
check_modules(os.path.join(probe, "exefs"))
|
|
bid = build_id(os.path.join(probe, "exefs", "main"))
|
|
shutil.rmtree(probe, ignore_errors=True)
|
|
else:
|
|
refuse_packed_romfs(a.romfs) # (Setup passes the romfs folder too since 11-18)
|
|
bid = build_id(os.path.join(a.exefs, "main"))
|
|
if bid.startswith(BUILD_ID):
|
|
say("check", "ok", "This is the version Triforce supports (build %s)." % bid[:8])
|
|
else:
|
|
say("check", "bad", "This is build %s. Triforce supports build %s." % (bid[:8], BUILD_ID))
|
|
|
|
|
|
def visual_studio_environment():
|
|
"""What Visual Studio's own vcvars64.bat sets, for a compiler that does not find it by itself; None without one"""
|
|
vswhere = os.path.join(os.environ.get("ProgramFiles(x86)", "C:/Program Files (x86)"), "Microsoft Visual Studio", "Installer", "vswhere.exe")
|
|
if not os.path.exists(vswhere):
|
|
return None
|
|
r = subprocess.run([vswhere, "-latest", "-prerelease", "-products", "*", "-requires", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64",
|
|
"-property", "installationPath"], capture_output=True, text=True, encoding="utf-8", errors="replace")
|
|
lines = r.stdout.strip().splitlines()
|
|
if not lines:
|
|
return None
|
|
bat = os.path.join(lines[-1], "VC", "Auxiliary", "Build", "vcvars64.bat")
|
|
if not os.path.exists(bat):
|
|
return None
|
|
r = subprocess.run('"%s" >nul && set' % bat, shell=True, capture_output=True, text=True, encoding="mbcs", errors="replace")
|
|
keep = ("INCLUDE", "LIB", "LIBPATH", "PATH", "VCINSTALLDIR", "VCTOOLSINSTALLDIR", "VCTOOLSVERSION", "WINDOWSSDKDIR", "WINDOWSSDKVERSION",
|
|
"UCRTVERSION", "UNIVERSALCRTSDKDIR")
|
|
found = {}
|
|
for line in r.stdout.splitlines():
|
|
name, eq, value = line.partition("=")
|
|
if eq and name.upper() in keep:
|
|
found[name] = value
|
|
return found if any(k.upper() == "INCLUDE" for k in found) else None
|
|
|
|
|
|
def windows_kits_environment(base):
|
|
"""INCLUDE and LIB with the Windows SDK's C library (ucrt) and headers added from where its installer puts them: Visual Studio's
|
|
vcvars leaves them out when the SDK came apart from it (issue 89: 'stdio.h' file not found with MSVC installed). None without one"""
|
|
kits = os.path.join(os.environ.get("ProgramFiles(x86)", "C:/Program Files (x86)"), "Windows Kits", "10")
|
|
inc = os.path.join(kits, "Include")
|
|
try:
|
|
versions = sorted((v for v in os.listdir(inc) if os.path.exists(os.path.join(inc, v, "ucrt", "stdio.h"))),
|
|
key=lambda v: [int(x) if x.isdigit() else 0 for x in v.split(".")])
|
|
except OSError:
|
|
return None
|
|
if not versions:
|
|
return None
|
|
v = versions[-1]
|
|
env = dict(base or {})
|
|
get = lambda k: next((env[n] for n in env if n.upper() == k), os.environ.get(k, ""))
|
|
env["INCLUDE"] = ";".join(x for x in [get("INCLUDE")] + [os.path.join(inc, v, d) for d in ("ucrt", "um", "shared", "winrt")] if x)
|
|
env["LIB"] = ";".join(x for x in [get("LIB")] + [os.path.join(kits, "Lib", v, d, "x64") for d in ("ucrt", "um")] if x)
|
|
print("the Windows SDK %s from %s" % (v, kits), flush=True)
|
|
return env
|
|
|
|
|
|
def compile_test(toolchain):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
src = os.path.join(d, "t.c")
|
|
with open(src, "w") as f:
|
|
# (_UCRT: the C library of Visual Studio 2015 and later. An older one's stdio.h compiled, and the link failed 40 minutes later
|
|
# on __iob_func: issue 80)
|
|
f.write(chr(10).join(("#include <stdio.h>", "#ifndef _UCRT", "#error \"stdio.h is from an old C library (before Visual Studio 2015)\"",
|
|
"#endif", "#include <windows.h>", "int main(void) { return 0; }", "")))
|
|
return subprocess.run([toolchain.need(toolchain.find("clang"))] + toolchain.native_target() + ["-c", src, "-o", os.path.join(d, "t.o")],
|
|
capture_output=True, text=True, encoding="utf-8", errors="replace")
|
|
|
|
|
|
def link_test(toolchain):
|
|
"""A program links, not only compiles (issue 439: a default library a recent Windows SDK no longer ships stopped the
|
|
build at 99%, after the game had been read and translated)"""
|
|
with tempfile.TemporaryDirectory() as d:
|
|
src = os.path.join(d, "t.c")
|
|
with open(src, "w") as f:
|
|
f.write(chr(10).join(("#include <stdio.h>", "int main(void) { return 0; }", "")))
|
|
return subprocess.run([toolchain.need(toolchain.find("clang"))] + toolchain.native_target() + toolchain.native_link() +
|
|
[src, "-o", os.path.join(d, "t.exe")], capture_output=True, text=True, encoding="utf-8", errors="replace")
|
|
|
|
|
|
def check_cpp_library(toolchain):
|
|
"""Check the renderer's C++20 library features before the long build (issues 39 and 100)."""
|
|
with tempfile.TemporaryDirectory() as d:
|
|
src = os.path.join(d, "t.cpp")
|
|
with open(src, "w") as f:
|
|
# Projected find: frontend/maxwell/structured_control_flow.cpp; predicate find and
|
|
# comparator sort: ir_opt/texture_pass.cpp (under gpu/vk/eden/shader_recompiler).
|
|
f.write(chr(10).join(("#include <algorithm>", "#include <format>", "#include <string>", "#include <vector>",
|
|
"std::string t(int v) { return std::vformat(\"{}\", std::make_format_args(v)); }",
|
|
"namespace { struct Entry { int value; }; }",
|
|
"bool ranges_test() {",
|
|
" std::vector<Entry> entries{{2}, {1}};",
|
|
" const auto found = std::ranges::find(entries, 1, &Entry::value);",
|
|
" const auto matching = std::ranges::find_if(entries, [](const Entry& e) { return e.value == 2; });",
|
|
" std::ranges::sort(entries, [](const Entry& a, const Entry& b) { return a.value < b.value; });",
|
|
" return found != entries.end() && matching != entries.end();",
|
|
"}", "")))
|
|
r = subprocess.run([toolchain.need(toolchain.find("clang++"))] + toolchain.native_target() + ["-std=c++20", "-c", src, "-o", os.path.join(d, "t.o")],
|
|
capture_output=True, text=True, encoding="utf-8", errors="replace")
|
|
if r.returncode:
|
|
print(r.stdout + r.stderr, flush=True)
|
|
fail("Your Visual Studio is too old for this game's renderer. Open the Visual Studio Installer, press Update on "
|
|
"Visual Studio 2022 (or install it), then build again.")
|
|
|
|
|
|
def check_linux_compiler():
|
|
"""(linux) A C compiler that builds and links a program here (the AppImage's own clang and sysroot, else the system's):
|
|
found now, not 40 minutes in"""
|
|
sys.path.insert(0, os.path.join(ROOT, "third_party", "a64-kit", "tools"))
|
|
import toolchain
|
|
cc = toolchain.find("clang")
|
|
if not os.path.isfile(cc):
|
|
fail("No C compiler was found. Install clang (and lld).")
|
|
sysroot = os.environ.get("HOOKSHOT_SYSROOT")
|
|
with tempfile.TemporaryDirectory() as d:
|
|
src = os.path.join(d, "t.c")
|
|
with open(src, "w") as f:
|
|
f.write(chr(10).join(("#include <stdio.h>", "#include <pthread.h>", "int main(void) { return 0; }", "")))
|
|
r = subprocess.run([cc] + (["--sysroot=" + sysroot] if sysroot else []) + ["-fuse-ld=lld", src, "-o", os.path.join(d, "t")],
|
|
capture_output=True, text=True, encoding="utf-8", errors="replace")
|
|
if r.returncode:
|
|
print(r.stdout + r.stderr, flush=True)
|
|
errors = [l for l in r.stderr.splitlines() if "error" in l]
|
|
fail("The compiler could not build a test program: %s (install clang, lld and your system's C library headers: "
|
|
"libc6-dev on Debian and Ubuntu, glibc-devel on Fedora)" % (errors[0] if errors else "no message"))
|
|
|
|
|
|
def requested_jobs(value):
|
|
"""The builder's explicit worker override, or None to keep its automatic selection."""
|
|
if value is None or value == "":
|
|
return None
|
|
error = "HOOKSHOT_JOBS must be a whole number: use 1 or more, or 0 for automatic."
|
|
try:
|
|
jobs = int(value)
|
|
except ValueError:
|
|
raise ValueError(error) from None
|
|
if jobs < 0:
|
|
raise ValueError(error)
|
|
return jobs or None
|
|
|
|
|
|
def default_jobs():
|
|
"""(issue 418) One compiler per CPU thread: nx2/build.py starts each one only when the memory free holds it (MemoryGate), so the
|
|
count no longer has to fit the biggest files in the memory free at the start. HOOKSHOT_JOBS or --jobs choose"""
|
|
cpus = os.cpu_count() or 2
|
|
print("build: up to %d compilers at once (one per CPU thread), as the memory allows; HOOKSHOT_JOBS=<n> to change" % cpus, flush=True)
|
|
return cpus
|
|
|
|
|
|
def chosen_jobs(given):
|
|
"""--jobs when it is given (1 or more), else HOOKSHOT_JOBS, else what default_jobs chooses: worked out only when it is
|
|
needed, so the "compilers at once" line is the number the build uses"""
|
|
if given:
|
|
return given
|
|
try:
|
|
return requested_jobs(os.environ.get("HOOKSHOT_JOBS")) or default_jobs()
|
|
except ValueError as e:
|
|
fail(str(e))
|
|
|
|
|
|
def bundled_mingw():
|
|
"""(as TotK Recomp's installer) llvm-mingw in the tools folder: its clang, headers and libraries build the game, and nothing of
|
|
Visual Studio or the Windows SDK is needed. HOOKSHOT_MINGW set already wins; HOOKSHOT_MSVC=1: the Visual Studio route"""
|
|
if LINUX or os.environ.get("HOOKSHOT_MSVC") == "1":
|
|
return os.environ.get("HOOKSHOT_MINGW", "")
|
|
if not os.environ.get("HOOKSHOT_MINGW"):
|
|
here = os.path.join(ROOT, "llvm-mingw")
|
|
if os.path.isfile(os.path.join(here, "bin", "clang" + (".exe" if os.name == "nt" else ""))):
|
|
os.environ["HOOKSHOT_MINGW"] = here
|
|
elif os.path.isdir(here):
|
|
# (the folder is there and its compiler is not: an antivirus took it (false positives on unsigned tools), or the
|
|
# zip was not unpacked whole. It went on to the Visual Studio route and said "The Windows SDK was not found")
|
|
missing = [n for n in ("clang.exe", "clang++.exe", "ld.lld.exe")
|
|
if not os.path.isfile(os.path.join(here, "bin", n))]
|
|
fail("Files of the builder's own compiler are missing (%s in %s). Your antivirus most likely removed them "
|
|
"(a false alarm on unsigned programs): restore them from its quarantine or allow the Hookshot Builder folder, "
|
|
"then unzip HookshotBuilder.zip again and build. Visual Studio is not needed."
|
|
% (", ".join(missing) or "bin", os.path.join(here, "bin")))
|
|
return os.environ.get("HOOKSHOT_MINGW", "")
|
|
|
|
|
|
def check_mingw():
|
|
"""The bundled compiler builds and links a Windows program: found now, not 40 minutes in"""
|
|
sys.path.insert(0, os.path.join(ROOT, "third_party", "a64-kit", "tools"))
|
|
import toolchain
|
|
cc = toolchain.find("clang") # (HOOKSHOT_MINGW's, else the one toolchain.py finds: the same as the build uses)
|
|
if not os.path.isfile(cc): # (HOOKSHOT_MINGW set to a folder without the compiler: said here, not as a silent SystemExit)
|
|
fail("The builder's own compiler was not found at %s. Set HOOKSHOT_MINGW to an llvm-mingw folder, or unpack the "
|
|
"Hookshot Builder zip again into a folder your antivirus allows." % cc)
|
|
with tempfile.TemporaryDirectory() as d:
|
|
src = os.path.join(d, "t.c")
|
|
with open(src, "w") as f:
|
|
f.write(chr(10).join(("#include <stdio.h>", "#include <windows.h>", "int main(void) { return 0; }", "")))
|
|
r = subprocess.run([cc] + toolchain.native_target() + toolchain.native_link() +
|
|
[src, "-o", os.path.join(d, "t.exe")], capture_output=True, text=True, encoding="utf-8", errors="replace")
|
|
if r.returncode:
|
|
print(r.stdout + r.stderr, flush=True)
|
|
errors = [l for l in r.stderr.splitlines() if "error" in l]
|
|
fail("The builder's own compiler (tools/llvm-mingw) could not build a test program: %s. Unpack the zip again into a folder "
|
|
"your antivirus allows." % (errors[0] if errors else "no message"))
|
|
print("the compiler: %s (no Visual Studio needed)" % os.environ["HOOKSHOT_MINGW"], flush=True)
|
|
|
|
|
|
def check_windows_sdk():
|
|
"""The compiler is bundled; Microsoft's C library and Windows SDK may not be, by their licences. Found now, not 40 minutes in."""
|
|
if LINUX:
|
|
return check_linux_compiler()
|
|
if bundled_mingw():
|
|
return check_mingw()
|
|
if CROSS:
|
|
fail("Building for Windows on Linux needs llvm-mingw: unpack its Linux release (github.com/mstorsjo/llvm-mingw) and set "
|
|
"HOOKSHOT_MINGW to its folder.")
|
|
sys.path.insert(0, os.path.join(ROOT, "third_party", "a64-kit", "tools"))
|
|
import toolchain
|
|
cc = toolchain.find("clang")
|
|
if not os.path.isfile(cc): # (toolchain.need() would raise SystemExit, which the builder cannot show: said in its protocol)
|
|
fail("No C compiler was found. Install clang (and lld) and put it on PATH, set EMSDK to an emsdk checkout, or set "
|
|
"HOOKSHOT_MINGW to an llvm-mingw folder. (%s)" % cc)
|
|
# (an INCLUDE set for the whole machine, by an old Visual Studio or another tool, is all clang reads: without the Windows SDK's C
|
|
# library in it, it is dropped and the compiler looks for Visual Studio itself. Issues 80 and 89)
|
|
for k in [n for n in os.environ if n.upper() == "INCLUDE"]:
|
|
if "ucrt" not in os.environ[k].lower():
|
|
print("INCLUDE is set on this PC without the Windows SDK's C library: not used (%s)" % os.environ[k][:200], flush=True)
|
|
for n in [n for n in os.environ if n.upper() in ("INCLUDE", "LIB", "LIBPATH")]:
|
|
del os.environ[n]
|
|
break
|
|
r = compile_test(toolchain)
|
|
found = None
|
|
if r.returncode: # (clang looks for Visual Studio its own way, and does not always find it: then ask Visual Studio)
|
|
found = visual_studio_environment()
|
|
if found:
|
|
print("the compiler did not find Visual Studio by itself: using vcvars64", flush=True)
|
|
os.environ.update(found)
|
|
r = compile_test(toolchain)
|
|
if r.returncode and found and ("stdio.h" in r.stderr or "windows.h" in r.stderr.lower()):
|
|
# (only on top of vcvars' INCLUDE: clang takes INCLUDE as all of its system headers once it is set)
|
|
kits = windows_kits_environment(found)
|
|
if kits:
|
|
for k in [n for n in os.environ if n.upper() in ("INCLUDE", "LIB")]:
|
|
del os.environ[k]
|
|
os.environ.update(kits)
|
|
r = compile_test(toolchain)
|
|
if not r.returncode:
|
|
check_cpp_library(toolchain)
|
|
link = link_test(toolchain)
|
|
if not link.returncode:
|
|
return
|
|
print(link.stdout + link.stderr, flush=True)
|
|
errors = [l for l in link.stderr.splitlines() if "error" in l]
|
|
why = errors[0].split("error:", 1)[-1].strip() if errors else (link.stderr.strip().splitlines() or ["no message"])[-1]
|
|
fail("The compiler could not link a test program: %s. In the Visual Studio Installer choose Modify on Visual Studio 2022, "
|
|
"tick \"Desktop development with C++\" with a Windows 11 SDK, then build again." % why)
|
|
print(r.stdout + r.stderr, flush=True) # (all of it, for Show details)
|
|
errors = [l for l in r.stderr.splitlines() if "error:" in l]
|
|
why = errors[0].split("error:", 1)[1].strip() if errors else (r.stderr.strip().splitlines() or ["no message"])[-1]
|
|
if "old c library" in why.lower():
|
|
fail("The compiler found an old Visual Studio's C library (from before 2015) instead of the Windows SDK's. In the Visual Studio "
|
|
"Installer choose Modify on Visual Studio 2022, tick \"Desktop development with C++\" with a Windows 11 SDK, then build again. (%s)" % why)
|
|
if "windows.h" in why.lower() or "stdio.h" in why.lower():
|
|
fail("The Windows SDK was not found. In the Visual Studio Installer choose Modify, and under \"Desktop development with C++\" tick a "
|
|
"Windows 11 SDK, then build again. (%s)" % why)
|
|
if "file not found" in why:
|
|
fail("Visual Studio's C++ tools were not found. In the Visual Studio Installer choose Modify, tick \"Desktop development with C++\" "
|
|
"and its MSVC x64/x86 build tools, then build again. (%s)" % why)
|
|
fail("The compiler could not run a test build: %s" % why)
|
|
|
|
|
|
OLD_NAMES = [(".hookshot_exefs", ".triforce_exefs"), (".hookshot_romfs", ".triforce_romfs"), ("hookshot_version.txt", "triforce_version.txt"),
|
|
("hookshot_env.txt", "triforce_env.txt"), ("hookshot_options.ini", "triforce_options.ini"),
|
|
("hookshot_shaders.bin", "triforce_shaders.bin"), ("hookshot_pipelines.bin", "triforce_pipelines.bin"),
|
|
("hookshot_programs.bin", "triforce_programs.bin"), ("hookshot_prepared.bin", "triforce_prepared.bin"),
|
|
("hookshot_prepare_skipped", "triforce_prepare_skipped"),
|
|
(os.path.join("user", "hookshot_update_skip.txt"), os.path.join("user", "triforce_update_skip.txt"))]
|
|
OLD_PROGRAM = ["hookshot", "hookshot_vk.so"] if LINUX else ["hookshot.exe", "hookshot_vk.dll", "hookshot_log.txt"]
|
|
|
|
|
|
def migrate_names(out):
|
|
"""(Triforce) A game folder built when the files were named hookshot_*: its files under their new names, so an update finds its game
|
|
files (.triforce_exefs) and the shader caches stay (no first-start compile again). A name already taken is left as it is"""
|
|
for old, new in OLD_NAMES:
|
|
a, b = os.path.join(out, old), os.path.join(out, new)
|
|
if os.path.exists(a) and not os.path.exists(b):
|
|
try:
|
|
os.replace(a, b)
|
|
print("renamed %s -> %s" % (old, new), flush=True)
|
|
except OSError as e:
|
|
print("could not rename %s: %s" % (old, e), flush=True)
|
|
|
|
|
|
def drop_old_program(out):
|
|
"""(Triforce) the old hookshot executable and renderer, once Triforce's are in the folder"""
|
|
if not os.path.exists(os.path.join(out, BESIDE[0])):
|
|
return
|
|
for name in OLD_PROGRAM:
|
|
p = os.path.join(out, name)
|
|
if os.path.exists(p):
|
|
try:
|
|
os.remove(p)
|
|
print("removed the old %s" % name, flush=True)
|
|
except OSError as e:
|
|
print("could not remove the old %s: %s" % (name, e), flush=True)
|
|
|
|
|
|
def stop_stray_compilers(work, default_work):
|
|
"""(issues 501, 530) A build left running by an earlier one on this work folder (the builder closed, crashed, or replaced by an
|
|
update mid-build) keeps its files open: the next build stopped at once with 'OSError: [Errno 22]' writing sa/gen/a64_fpx.h, or
|
|
WinError 1224 on the play profile. Only that build goes: an earlier make_game on this work folder, the build and lift steps
|
|
whose make_game is gone, and compilers whose command line names this work folder. Never this build or what started it."""
|
|
if LINUX:
|
|
return
|
|
ps = ("Get-CimInstance Win32_Process | Where-Object { $_.Name -like 'clang*' -or $_.Name -like 'ld.lld*' -or $_.Name -like 'python*' } "
|
|
"| Select-Object ProcessId,ParentProcessId,Name,CommandLine | ConvertTo-Json -Compress")
|
|
try:
|
|
import json
|
|
r = subprocess.run(["powershell", "-NoProfile", "-NonInteractive", "-Command", ps], capture_output=True, text=True, timeout=90)
|
|
procs = json.loads(r.stdout or "[]")
|
|
if isinstance(procs, dict):
|
|
procs = [procs]
|
|
alive = {q["ProcessId"]: q for q in procs}
|
|
mine, pid = set(), os.getpid() # (this process and the chain that started it)
|
|
while pid in alive and pid not in mine:
|
|
mine.add(pid)
|
|
pid = alive[pid]["ParentProcessId"]
|
|
mine.add(pid)
|
|
w = os.path.normcase(os.path.abspath(work))
|
|
names = lambda c: os.path.normcase((c or "").replace("/", "\\"))
|
|
doomed = {}
|
|
for q in procs:
|
|
c, n = names(q.get("CommandLine")), q["Name"].lower()
|
|
if q["ProcessId"] in mine:
|
|
continue
|
|
if "make_game.py" in c and (w in c or ("--work" not in c and w == os.path.normcase(default_work))):
|
|
doomed[q["ProcessId"]] = n
|
|
elif ("build.py" in c or "lift.py" in c) and "--standalone" in c and q["ParentProcessId"] not in alive:
|
|
doomed[q["ProcessId"]] = n # (its make_game is gone)
|
|
elif not n.startswith("python") and w in c:
|
|
doomed[q["ProcessId"]] = n
|
|
for k in doomed:
|
|
subprocess.run(["taskkill", "/F", "/T", "/PID", str(k)], capture_output=True, timeout=60)
|
|
if doomed:
|
|
print("stopped %d process(es) an earlier build left running on this work folder: %s"
|
|
% (len(doomed), " ".join(sorted(set(doomed.values())))), flush=True)
|
|
time.sleep(2)
|
|
except Exception as e: # (not finding them is no reason to stop)
|
|
print("could not look for an earlier build's processes: %s" % e, flush=True)
|
|
|
|
|
|
def makefile_words(text):
|
|
"""Decode clang -M words, preserving ordinary Windows backslashes."""
|
|
text = re.sub(r"\\\r?\n", " ", text)
|
|
words = re.findall(r"(?:\\[ \t]|\S)+", text)
|
|
# clang doubles backslashes before an escaped space, but adds only one before #.
|
|
def unescape(m):
|
|
slashes, char = m.groups()
|
|
return "\\" * ((len(slashes) - 1) // 2 if char != "#" else len(slashes) - 1) + char
|
|
return [re.sub(r"(\\+)([ \t#])", unescape, word).replace("$$", "$") for word in words]
|
|
|
|
|
|
def _drop_stale_linux(work):
|
|
"""The AppImage's sysroot is a new /tmp/.mount_... path every start. The stamp is the compiler's version and stdio.h's
|
|
bytes, not that path, so an update reuses the compiled game when the headers did not change (the Windows stamp leaves
|
|
the builder's install folder out for the same reason, issue 565)."""
|
|
import hashlib
|
|
sysroot = os.environ.get("HOOKSHOT_SYSROOT", "")
|
|
stdio = os.path.join(sysroot, "usr", "include", "stdio.h") if sysroot else "/usr/include/stdio.h"
|
|
try:
|
|
body = open(stdio, "rb").read()
|
|
except OSError:
|
|
return
|
|
sys.path.insert(0, os.path.join(ROOT, "third_party", "a64-kit", "tools"))
|
|
import toolchain
|
|
try:
|
|
cc = toolchain.need(toolchain.find("clang"))
|
|
ver = subprocess.run([cc, "--version"], capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=30)
|
|
except Exception:
|
|
return
|
|
first = ver.stdout.splitlines()[:1]
|
|
if ver.returncode or not first:
|
|
return
|
|
stamp = hashlib.sha1(body + first[0].encode(), usedforsecurity=False).hexdigest()
|
|
note = os.path.join(work, "system_headers.txt")
|
|
try:
|
|
with open(note) as f:
|
|
if f.read().strip() == stamp:
|
|
return
|
|
except OSError:
|
|
pass
|
|
for sub in (os.path.join(work, "objcache"), os.path.join(work, "sa", "build", "obj")):
|
|
if os.path.isdir(sub):
|
|
print("the system headers changed since the last build: its compiled objects are not reused (%s)" % sub, flush=True)
|
|
shutil.rmtree(sub, ignore_errors=True)
|
|
with open(note, "w") as f:
|
|
f.write(stamp + chr(10))
|
|
|
|
|
|
def prune_object_cache(work):
|
|
"""(issue 1287) The object cache down to the objects this build used: each object in sa/build/obj names its key in <obj>.sha1
|
|
(nx2/build.py compile_one); cached objects no stamp names (earlier releases', other settings') go. Returns the bytes kept.
|
|
Nothing is pruned when no stamp is found (a build that did not reach the compile keeps the cache as it was)"""
|
|
cache, obj_dir = os.path.join(work, "objcache"), os.path.join(work, "sa", "build", "obj")
|
|
used = set()
|
|
if os.path.isdir(obj_dir):
|
|
for f in os.listdir(obj_dir):
|
|
if f.endswith(".sha1"):
|
|
try:
|
|
with open(os.path.join(obj_dir, f)) as h:
|
|
used.add(h.read().strip())
|
|
except OSError:
|
|
pass
|
|
kept = 0
|
|
if not os.path.isdir(cache):
|
|
return kept
|
|
for f in os.listdir(cache):
|
|
p = os.path.join(cache, f)
|
|
if used and f.endswith(".o") and f[:-2] not in used:
|
|
try:
|
|
os.remove(p)
|
|
except OSError:
|
|
pass
|
|
continue
|
|
try:
|
|
kept += os.path.getsize(p)
|
|
except OSError:
|
|
pass
|
|
return kept
|
|
|
|
|
|
def clear_folder(path):
|
|
"""(issue 1354) A folder this build makes again from nothing (the game's code, exefs): gone before it is made. rmtree with
|
|
ignore_errors left it behind whenever a file in it was held (an antivirus scan, an explorer window, a game still closing), and the
|
|
copy into it stopped with FileExistsError at every update. Tried a few times, then moved aside (deleted at the next build), and
|
|
only then a clear stop"""
|
|
for attempt in range(5):
|
|
shutil.rmtree(path, ignore_errors=True)
|
|
if not os.path.exists(path):
|
|
return
|
|
time.sleep(1)
|
|
parent, name = os.path.split(path.rstrip("\\/"))
|
|
for stale in os.listdir(parent or "."): # (what an earlier build moved aside: gone now, if it can be)
|
|
if stale.startswith(name + ".old-"):
|
|
shutil.rmtree(os.path.join(parent, stale), ignore_errors=True)
|
|
aside = "%s.old-%d" % (path.rstrip("\\/"), os.getpid())
|
|
try:
|
|
os.rename(path, aside)
|
|
print("could not delete %s (a file in it is in use): moved aside to %s" % (path, aside), flush=True)
|
|
return
|
|
except OSError as e:
|
|
fail("Could not clear %s (%s). A file in it is in use: close any window showing that folder, wait for your antivirus "
|
|
"to finish, then try again." % (path, e))
|
|
|
|
|
|
def drop_stale_objects(work):
|
|
"""(issue 231) Objects compiled against other system headers are not reused. A first build that compiled with an old Visual Studio's
|
|
stdio.h (before 2015) left its objects in the cache; once the PC was fixed every later build reused them and the link failed on
|
|
__iob_func. The headers the compiler finds are noted, and when they change the compiled objects go"""
|
|
if LINUX:
|
|
_drop_stale_linux(work)
|
|
return
|
|
sys.path.insert(0, os.path.join(ROOT, "third_party", "a64-kit", "tools"))
|
|
import hashlib
|
|
import toolchain
|
|
with tempfile.TemporaryDirectory() as d:
|
|
src = os.path.join(d, "t.c")
|
|
with open(src, "w") as f:
|
|
f.write(chr(10).join(("#include <stdio.h>", "#include <stdlib.h>", "#include <windows.h>", "")))
|
|
r = subprocess.run([toolchain.need(toolchain.find("clang"))] + toolchain.native_target() + ["-M", src], capture_output=True, text=True,
|
|
encoding="utf-8", errors="replace")
|
|
if r.returncode:
|
|
return
|
|
found = sorted({h.lower() for h in makefile_words(r.stdout) if h.endswith(".h")})
|
|
|
|
# (issue 565) The paths are taken relative to the builder's own folder, with the compiler's version: under
|
|
# versions\<tag>\ every release looked like new headers, and an update compiled everything again (2+ hours). clang
|
|
# writes them with / where the folder has \: compared with / on both sides, each path's own spelling kept for the old stamp
|
|
mine = os.path.normcase(os.path.abspath(ROOT)).lower().replace(os.sep, "/")
|
|
ours = lambda h: h.replace(os.sep, "/").startswith(mine + "/")
|
|
|
|
def stamp_for(root): # (the old stamp: the headers' full paths, with the builder's own folder as it was in that version)
|
|
root = root.replace(os.sep, "/")
|
|
moved = [((root if "/" in h[:len(mine)] else root.replace("/", os.sep)) + h[len(mine):]) if ours(h) else h for h in found]
|
|
return hashlib.sha1(chr(10).join(sorted(moved)).encode(), usedforsecurity=False).hexdigest()
|
|
|
|
rel = sorted(("<tools>" + h[len(mine):].replace(os.sep, "/")) if ours(h) else h for h in found)
|
|
ver = subprocess.run([toolchain.need(toolchain.find("clang")), "--version"], capture_output=True, text=True, encoding="utf-8",
|
|
errors="replace").stdout.splitlines()[:1]
|
|
stamp = hashlib.sha1(chr(10).join(ver + rel).encode(), usedforsecurity=False).hexdigest()
|
|
note = os.path.join(work, "system_headers.txt")
|
|
try:
|
|
with open(note) as f:
|
|
had = f.read().strip()
|
|
except OSError:
|
|
had = ""
|
|
if had == stamp:
|
|
return
|
|
# (a stamp an installed version of the builder wrote the old way: the same headers, in its own folder; kept, and written anew)
|
|
versions = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(ROOT))))
|
|
olds = {stamp_for(mine)}
|
|
if os.path.basename(versions).lower() == "versions" and os.path.isdir(versions):
|
|
for v in os.listdir(versions):
|
|
olds.add(stamp_for(os.path.normcase(os.path.join(versions, v, "HookshotBuilder", "tools")).lower()))
|
|
if had and had in olds:
|
|
print("the system headers are the same as the last build's (an earlier version of the builder): its objects are reused", flush=True)
|
|
with open(note, "w") as f:
|
|
f.write(stamp + chr(10))
|
|
return
|
|
for sub in (os.path.join(work, "objcache"), os.path.join(work, "sa", "build", "obj")):
|
|
if os.path.isdir(sub):
|
|
print("the system headers changed since the last build: its compiled objects are not reused (%s)" % sub, flush=True)
|
|
shutil.rmtree(sub, ignore_errors=True)
|
|
with open(note, "w") as f:
|
|
f.write(stamp + chr(10))
|
|
|
|
|
|
def seed_pipelines(out):
|
|
"""The pipelines recorded on the development PC (hookshot_pipelines_seed.bin in the tools: program keys and draw states, no game
|
|
code), as the game folder's list, so its first start builds them before the title, as TotK Recomp's shipped shader list does,
|
|
instead of compiling each when it is first drawn (the stutter of a first visit). Each release ships the list its own build
|
|
made: the file's first 8 bytes are that build's stamp, and a list the folder has from the same build is kept (it grows as
|
|
the player plays)"""
|
|
seed = os.path.join(ROOT, "hookshot_pipelines_seed.bin")
|
|
dst = os.path.join(out, "triforce_pipelines.bin")
|
|
if not os.path.isfile(seed):
|
|
return
|
|
with open(seed, "rb") as f:
|
|
stamp = f.read(8)
|
|
try:
|
|
with open(dst, "rb") as f:
|
|
if f.read(8) == stamp:
|
|
return
|
|
except OSError:
|
|
pass
|
|
shutil.copyfile(seed, dst)
|
|
print("the shipped pipeline list: %d KB, built before the first title" % (os.path.getsize(seed) >> 10), flush=True)
|
|
|
|
|
|
def ship_state_list(out):
|
|
"""The pipeline state list made from a play of the game on the development PC (work/pipeline_states.bin: each pipeline's stage
|
|
code keys, vertex and target formats and draw state; hashes and state only, no game code), as the folder's
|
|
triforce_pipeline_states_shipped.bin: the first start's shader preparation builds those pipelines before the title, from the
|
|
player's own shaders (triforce_shaders.bin), instead of compiling each when it is first drawn. Good for any build: optional"""
|
|
src = os.path.join(ROOT, "work", "pipeline_states.bin")
|
|
dst = os.path.join(out, "triforce_pipeline_states_shipped.bin")
|
|
if not os.path.isfile(src):
|
|
return
|
|
if not (os.path.isfile(dst) and filecmp.cmp(src, dst, shallow=False)):
|
|
shutil.copyfile(src, dst)
|
|
print("the shipped pipeline state list: %d pipelines" % ((os.path.getsize(src) - 12) // 528), flush=True)
|
|
|
|
|
|
def ship_triforce_assets(out):
|
|
"""(Triforce, v0.5.0) Triforce's own files the game reads beside it (assets/triforce_assets: the Mods screen's mask, CC BY 4.0,
|
|
its credits in CREDITS.txt), as <out>/triforce_assets. None is a game file. Without them the game runs as before: the Mask
|
|
Salesman stands without the mask (gpu/vk/vk_salesman.cpp)"""
|
|
src = os.path.join(ROOT, "assets", "triforce_assets")
|
|
if not os.path.isdir(src):
|
|
return
|
|
dst = os.path.join(out, "triforce_assets")
|
|
os.makedirs(dst, exist_ok=True)
|
|
for name in sorted(os.listdir(src)):
|
|
s, d = os.path.join(src, name), os.path.join(dst, name)
|
|
if os.path.isfile(s) and not (os.path.isfile(d) and filecmp.cmp(s, d, shallow=False)):
|
|
copy_writable(s, d)
|
|
print("Triforce's own files beside the game: %s" % ", ".join(sorted(os.listdir(src))), flush=True)
|
|
|
|
|
|
# (the builders read this line's start: the step was skipped or failed, and the shaders' row says so instead of a check mark)
|
|
NOT_PREPARED = "the shaders were not prepared now"
|
|
# (a run that prints nothing this long is stopped. Not its whole length: that is 25 minutes on some AMD GPUs, and the driver's own
|
|
# step near the end can sit at 99% for minutes)
|
|
PREPARE_SILENCE = 15 * 60
|
|
GAME_ENV_DROP = ("HOOKSHOT_", "PYTHON", "APPDIR", "APPIMAGE", "ARGV0", "OWD")
|
|
|
|
|
|
def span(seconds):
|
|
return "%d minutes" % (seconds // 60) if seconds >= 120 else "%g seconds" % seconds
|
|
|
|
|
|
def game_env(environ=None):
|
|
"""The environment the game runs in when the tools start it: without what the AppImage runtime and the tools put there (their
|
|
libraries first on the search path, Python's settings, the HOOKSHOT_ build switches), so it loads its own SDL and the system's
|
|
Vulkan, as when the player starts it. The Linux builder's Play button uses it too; builder/linux/hookshot-launch drops the same"""
|
|
env = {k: v for k, v in (os.environ if environ is None else environ).items() if not k.startswith(GAME_ENV_DROP)}
|
|
env.pop("LD_LIBRARY_PATH", None)
|
|
return env
|
|
|
|
|
|
def not_prepared(why):
|
|
print("%s (%s): the game prepares them at its first start" % (NOT_PREPARED, why), flush=True)
|
|
|
|
|
|
def prepare_skipped(a, environ=None):
|
|
"""Why the build leaves the shaders to the first start, or None: it prepares them"""
|
|
environ = os.environ if environ is None else environ
|
|
if a.no_prepare_shaders:
|
|
return '"Prepare shaders now" is unticked'
|
|
if environ.get("HOOKSHOT_NO_PREPARE", "0") != "0": # (set and not 0, as the game reads HOOKSHOT_NO_UPDATE_CHECK)
|
|
return "HOOKSHOT_NO_PREPARE is set"
|
|
if CROSS: # (Triforce.exe can't run here, and under Wine it would prepare for this PC's GPU, not the one that plays it)
|
|
return "a Windows game built on Linux prepares them on the PC it runs on"
|
|
return None
|
|
|
|
|
|
def show_prepared(line):
|
|
"""The game's @@stage and @@progress lines go on to the builder; its own @@done and @@error would end or fail the build there,
|
|
so they go on as plain text"""
|
|
return line if line.startswith(("@@stage ", "@@progress ")) or not line.startswith("@@") else line[2:]
|
|
|
|
|
|
def prepare_shaders(cmd, folder=None, silence=PREPARE_SILENCE):
|
|
"""The shaders prepared now, under the builder's progress bar, not at the game's first start behind a black window (minutes on
|
|
some GPUs, issue 711): hookshot --prepare-shaders (host/hos_main.c, gpu/vk/vk_main.cpp), the same work as that first start, in
|
|
the game's folder and environment, as the builders' Play starts it. A PC whose Vulkan can't (exit 3), a run silent for `silence` seconds (then stopped), or
|
|
any other failure leaves the work to the first start, as before: never a failed build. Returns its exit code (None: stopped)"""
|
|
try:
|
|
code = run(cmd, check=False, show=show_prepared, cwd=folder or os.path.dirname(cmd[0]) or None,
|
|
base=game_env(), silence=silence)
|
|
except OSError as e:
|
|
not_prepared(str(e))
|
|
return 1
|
|
if code is None:
|
|
not_prepared("the game printed nothing for %s and was stopped" % span(silence))
|
|
elif code == 3:
|
|
not_prepared("no usable Vulkan here")
|
|
elif code:
|
|
not_prepared("the game stopped with exit status %d" % code)
|
|
return code
|
|
|
|
|
|
def folder_language(out):
|
|
"""the language a game folder was built in: the code line of its language.txt"""
|
|
try:
|
|
with open(os.path.join(out, "language.txt"), encoding="utf-8") as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if line and not line.startswith("#") and any(line == code for code, _ in LANGUAGES):
|
|
return line
|
|
except OSError:
|
|
pass
|
|
return None
|
|
|
|
|
|
def profile_readable(prof):
|
|
"""(linux) the profile is made by a recent clang (the Windows tools'): an older one, the system's, cannot read it and stops the
|
|
build. Then the game is built without it (a little slower in play), not at all"""
|
|
if not LINUX:
|
|
return True
|
|
sys.path.insert(0, os.path.join(ROOT, "third_party", "a64-kit", "tools"))
|
|
import toolchain
|
|
with tempfile.TemporaryDirectory() as d:
|
|
src = os.path.join(d, "t.c")
|
|
with open(src, "w") as f:
|
|
f.write("int f(int x) { return x + 1; }" + chr(10))
|
|
r = subprocess.run([toolchain.find("clang"), "-O2", "-Wno-everything", "-fprofile-instr-use=" + prof, "-c", src, "-o",
|
|
os.path.join(d, "t.o")], capture_output=True, text=True, encoding="utf-8", errors="replace")
|
|
if r.returncode:
|
|
print("this compiler cannot read the play profile (%s): building without it" % (r.stderr.strip().splitlines() or ["?"])[-1],
|
|
flush=True)
|
|
return r.returncode == 0
|
|
|
|
|
|
def copy_writable(src, dst):
|
|
"""src's bytes at dst, writable whatever src's mode: copy2 took the mode along, and from a read-only install (a Nix
|
|
store, a packaged tools folder) the copies in the work folder were read-only, so lift.py could not write chunks.txt
|
|
again (PermissionError) and every later build failed on the same file"""
|
|
if os.path.exists(dst):
|
|
os.chmod(dst, stat.S_IREAD | stat.S_IWRITE)
|
|
shutil.copyfile(src, dst)
|
|
|
|
|
|
def refuse_dump_links(folder):
|
|
"""An extracted dump holds only files and folders: a link in it was followed, and hard-linked or copied the file it
|
|
names (anywhere on this PC) into the game folder. The folder itself may be one. (Walked as a long path: #1468)"""
|
|
for d, dirs, files in os.walk(long_path(folder)):
|
|
for name in dirs + files:
|
|
path = os.path.join(d, name)
|
|
attrs = os.lstat(path)
|
|
if stat.S_ISLNK(attrs.st_mode) or getattr(attrs, "st_file_attributes", 0) & getattr(
|
|
stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400):
|
|
fail("The game files at %s contain a link (%s): copy them without links and choose that folder."
|
|
% (folder, path))
|
|
|
|
|
|
PACKED_ROMFS = ("This romfs folder is NX2navi's: it holds the game's data packed in one file (romfs.bin), which Triforce cannot "
|
|
"read, so the game would crash at start. Choose your game file (.nup) instead, or romfs and exefs folders with "
|
|
"the game's files unpacked in them.")
|
|
|
|
|
|
def refuse_packed_romfs(folder):
|
|
"""(Triforce, 11-18) NX2navi keeps the game's RomFS as one image, <its game folder>\\romfs\\romfs.bin (+ index.json), not as
|
|
files: chosen as the romfs folder, it passed (only the exefs is checked) and the game was built without its data. It then
|
|
crashed at start in the game's own setup, a 0x268-byte allocation coming back null (f_80c4afa0: two players, 11-18)"""
|
|
if folder and os.path.isfile(os.path.join(folder, "romfs.bin")):
|
|
fail(PACKED_ROMFS)
|
|
|
|
|
|
def long_path(p):
|
|
"""(Windows) p as an extended-length path (\\\\?\\...): a game folder whose deepest file passes 260 characters still links
|
|
and is read (#1468); Python takes these without Windows' long-paths setting. Elsewhere, or one already: as is (nup.py's)"""
|
|
if os.name != "nt" or not p:
|
|
return p
|
|
p = os.path.abspath(p)
|
|
if p.startswith("\\\\?\\"):
|
|
return p
|
|
return "\\\\?\\UNC\\" + p[2:] if p.startswith("\\\\") else "\\\\?\\" + p
|
|
|
|
|
|
def link_or_copy(src, dst):
|
|
"""the romfs by hard links where the disk allows: 26 GB not written twice"""
|
|
src, dst = long_path(src), long_path(dst)
|
|
files = [os.path.join(d, f) for d, _, fs in os.walk(src) for f in fs]
|
|
for i, s in enumerate(files):
|
|
t = os.path.join(dst, os.path.relpath(s, src))
|
|
os.makedirs(os.path.dirname(t), exist_ok=True)
|
|
if not os.path.exists(t):
|
|
try:
|
|
os.link(s, t)
|
|
except OSError:
|
|
# A retry skips existing targets, so publish the copy only after it is complete.
|
|
fd, staged = tempfile.mkstemp(prefix=".romfs-", dir=os.path.dirname(t))
|
|
try:
|
|
os.close(fd)
|
|
shutil.copy2(s, staged)
|
|
os.replace(staged, t)
|
|
except BaseException:
|
|
with contextlib.suppress(FileNotFoundError):
|
|
if os.name == "nt":
|
|
# copy2 can carry the source's read-only attribute over.
|
|
os.chmod(staged, stat.S_IWRITE)
|
|
os.unlink(staged)
|
|
raise
|
|
if i % 500 == 0:
|
|
say("progress", "%.3f" % (i / max(1, len(files))))
|
|
|
|
|
|
def shader_census_cmd(romfs, out):
|
|
"""The shader list, under the same UTF-8 mode as the other steps. -I alone follows the console's code page, and an
|
|
archive name that page cannot print stopped this step: the game then compiled its shaders as it played."""
|
|
return PY + [os.path.join(HERE, "shader_census.py"), long_path(romfs), "--out", out]
|
|
|
|
|
|
LIFT_KEY = ".lift_key" # (in <work>/sa/gen: what the lifted C there was made from)
|
|
|
|
|
|
def compile_alone_failed(lines):
|
|
"""Whether a build that failed in nx2/build.py failed in its compile and never came to a link: build.py's own lines say
|
|
so ("compiled in 412s, 2 failed", and no "link FAILED" after it). The objects that compile did make are then whole,
|
|
each stamped and cached under the hash of its source and flags only once its compiler ended well, and are worth
|
|
keeping: a compiler that ran out of memory, or a disk that filled, used to cost the next build every one of the ~600
|
|
files again (make_game deleted them all), an hour and more on the PCs where a compile fails in the first place. A
|
|
build that got as far as its link and failed there, or that ended without saying how its compile went, is not
|
|
trusted: its objects go, as before (a stale object linked "compiled in 0s" at every try, issue 1197)"""
|
|
failed = None
|
|
for line in lines:
|
|
m = re.match(r"compiled in \d+s, (\d+) failed", line)
|
|
if m:
|
|
failed = int(m.group(1))
|
|
elif "link FAILED" in line:
|
|
return False
|
|
return bool(failed)
|
|
PROMPT_KEY = ".made_from" # (in <out>/prompts: prompts_key of what that folder was made from)
|
|
|
|
|
|
def prompts_key(out):
|
|
"""What the button prompts of other controllers are made from, as one hash: the tools that draw them (nx2/prompt_*.py,
|
|
the faces' drawings and parts in nx2/prompt_faces, the icon fonts in third_party/kenney_input), the packages they are
|
|
drawn with (nx2/prompt_requirements.txt) and the game's files, by the unpack's stamp (.triforce_romfs: the .nup's name
|
|
and size). None when the romfs has no such stamp (the player's own folders, linked in): its prompts are made at every
|
|
build, as before. (Triforce) The keyboard and mouse face's too: nx2/prompt_keys.py, which draws it, and the font its
|
|
labels are set in (third_party/fonts/libertinus: that one file, the menus' other fonts are not the prompts')"""
|
|
stamp = os.path.join(out, ".triforce_romfs")
|
|
if not os.path.isfile(stamp):
|
|
return None
|
|
# (each file under a name that does not depend on where the tools are installed)
|
|
files = [("nx2/" + n, os.path.join(HERE, n))
|
|
for n in ("prompt_assets.py", "prompt_draw.py", "prompt_faces.py", "prompt_requirements.txt", "prompt_keys.py")]
|
|
labels = "third_party/fonts/libertinus/LibertinusSans-Bold.ttf"
|
|
files.append((labels, os.path.join(ROOT, *labels.split("/"))))
|
|
for name, root in (("nx2/prompt_faces", os.path.join(HERE, "prompt_faces")),
|
|
("third_party/kenney_input", os.path.join(ROOT, "third_party", "kenney_input"))):
|
|
for folder, dirs, names in os.walk(root):
|
|
dirs.sort()
|
|
files += [(name + "/" + os.path.relpath(os.path.join(folder, n), root).replace(os.sep, "/"), os.path.join(folder, n))
|
|
for n in sorted(names)]
|
|
h = hashlib.sha1()
|
|
with open(stamp, "rb") as f:
|
|
h.update(f.read())
|
|
for name, path in files:
|
|
if os.path.isfile(path):
|
|
h.update(name.encode())
|
|
with open(path, "rb") as f:
|
|
h.update(f.read())
|
|
return h.hexdigest()
|
|
|
|
|
|
class EarlyCompile:
|
|
"""(Windows) The files that need nothing of the translated game, compiled while the game is being translated
|
|
(nx2/build.py RX_EARLY): libopus and the two renderers with their libraries, ~520 of the build's ~630 files. The lift was
|
|
one process there and left every other processor idle for its two minutes, and then the compile had those files to do
|
|
on top of the game's. (The lift's emit has processes of its own there now, nx2/emit_pool.py: at most half the
|
|
processors, and its first part, the discovery, is still one process.) finish() waits for it before the build proper starts, which then finds their objects made.
|
|
A failure here costs nothing: the build compiles whatever is missing, as before. HOOKSHOT_EARLY_COMPILE=0: not done"""
|
|
|
|
def __init__(self, work, env, jobs):
|
|
self.proc = None
|
|
if os.name != "nt" or os.environ.get("HOOKSHOT_EARLY_COMPILE") == "0":
|
|
return
|
|
self.log = os.path.join(work, "early_compile.log")
|
|
e = dict(os.environ, PYTHONIOENCODING="utf-8")
|
|
e.update(env)
|
|
e["RX_EARLY"] = "1"
|
|
e.pop("HOOKSHOT_TIMES", None) # (no countdown lines: nobody reads them here)
|
|
with open(self.log, "w", encoding="utf-8") as log:
|
|
self.proc = subprocess.Popen(PY + ["nx2/build.py", "--standalone", "--jobs", str(max(1, jobs - 1))], cwd=ROOT, env=e,
|
|
stdout=log, stderr=subprocess.STDOUT)
|
|
no_throttle(getattr(self.proc, "_handle", None))
|
|
|
|
def stop(self):
|
|
if self.proc is not None and self.proc.poll() is None:
|
|
self.proc.kill()
|
|
self.proc.wait()
|
|
|
|
def finish(self):
|
|
if self.proc is None:
|
|
return
|
|
waited = time.time()
|
|
code = self.proc.wait()
|
|
said = ""
|
|
try:
|
|
with open(self.log, encoding="utf-8", errors="replace") as f:
|
|
said = next((line.strip() for line in reversed(f.read().splitlines()) if line.startswith("compiled in ")), "")
|
|
except OSError:
|
|
pass
|
|
print("the files that need no translated code were compiled beside the lift: %s%s (%s more after the lift ended)%s"
|
|
% (said or "no result", "" if not code else ", stopped with status %d" % code, clock(time.time() - waited),
|
|
"" if not code else ": the build compiles what is missing"), flush=True)
|
|
self.proc = None
|
|
|
|
|
|
class Prompts:
|
|
"""The button prompts of other controllers (host/prompts.c): the game's icon font and textures redrawn per controller by
|
|
nx2/prompt_assets.py, with fontTools. It took 87 s at the end of every build and every update, with nothing else
|
|
running. It reads only the game's files, so it now starts as soon as they are read, beside the lift and the compile,
|
|
into <out>/prompts.new, and finish() puts the folder in place; and it is not run at all when the folder there was made
|
|
from the same tools and the same game files (prompts_key). Without it, or when it fails, the game shows its own icons,
|
|
as before"""
|
|
|
|
def __init__(self, romfs, out, work):
|
|
self.out, self.folder, self.new = out, os.path.join(out, "prompts"), os.path.join(out, "prompts.new")
|
|
self.key = prompts_key(out)
|
|
self.proc = self.log = None
|
|
self.state = "no fontTools" if importlib.util.find_spec("fontTools") is None else None
|
|
if self.state:
|
|
return
|
|
try:
|
|
with open(os.path.join(self.folder, PROMPT_KEY), encoding="utf-8") as f:
|
|
if self.key and f.read().strip() == self.key:
|
|
self.state = "kept"
|
|
return
|
|
except OSError:
|
|
pass
|
|
shutil.rmtree(self.new, ignore_errors=True)
|
|
self.log = os.path.join(work, "prompts.log")
|
|
with open(self.log, "w", encoding="utf-8") as log:
|
|
self.proc = subprocess.Popen(PY + [os.path.join(HERE, "prompt_assets.py"), "--romfs", romfs, "--out", self.new],
|
|
cwd=ROOT, env=dict(os.environ, PYTHONIOENCODING="utf-8"), stdout=log,
|
|
stderr=subprocess.STDOUT)
|
|
no_throttle(getattr(self.proc, "_handle", None))
|
|
|
|
def stop(self):
|
|
"""(the build failed or was stopped) the drawing ended and what it wrote taken away: the folder in place stays"""
|
|
if self.proc is not None and self.proc.poll() is None:
|
|
self.proc.kill()
|
|
self.proc.wait()
|
|
shutil.rmtree(self.new, ignore_errors=True)
|
|
|
|
def finish(self):
|
|
"""Waits for the drawing, says what it said, and puts the new folder in the old one's place"""
|
|
if self.state == "no fontTools":
|
|
shutil.rmtree(self.folder, ignore_errors=True)
|
|
print("the button prompts of other controllers were not made (no fontTools): the game shows its own", flush=True)
|
|
return
|
|
if self.state == "kept":
|
|
print("the button prompts of other controllers are as the last build made them (the same tools and game files): kept",
|
|
flush=True)
|
|
return
|
|
code = self.proc.wait()
|
|
try:
|
|
with open(self.log, encoding="utf-8", errors="replace") as f:
|
|
sys.stderr.write(f.read())
|
|
sys.stderr.flush()
|
|
except OSError:
|
|
pass
|
|
shutil.rmtree(self.folder, ignore_errors=True)
|
|
if code:
|
|
print("the button prompts of other controllers were not made: the game shows its own", flush=True)
|
|
shutil.rmtree(self.new, ignore_errors=True)
|
|
return
|
|
os.replace(self.new, self.folder)
|
|
if self.key:
|
|
with open(os.path.join(self.folder, PROMPT_KEY), "w", encoding="utf-8") as f:
|
|
f.write(self.key + chr(10))
|
|
|
|
|
|
# (Triforce) Files of nx2/ that are not the translator's and that the lift (nx2/lift.py, nx2/split_hot.py and what they
|
|
# import) cannot read: this driver, the steps before and after the lift, the version, the button prompts, and the tests
|
|
# kept in nx2/. They were part of the lift key, so an update that changed one of them, and every release changes the
|
|
# version and most change this file, translated the whole game again on every player's PC: 2:48 of a 3:05 update build
|
|
# on a 16-thread PC, for C that came out the same byte for byte. tests/test_lift_key_files.py holds the list to what
|
|
# the lift can import and to what its files name.
|
|
LIFT_KEY_NOT = ("make_game.py", "build.py", "nup.py", "version.py", "shader_census.py", "prompt_assets.py", "prompt_draw.py",
|
|
"prompt_faces.py", "prompt_requirements.txt", "prompt_keys.py",
|
|
"code_mods.py") # (the mods' code patches: what it does to the game's code is in the key, the exefs's files)
|
|
|
|
|
|
def lift_key_file(rel):
|
|
"""Whether a file of nx2/ (rel: its path under nx2/, with /) is part of the lift key: all but LIFT_KEY_NOT, the
|
|
prompt_faces folder and the tests (test_*.py)"""
|
|
name = rel.rsplit("/", 1)[-1]
|
|
return not (rel in LIFT_KEY_NOT or rel.startswith("prompt_faces/") or (name.startswith("test_") and name.endswith(".py")))
|
|
|
|
|
|
def lift_key(exefs, work, env, chunks=True):
|
|
"""What the lift's output depends on: the game's code, the translator (nx2/ but for LIFT_KEY_NOT, the vendored a64-kit), the HLE
|
|
functions host/ defines (hooked in the lift; the rest of host/ is compiled after it, so a change there alone does not lift again),
|
|
the hot list and chunk boundaries, the lift's environment and the Python running it. A file the lift never reads may be in it (a
|
|
new build then lifts again); one it reads must be. chunks=False: all of it but the chunk boundaries (key_after_lift)"""
|
|
import kit # noqa: F401 (a64-kit on the path, for standalone)
|
|
import standalone
|
|
h = hashlib.sha1(sys.version.encode(), usedforsecurity=False)
|
|
h.update(" ".join(sorted(standalone.implemented())).encode())
|
|
# (each file named from its own base: the exefs and the work folder may be on another drive than the source, which relpath refuses)
|
|
files = [(exefs, os.path.join(exefs, f)) for f in sorted(os.listdir(exefs))]
|
|
for top in (HERE, os.path.join(ROOT, "third_party", "a64-kit")):
|
|
for d, dirs, fs in os.walk(top):
|
|
dirs[:] = sorted(x for x in dirs if x != "__pycache__")
|
|
files += [(ROOT, os.path.join(d, f)) for f in sorted(fs) if f.endswith((".py", ".c", ".h", ".cpp", ".hpp", ".txt", ".json"))
|
|
and (top != HERE or lift_key_file(os.path.relpath(os.path.join(d, f), HERE).replace(os.sep, "/")))]
|
|
files.append((work, os.path.join(work, "hot_funcs.txt")))
|
|
if chunks:
|
|
files.append((work, os.path.join(work, "image", "chunks.txt")))
|
|
for base, f in files:
|
|
if os.path.isfile(f):
|
|
h.update(os.path.relpath(f, base).replace(os.sep, "/").encode() + bytes(1))
|
|
with open(f, "rb") as src:
|
|
h.update(hashlib.sha1(src.read(), usedforsecurity=False).digest())
|
|
lift_env = dict(env, **{k: v for k, v in os.environ.items() if k.startswith(("A64_", "OOT_", "RX_"))})
|
|
h.update(repr(sorted(lift_env.items())).encode())
|
|
return h.hexdigest()
|
|
|
|
|
|
def key_after_lift(exefs, work, env, key, before, copied):
|
|
"""(Triforce, epicmemes46's 5df76bb) The key a finished lift is kept under. key: the one taken before it; before: that one
|
|
without the chunk boundaries (lift_key's chunks=False); copied: the boundaries were copied in from the source's
|
|
work/image/chunks.txt. The lift writes <work>/image/chunks.txt (nx2/chunks.py save): into a work folder that had none, the
|
|
boundaries the sizes gave. The key taken before it had none in it, the next build's has those, and so the build after every
|
|
lift into a fresh work folder translated the whole game again, the same game with the same tools. Boundaries the lift wrote
|
|
are what its own inputs give, so the key is taken again now, as the next build takes it, when nothing but chunks.txt changed
|
|
meanwhile (an update of the tools during the lift keeps the key from before it, and the next build lifts again). Boundaries
|
|
copied in from the source are an input, copied again at each build: the key from before the lift stays"""
|
|
if copied or lift_key(exefs, work, env, chunks=False) != before:
|
|
return key
|
|
return lift_key(exefs, work, env)
|
|
|
|
|
|
def gen_print(gen):
|
|
"""The lifted C's files as the lift left them (names, sizes, times): another builder's lift into the same work folder
|
|
(an older one, or another branch's, which knows nothing of the key) changes it, and the key no longer vouches for it"""
|
|
h = hashlib.sha1(usedforsecurity=False)
|
|
for name in sorted(os.listdir(gen)):
|
|
p = os.path.join(gen, name)
|
|
if name != LIFT_KEY and os.path.isfile(p):
|
|
st = os.stat(p)
|
|
h.update(("%s %d %d\n" % (name, st.st_size, st.st_mtime_ns)).encode())
|
|
return h.hexdigest()
|
|
|
|
|
|
def write_lift_key(work, key):
|
|
gen = os.path.join(work, "sa", "gen")
|
|
with open(os.path.join(gen, LIFT_KEY), "w") as f:
|
|
f.write(key + "\n" + gen_print(gen) + "\n")
|
|
|
|
|
|
def lifted_cut_short(gen):
|
|
"""The lifted C files (code_*.c) that do not end as every lifted file does, a function's closing brace and a newline: cut short
|
|
(a run stopped while writing one, a full disk, a second build in the same folder: issue 1197)"""
|
|
bad = []
|
|
if not os.path.isdir(gen):
|
|
return bad
|
|
for f in sorted(os.listdir(gen)):
|
|
if not (f.startswith("code_") and f.endswith(".c")):
|
|
continue
|
|
try:
|
|
with open(os.path.join(gen, f), "rb") as h:
|
|
h.seek(0, 2)
|
|
n = h.tell()
|
|
h.seek(max(0, n - 2))
|
|
end = h.read()
|
|
except OSError:
|
|
end = b""
|
|
if end != b"}\n":
|
|
bad.append(f)
|
|
return bad
|
|
|
|
|
|
def lifted_before(work, key):
|
|
"""The lift (and split_hot) in <work>/sa finished before with this key, its C is as that lift left it, and what the later
|
|
steps read of it is there"""
|
|
gen, image = os.path.join(work, "sa", "gen"), os.path.join(work, "sa", "image")
|
|
try:
|
|
with open(os.path.join(gen, LIFT_KEY)) as f:
|
|
lines = f.read().split()
|
|
except OSError:
|
|
return False
|
|
if len(lines) != 2 or lines[0] != key or lines[1] != gen_print(gen):
|
|
return False
|
|
return os.path.isfile(os.path.join(image, "image.bin")) and os.path.isfile(os.path.join(gen, "a64_table.c"))
|
|
|
|
|
|
def ngx_script():
|
|
"""scripts/ngx_sdk.py: NVIDIA's DLSS SDK, its file table and where each file goes"""
|
|
import importlib.util
|
|
path = os.path.join(ROOT, "scripts", "ngx_sdk.py")
|
|
if not os.path.isfile(path): # (the AppImage carries no scripts/: its Vulkan back end comes built, without DLSS)
|
|
fail("NVIDIA DLSS (HOOKSHOT_NGX_SDK) needs %s, which this builder does not carry: its renderer is built without DLSS. "
|
|
"Unset HOOKSHOT_NGX_SDK, or build Hookshot from source for DLSS." % path)
|
|
spec = importlib.util.spec_from_file_location("ngx_sdk", path)
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def ngx_platform():
|
|
return "linux" if LINUX else "windows"
|
|
|
|
|
|
def fetch_ngx_sdk(a):
|
|
"""--fetch-ngx-sdk (the Hookshot Builder's NVIDIA DLSS tick): NVIDIA's DLSS SDK into HOOKSHOT_NGX_SDK before the build, once
|
|
NVIDIA's license was accepted (--accept-nvidia-license: the builder asked first). Only what is missing or not the file
|
|
expected is downloaded"""
|
|
sdk = os.environ.get("HOOKSHOT_NGX_SDK", "")
|
|
if not sdk:
|
|
fail("--fetch-ngx-sdk needs HOOKSHOT_NGX_SDK, the folder NVIDIA's DLSS SDK goes into.")
|
|
if not a.accept_nvidia_license:
|
|
fail("NVIDIA DLSS needs NVIDIA's license accepted first (--accept-nvidia-license).")
|
|
ngx = ngx_script()
|
|
print(ngx.NOTICE, flush=True)
|
|
print("NVIDIA's license accepted: getting NVIDIA's DLSS SDK into %s" % sdk, flush=True)
|
|
try:
|
|
ngx.fetch(sdk, ngx_platform(), say=lambda line: print(line, flush=True))
|
|
except (OSError, ValueError) as e:
|
|
fail("NVIDIA's DLSS SDK could not be downloaded (%s). Check the connection and build again, or untick NVIDIA DLSS." % e)
|
|
|
|
|
|
def fsr_script():
|
|
"""scripts/fsr_sdk.py: AMD's two FSR 4 libraries; None where the builder carries no scripts/"""
|
|
import importlib.util
|
|
path = os.path.join(ROOT, "scripts", "fsr_sdk.py")
|
|
if not os.path.isfile(path):
|
|
return None
|
|
spec = importlib.util.spec_from_file_location("fsr_sdk", path)
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def fsr_runtime_dir(work):
|
|
"""Where AMD's FSR 4 libraries are kept between builds: HOOKSHOT_FSR_SDK, else the fsr-sdk folder of the work folder"""
|
|
return os.environ.get("HOOKSHOT_FSR_SDK", "") or os.path.join(work, "fsr-sdk")
|
|
|
|
|
|
def fetch_fsr_runtime(work):
|
|
"""(--fsr, Windows) AMD's two signed FSR 4 libraries into fsr_runtime_dir, only what is missing. A download that fails
|
|
is said and the build goes on: the game then does not offer AMD FSR 4 until a build gets them. (Triforce) And its
|
|
frame generation library with them (scripts/fsr_sdk.py OPTIONAL): without that one the game has FSR 4 as before and
|
|
the Frame generation row no AMD FSR 4"""
|
|
fsr = fsr_script()
|
|
if LINUX or fsr is None:
|
|
return
|
|
print(fsr.NOTICE, flush=True)
|
|
try:
|
|
fsr.fetch(fsr_runtime_dir(work), say=lambda line: print(line, flush=True))
|
|
except (OSError, ValueError) as e:
|
|
print("AMD's FSR 4 libraries could not be downloaded (%s): the game will not offer AMD FSR 4. Build again with a "
|
|
"connection to get them." % e, flush=True)
|
|
|
|
|
|
def place_fsr_runtime(out, work, wanted):
|
|
"""(AMD FSR 4) a game built with it gets AMD's two libraries and their license text beside its exe, where
|
|
gpu/vk/vk_fsr4.cpp looks, and (Triforce) AMD's frame generation library when it was fetched; a folder built with them
|
|
before and without FSR 4 now loses them, all three. Returns what was placed"""
|
|
fsr = fsr_script()
|
|
if LINUX or fsr is None:
|
|
return []
|
|
if not wanted or not fsr.complete(fsr_runtime_dir(work)):
|
|
if not wanted:
|
|
fsr.remove_runtime(out)
|
|
return []
|
|
placed = fsr.install_runtime(fsr_runtime_dir(work), out)
|
|
print("AMD FSR 4: %s (AMD's, MIT license)" % ", ".join(os.path.basename(p) for p in placed), flush=True)
|
|
return placed
|
|
|
|
|
|
def streamline_script():
|
|
"""scripts/streamline_sdk.py: NVIDIA's Streamline files for DLSS frame generation; None where the builder carries no
|
|
scripts/ (frame generation is then left out, DLSS itself is not)"""
|
|
import importlib.util
|
|
path = os.path.join(ROOT, "scripts", "streamline_sdk.py")
|
|
if not os.path.isfile(path):
|
|
return None
|
|
spec = importlib.util.spec_from_file_location("streamline_sdk", path)
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def fetch_streamline_sdk(a):
|
|
"""With --fetch-ngx-sdk on Windows (the NVIDIA DLSS tick, NVIDIA's licenses accepted): NVIDIA's Streamline files for DLSS
|
|
frame generation too, into HOOKSHOT_STREAMLINE_SDK (default: the streamline folder in HOOKSHOT_NGX_SDK), which
|
|
nx2/build.py then builds gpu/vk/vk_dlssg.cpp with. A download that fails leaves frame generation out of this build and
|
|
says so: the build goes on, with DLSS as before"""
|
|
if LINUX or not a.accept_nvidia_license:
|
|
return
|
|
sl = streamline_script()
|
|
if sl is None:
|
|
return
|
|
sdk = os.environ.get("HOOKSHOT_STREAMLINE_SDK", "") or os.path.join(os.environ.get("HOOKSHOT_NGX_SDK", ""), "streamline")
|
|
print(sl.NOTICE, flush=True)
|
|
try:
|
|
sl.fetch(sdk, say=lambda line: print(line, flush=True))
|
|
except (OSError, ValueError) as e:
|
|
print("NVIDIA's Streamline could not be downloaded (%s): this build has no DLSS frame generation. Build again "
|
|
"with a connection to get it." % e, flush=True)
|
|
os.environ.pop("HOOKSHOT_STREAMLINE_SDK", None)
|
|
return
|
|
os.environ["HOOKSHOT_STREAMLINE_SDK"] = sdk
|
|
|
|
|
|
def place_dlssg_runtime(out):
|
|
"""(NVIDIA DLSS frame generation) a game built with NVIDIA's Streamline (HOOKSHOT_STREAMLINE_SDK, as nx2/build.py takes it)
|
|
gets Streamline's DLLs, nvngx_dlssg.dll and their license texts in its dlssg folder, where gpu/vk/vk_dlssg.cpp looks. A
|
|
folder built with them before and without them now loses them. Returns the folder, or None
|
|
|
|
(Triforce, 11-18) DLSS's own runtime too, a copy of the one place_ngx_runtime put in ngx (so this runs after it): with
|
|
Streamline up, NGX looks for nvngx_dlss.dll only in the dlssg folder, and without it there DLSS upscaling did not start
|
|
("not available on this GPU (0xbad00004)"), so the menu offered no DLSS while frame generation was on"""
|
|
sl = streamline_script()
|
|
if sl is None:
|
|
return None
|
|
sdk = os.environ.get("HOOKSHOT_STREAMLINE_SDK", "")
|
|
dlss_copy = os.path.join(out, "dlssg", "nvngx_dlss.dll")
|
|
if LINUX or not sl.complete(sdk):
|
|
with contextlib.suppress(OSError): # (before Streamline's files: the folder is left empty and goes with them)
|
|
os.remove(dlss_copy)
|
|
sl.remove_runtime(out)
|
|
return None
|
|
dst = sl.install_runtime(sdk, out)
|
|
dlss = os.path.join(out, "ngx", "nvngx_dlss.dll")
|
|
if os.path.isfile(dlss):
|
|
copy_writable(dlss, dlss_copy)
|
|
else: # (a build without NVIDIA DLSS: no stale copy of an earlier one's)
|
|
with contextlib.suppress(OSError):
|
|
os.remove(dlss_copy)
|
|
print("NVIDIA DLSS frame generation: %s (NVIDIA's, under NVIDIA's licenses: never share this game folder)" % dst, flush=True)
|
|
return dst
|
|
|
|
|
|
def ngx_built():
|
|
"""This build has NVIDIA DLSS: HOOKSHOT_NGX_SDK holds this platform's library (what nx2/build.py takes)"""
|
|
sdk = os.environ.get("HOOKSHOT_NGX_SDK", "")
|
|
return bool(sdk) and os.path.isfile(ngx_script().library(sdk, ngx_platform()))
|
|
|
|
|
|
FSR_MARK = ".hookshot_fsr" # (in a game folder built with AMD FSR 4: --update builds it with FSR again)
|
|
|
|
|
|
def fsr_wanted(a, out):
|
|
"""AMD FSR 4 in this build (--fsr; or --update of a folder built with it, unless --no-fsr): nothing to download, AMD's
|
|
MIT headers are in third_party/fidelityfx and its DLLs are the player's (the readme)"""
|
|
if getattr(a, "no_fsr", False):
|
|
return False
|
|
return bool(getattr(a, "fsr", False)) or bool(a.update and os.path.isfile(os.path.join(out, FSR_MARK)))
|
|
|
|
|
|
RENODX_MARK = ".hookshot_renodx" # (in a game folder built with RenoDX's HDR: --update builds it with RenoDX again)
|
|
|
|
|
|
def renodx_wanted(a, out):
|
|
"""RenoDX's HDR in this build. (Triforce, 11-13: every build has it, not only one that asks with --renodx; --no-renodx
|
|
builds without. It acts only with the Insert menu's HDR set to Native.)"""
|
|
return not getattr(a, "no_renodx", False)
|
|
|
|
|
|
def prepare_renodx(work):
|
|
"""--renodx: RenoDX's shaders fetched and compiled before the build (scripts/renodx_shaders.py), into HOOKSHOT_RENODX when
|
|
set, else a renodx folder beside the work folder. Only what is missing or not the file expected is downloaded, and the
|
|
shaders are compiled again only when the script's pinned sources or compiler changed. Returns the folder"""
|
|
import importlib.util
|
|
path = os.path.join(ROOT, "scripts", "renodx_shaders.py")
|
|
if not os.path.isfile(path): # (the AppImage carries no scripts/: its Vulkan back end comes built, without RenoDX)
|
|
print("RenoDX's HDR: this builder does not carry %s: built without RenoDX" % path, flush=True)
|
|
return None
|
|
spec = importlib.util.spec_from_file_location("renodx_shaders", path)
|
|
renodx = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(renodx)
|
|
folder = os.environ.get("HOOKSHOT_RENODX") or os.path.join(os.path.dirname(work), "renodx")
|
|
print(renodx.NOTICE, flush=True)
|
|
try:
|
|
renodx.prepare(folder, "linux" if LINUX else "windows", say=lambda line: print(line, flush=True))
|
|
except (OSError, ValueError) as e:
|
|
# (Triforce, 11-13: RenoDX is in every build now, so a PC without a connection, or a download that is not the file
|
|
# expected, must not stop the game's build: this one is made without RenoDX and says so; the next build tries again)
|
|
print("RenoDX's shaders could not be fetched or compiled (%s): this build is made without RenoDX's HDR. The next "
|
|
"build tries again." % e, flush=True)
|
|
return None
|
|
return folder
|
|
|
|
|
|
def place_ngx_runtime(out):
|
|
"""(NVIDIA DLSS) a game built with NVIDIA's SDK (HOOKSHOT_NGX_SDK, as nx2/build.py takes it) gets DLSS's runtime in its ngx
|
|
folder, where gpu/vk/vk_dlss.cpp looks first: nvngx_dlss.dll on Windows, libnvidia-ngx-dlss.so.<version> on Linux. Never
|
|
NGX's core (_nvngx.dll is the driver's) and nothing beside the exe. Returns the file placed, or None"""
|
|
ngx = ngx_script()
|
|
platform = ngx_platform()
|
|
sdk = os.environ.get("HOOKSHOT_NGX_SDK", "")
|
|
if not sdk or not (os.path.isfile(ngx.library(sdk, platform)) and os.path.isfile(ngx.runtime(sdk, platform))):
|
|
# (a folder built with DLSS before and rebuilt without it loses NVIDIA's runtime too)
|
|
stale = os.path.join(out, "ngx", ngx.runtime_name(platform))
|
|
if os.path.isfile(stale):
|
|
with contextlib.suppress(OSError):
|
|
os.remove(stale)
|
|
os.rmdir(os.path.dirname(stale))
|
|
return None
|
|
dst = ngx.install_runtime(sdk, platform, os.path.join(out, "ngx"))
|
|
print("NVIDIA DLSS: %s (NVIDIA's, under NVIDIA's license: never share this game folder)" % dst, flush=True)
|
|
return dst
|
|
|
|
|
|
def parse_args():
|
|
p = argparse.ArgumentParser(description=__doc__.split("\n")[0])
|
|
p.add_argument("--nup")
|
|
p.add_argument("--romfs")
|
|
p.add_argument("--exefs")
|
|
p.add_argument("--out")
|
|
# (Triforce: its own work folder, not HookshotBuilder's: upstream Hookshot's builder works there too)
|
|
p.add_argument("--work", default=os.path.join(os.environ.get("XDG_CACHE_HOME") or os.path.expanduser("~/.cache"),
|
|
"triforce-build-windows" if CROSS else "triforce-build") # (the objects apart)
|
|
if os.name != "nt" else os.path.join(os.environ.get("LOCALAPPDATA", ROOT), "Triforce", "build"))
|
|
# (HOOKSHOT_JOBS: fewer compilers on a PC short of memory; the AppImage's builds pass no --jobs)
|
|
p.add_argument("--jobs", type=int)
|
|
p.add_argument("--language", default="en-US", help="the game's language: a code from LANGUAGES, written to language.txt")
|
|
p.add_argument("--check", action="store_true")
|
|
p.add_argument("--game", help="your copy: a .nup file, or a folder with romfs and exefs in it (the same as --nup, or --romfs and --exefs)")
|
|
p.add_argument("--prepare", action="store_true", help="for RecompLauncher: no window, progress lines, --out ~/Games/Hookshot unless given")
|
|
p.add_argument("--update", help="a game folder built before: built again from the game files it keeps (the game's update offer)")
|
|
p.add_argument("--no-prepare-shaders", action="store_true",
|
|
help="leave the shaders' preparation to the game's first start (the builder's \"Prepare shaders now\" unticked)")
|
|
p.add_argument("--log", metavar="FILE", help="also save build output to a UTF-8 file (replaces the file if it exists)")
|
|
# (NVIDIA DLSS, off unless asked: NVIDIA's SDK into HOOKSHOT_NGX_SDK first, under NVIDIA's license. scripts/ngx_sdk.py)
|
|
p.add_argument("--fetch-ngx-sdk", action="store_true", help="download NVIDIA's DLSS SDK into HOOKSHOT_NGX_SDK before the build")
|
|
p.add_argument("--accept-nvidia-license", action="store_true", help="NVIDIA's DLSS SDK license, accepted (with --fetch-ngx-sdk)")
|
|
# (AMD FSR 4, off unless asked: nothing is downloaded. gpu/vk/vk_fsr4.cpp)
|
|
p.add_argument("--fsr", action="store_true", help="build with AMD FSR 4; on Windows AMD's two libraries, and the one for "
|
|
"its frame generation, are downloaded from AMD's repository and put beside the game (scripts/fsr_sdk.py)")
|
|
p.add_argument("--no-fsr", action="store_true", help="with --update: build without AMD FSR 4 even if the folder had it")
|
|
# (RenoDX's HDR, in every build since 11-13: its shaders and the Slang compiler are downloaded and compiled here.
|
|
# scripts/renodx_shaders.py, gpu/vk/vk_renodx.cpp)
|
|
p.add_argument("--renodx", action="store_true",
|
|
help="kept for old command lines: every build has RenoDX's HDR now (its shaders and the Slang compiler are downloaded, about 60 MB)")
|
|
p.add_argument("--no-renodx", action="store_true", help="build without RenoDX's HDR: nothing of it is downloaded")
|
|
a = p.parse_args()
|
|
if a.prepare: # (first: a bad --game below is said as RecompLauncher's "error" line)
|
|
global PREPARE
|
|
PREPARE = True
|
|
a.out = a.out or os.path.join(os.path.expanduser("~"), "Games", "Hookshot")
|
|
if a.game: # (a path from a launcher: said plainly before any folder is made, not as a traceback halfway in)
|
|
if os.path.isdir(a.game):
|
|
a.romfs, a.exefs = os.path.join(a.game, "romfs"), os.path.join(a.game, "exefs")
|
|
if not (os.path.isdir(a.romfs) and os.path.isdir(a.exefs)):
|
|
fail("The folder %s needs both a romfs and an exefs folder in it." % a.game)
|
|
elif os.path.isfile(a.game):
|
|
a.nup = a.game
|
|
else:
|
|
fail("Your copy of the game was not found: %s" % a.game)
|
|
a.default_work = p.get_default("work") # (main: the builder's own work folder, stop_stray_compilers)
|
|
return a
|
|
|
|
|
|
def spinning_disk(folder):
|
|
"""True on a hard drive, False on an SSD, None when the drive cannot be told (nx2/disk.py)"""
|
|
return _disk.spinning(folder)
|
|
|
|
|
|
def warn_slow_disk(*folders):
|
|
"""A build reads and writes tens of gigabytes, and on a hard drive it takes much longer than on an SSD. Say so, and build
|
|
anyway: the player may have no other folder with the room. Said once, for the first folder on a hard drive."""
|
|
for folder in folders:
|
|
if spinning_disk(folder) is True:
|
|
# (on stderr under --prepare: RecompLauncher reads stdout's progress lines only)
|
|
print("Warning: %s is on a hard drive. An SSD builds the game much faster: build there if you can. "
|
|
"The build goes ahead on this drive." % folder, file=sys.stderr if PREPARE else sys.stdout, flush=True)
|
|
return
|
|
|
|
|
|
def main(a=None):
|
|
if a is None:
|
|
a = parse_args()
|
|
if a.jobs is not None and a.jobs < 0:
|
|
fail("--jobs must be 1 or more, or 0 for automatic.")
|
|
a.jobs = chosen_jobs(a.jobs)
|
|
os.makedirs(a.work, exist_ok=True)
|
|
no_throttle()
|
|
if os.name == "nt": # (the PC kept awake while this runs: a build is 45 minutes, and sleep stopped it)
|
|
import ctypes
|
|
ctypes.WinDLL("kernel32").SetThreadExecutionState(0x80000000 | 0x00000001)
|
|
if a.check:
|
|
return check(a, a.work)
|
|
if a.update:
|
|
a.out = a.update
|
|
a.language = folder_language(a.update) or a.language
|
|
if not a.out or not (a.update or a.nup or (a.romfs and a.exefs)):
|
|
fail("needs --out, and --nup or --romfs with --exefs")
|
|
out = os.path.abspath(a.out)
|
|
work = os.path.abspath(a.work)
|
|
warn_slow_disk(out, work) # (a warning, not a stop: a hard drive still builds, only much slower)
|
|
os.makedirs(out, exist_ok=True)
|
|
migrate_names(out)
|
|
# (linux) the compiler probe waits until the compile. The game-free cache test stops at the lift, and a missing
|
|
# compiler is still said before the long compile, not after the package has been read
|
|
check_cpu() # (no game read yet: the AppImage on a CPU without AVX2 stops here)
|
|
if not LINUX:
|
|
check_windows_sdk()
|
|
stop_stray_compilers(work, a.default_work)
|
|
drop_stale_objects(work)
|
|
if getattr(a, "fetch_ngx_sdk", False):
|
|
fetch_ngx_sdk(a)
|
|
fetch_streamline_sdk(a) # (DLSS frame generation's files, with the same tick)
|
|
# (AMD FSR 4: the build's switch for nx2/build.py, and the folder's mark for the next --update)
|
|
if fsr_wanted(a, out):
|
|
os.environ["HOOKSHOT_FSR"] = "1"
|
|
fetch_fsr_runtime(work) # (AMD's two libraries, for the game folder: place_fsr_runtime at the end)
|
|
open(os.path.join(out, FSR_MARK), "w").close()
|
|
if os.environ.get("HOOKSHOT_VK_PREBUILT"): # (the AppImage's back end, built without FSR 4)
|
|
print("--fsr: this build uses the prebuilt Vulkan back end (HOOKSHOT_VK_PREBUILT), which has no AMD FSR 4: the "
|
|
"Upscaler row will not offer it", flush=True)
|
|
else:
|
|
os.environ.pop("HOOKSHOT_FSR", None)
|
|
if os.path.isfile(os.path.join(out, FSR_MARK)):
|
|
os.remove(os.path.join(out, FSR_MARK))
|
|
# (RenoDX's HDR: its shaders made first, then the build's switch for nx2/build.py and the folder's mark for the next
|
|
# --update)
|
|
if renodx_wanted(a, out):
|
|
if os.environ.get("HOOKSHOT_VK_PREBUILT"): # (the AppImage's back end, built without RenoDX)
|
|
print("RenoDX's HDR: this build uses the prebuilt Vulkan back end (HOOKSHOT_VK_PREBUILT), which has no RenoDX: built "
|
|
"without it", flush=True)
|
|
os.environ.pop("HOOKSHOT_RENODX", None)
|
|
else:
|
|
renodx_folder = prepare_renodx(work)
|
|
if renodx_folder:
|
|
os.environ["HOOKSHOT_RENODX"] = renodx_folder
|
|
open(os.path.join(out, RENODX_MARK), "w").close()
|
|
else:
|
|
os.environ.pop("HOOKSHOT_RENODX", None)
|
|
if os.path.isfile(os.path.join(out, RENODX_MARK)):
|
|
os.remove(os.path.join(out, RENODX_MARK))
|
|
else:
|
|
os.environ.pop("HOOKSHOT_RENODX", None)
|
|
if os.path.isfile(os.path.join(out, RENODX_MARK)):
|
|
os.remove(os.path.join(out, RENODX_MARK))
|
|
exefs = os.path.join(work, "exefs")
|
|
romfs = os.path.join(out, "romfs")
|
|
|
|
say("stage", "read", "Reading your game files")
|
|
clear_folder(exefs)
|
|
kept = os.path.join(out, ".triforce_exefs") # (the game's code as read, kept in the folder: what an update builds from again)
|
|
if a.update:
|
|
if not os.path.isdir(kept) or not os.path.isdir(romfs):
|
|
fail("This game folder has no copy of the game files to update from: it was built before updates came in. Choose your game files "
|
|
"in Triforce Setup and build into the same folder once; updates after that work by themselves.")
|
|
check_cache_paths(out, (kept, romfs))
|
|
refuse_packed_romfs(romfs) # (a folder built from NX2navi's before the check: it never started)
|
|
shutil.copytree(kept, exefs)
|
|
check_modules(exefs)
|
|
stand_in_damaged_files(out, romfs) # (a folder unpacked by an earlier builder from a 23.3 GB dump)
|
|
elif a.nup:
|
|
# (what an earlier build left is not kept without a reason to trust it: builders before the compressed .nup fix wrote those files as
|
|
# they lay, so a folder they left held broken game data, and the game crashed at start over its empty dictionaries)
|
|
game = os.path.join(out, ".game")
|
|
good = os.path.join(out, ".triforce_romfs")
|
|
check_cache_paths(out, (game, good, kept, romfs))
|
|
# (an update of the builder over the same .nup: nothing to read again, only what the build needs. Issues 21 and 28)
|
|
try:
|
|
st = os.stat(a.nup)
|
|
except OSError as e:
|
|
fail("Could not access your .nup file %s: %s. Check that the file is still there, reconnect its drive if needed, "
|
|
"or choose the file again in Hookshot Builder." % (a.nup, e))
|
|
stamp = "unpacked by this builder: %s %d" % (os.path.basename(a.nup), st.st_size)
|
|
same = False
|
|
unpacked_seen = None
|
|
if os.path.exists(good) and os.path.isdir(romfs) and os.path.isdir(kept):
|
|
with open(good) as f:
|
|
same = f.read().strip() == stamp
|
|
if same:
|
|
print("the game was read from this .nup before: reading it again is skipped", flush=True)
|
|
shutil.copytree(kept, exefs)
|
|
else:
|
|
shutil.rmtree(game, ignore_errors=True)
|
|
with contextlib.suppress(FileNotFoundError): # (a new unpack is checked again)
|
|
os.remove(os.path.join(out, ".triforce_romfs_checked"))
|
|
# Keep the last usable cache until the new package has been read and validated.
|
|
staged_exefs = os.path.join(game, "exefs")
|
|
try:
|
|
run(PY + ["nx2/nup.py", a.nup, "--extract", game])
|
|
check_modules(staged_exefs)
|
|
check_supported_build(staged_exefs)
|
|
if not os.path.isdir(os.path.join(game, "romfs")):
|
|
fail("The package did not contain a romfs folder.")
|
|
except (OSError, SystemExit):
|
|
try:
|
|
shutil.rmtree(game)
|
|
except FileNotFoundError:
|
|
pass
|
|
except OSError as e:
|
|
print("Could not remove the incomplete extraction at %s (%s)." % (game, e), flush=True)
|
|
raise
|
|
if os.path.exists(romfs):
|
|
shutil.rmtree(romfs)
|
|
unpacked_seen = unpack_seen(game) # (the .zs files as the unpack saw them: stand_in_damaged_files below)
|
|
shutil.move(os.path.join(game, "romfs"), romfs)
|
|
shutil.move(staged_exefs, exefs)
|
|
shutil.rmtree(game, ignore_errors=True)
|
|
shutil.rmtree(kept, ignore_errors=True)
|
|
shutil.copytree(exefs, kept)
|
|
with open(good, "w") as f:
|
|
f.write(stamp + chr(10))
|
|
check_modules(exefs)
|
|
# (also when the .nup was read before: a folder built from a 23.3 GB dump by an earlier builder gets them too)
|
|
stand_in_damaged_files(out, romfs, unpacked_seen)
|
|
else:
|
|
check_cache_paths(out, (kept, romfs))
|
|
refuse_packed_romfs(a.romfs)
|
|
refuse_dump_links(a.exefs)
|
|
refuse_dump_links(a.romfs)
|
|
shutil.copytree(a.exefs, exefs) # (unpacked in place: the user's copy stays as it is)
|
|
run(PY + ["nx2/unpack.py", exefs])
|
|
link_or_copy(a.romfs, romfs)
|
|
shutil.rmtree(kept, ignore_errors=True)
|
|
shutil.copytree(exefs, kept)
|
|
check_supported_build(exefs)
|
|
# (Triforce) code mods: the .ips and .pchtxt patches of the game folder's mods, applied to this copy of the game's code, which
|
|
# the lift reads and whose files its key hashes: never to the player's files nor to the copy kept for updates (kept, above).
|
|
# A patch added, taken away, switched off or edited so changes the key and the game is translated again (nx2/code_mods.py)
|
|
code_patches = _code_mods.build_step(exefs, out, say=lambda line: print(line, flush=True))
|
|
|
|
# (made once on the development PC: lists of addresses and counts, no game code)
|
|
for name in ("hot_funcs.txt", "hot_order.txt", "hookshot_pieces.prof", "hookshot.res"):
|
|
src = os.path.join(ROOT, "work", name)
|
|
dst = os.path.join(work, name)
|
|
# (copied only when it differs: a compiler still ending from a cancelled build keeps the profile mapped, WinError 1224)
|
|
if os.path.exists(src) and not (os.path.exists(dst) and filecmp.cmp(src, dst, shallow=False)):
|
|
copy_writable(src, dst)
|
|
os.makedirs(os.path.join(work, "image"), exist_ok=True)
|
|
src = os.path.join(ROOT, "work", "image", "chunks.txt") # (where the lifted code's files begin: addresses)
|
|
chunks_copied = os.path.exists(src) # ((Triforce) an input then; otherwise the lift writes it: key_after_lift)
|
|
if chunks_copied:
|
|
copy_writable(src, os.path.join(work, "image", "chunks.txt"))
|
|
env = {"OOT_WORK": work, "OOT_EXEFS": exefs, "OOT_ROMFS": romfs, "OOT_DUMP": out, "RX_RELEASE": "1", "A64_HOT_CHUNK": "1"}
|
|
if os.path.exists(os.path.join(work, "hot_funcs.txt")):
|
|
env["A64_HOT_FUNCS"] = os.path.join(work, "hot_funcs.txt")
|
|
if os.path.exists(os.path.join(work, "hookshot_pieces.prof")) and profile_readable(os.path.join(work, "hookshot_pieces.prof")):
|
|
env["RX_PGO"] = os.path.join(work, "hookshot_pieces.prof")
|
|
|
|
prompts = Prompts(romfs, out, work) # (drawn beside the lift and the compile: finish() below)
|
|
atexit.register(prompts.stop) # (a build that stops for any reason takes its drawing with it)
|
|
|
|
say("stage", "lift", "Translating the game to PC code")
|
|
# (the same game, tools and settings as the lift before: its C is what this lift would write, and nothing recompiles from it. A
|
|
# rebuild after a change to host/ alone skips the minutes of the lift)
|
|
key = lift_key(exefs, work, env)
|
|
gen_dir = os.path.join(work, "sa", "gen")
|
|
# (issue 1197: a lifted file cut short, "g_module_b" where g_module_base was, stopped the compile at every try. Before the compile
|
|
# every one must end as a function does: one that does not is translated again, once; a second time says what to check)
|
|
for _ in range(2):
|
|
if lifted_before(work, key): # (never on the second pass: its key was taken away below)
|
|
print("the game was translated with these tools before: translating it again is skipped", flush=True)
|
|
else:
|
|
key_unchunked = lift_key(exefs, work, env, chunks=False) # ((Triforce) the inputs but chunks.txt: key_after_lift)
|
|
key_file = os.path.join(work, "sa", "gen", LIFT_KEY)
|
|
if os.path.exists(key_file):
|
|
os.remove(key_file) # (a lift stopped halfway leaves no key behind)
|
|
# (hookshot) the lifted C of an earlier lift goes first: a file that lift made and this one does not (another builder's naming,
|
|
# a chunk cut short) was compiled with the new ones, undefined or doubly defined game functions at the link (issue 1176).
|
|
# The objects stay cached by their contents, so a file this lift writes the same is not compiled again
|
|
gen_dir = os.path.join(work, "sa", "gen")
|
|
if os.path.isdir(gen_dir):
|
|
for f in os.listdir(gen_dir):
|
|
if f.startswith("code_") and (f.endswith(".c") or f.endswith(".c.whole") or f.endswith(".part")):
|
|
try:
|
|
os.remove(os.path.join(gen_dir, f))
|
|
except FileNotFoundError:
|
|
pass # (already gone: another cleanup removed it)
|
|
except OSError as e:
|
|
fail("Could not remove earlier translated file %s: %s. The new translation has not started." %
|
|
(os.path.join(gen_dir, f), e))
|
|
early = EarlyCompile(work, env, a.jobs) # (what needs no lifted code is compiled meanwhile)
|
|
atexit.register(early.stop)
|
|
run(PY + ["nx2/lift.py", "--standalone", "--write", "--exefs", exefs, "--work", work], env)
|
|
if os.path.isfile(os.path.join(work, "sa", "gen", "code_hot.c")):
|
|
run(PY + ["nx2/split_hot.py", os.path.join(work, "sa", "gen")], env) # (eight files side by side: ~6 minutes, not 79)
|
|
early.finish()
|
|
key = key_after_lift(exefs, work, env, key, key_unchunked, chunks_copied) # ((Triforce) the chunks.txt it wrote in it)
|
|
write_lift_key(work, key)
|
|
cut = lifted_cut_short(gen_dir)
|
|
if not cut:
|
|
break
|
|
key_file = os.path.join(gen_dir, LIFT_KEY)
|
|
if os.path.exists(key_file):
|
|
os.remove(key_file)
|
|
print("translated files cut short (%s): the game is translated again" % ", ".join(cut[:6]), flush=True)
|
|
else:
|
|
fail("The translated game came out cut short twice (%s). Check that the drive with %s has free space, and that no antivirus "
|
|
"and no second Triforce Setup is working in that folder, then update again." % (", ".join(cut[:6]), work))
|
|
|
|
if LINUX:
|
|
check_linux_compiler()
|
|
# (the compile's timer: build.py reads the record of the earlier builds on this PC, says which files this run really
|
|
# rebuilds and how long the rest will take, and reports each module's seconds; they go into the record below)
|
|
env["HOOKSHOT_TIMES"] = times_path(work)
|
|
run_files, plan = {}, [0, 0]
|
|
say("stage", "compile", "Compiling")
|
|
compile_started = time.time()
|
|
done = {"n": 0.0, "of": 1.0}
|
|
# (the progress by each file's share of the compile time, not by the count of files: the build takes the biggest first, and
|
|
# the 515 small files were 89% of the count and 10% of the time, so 45 of 579 files read as 8% when it was ~45%, and the
|
|
# builder said "452 minutes left" an hour from the end. Seconds per MB measured on a full build: the eight hot files (-O2,
|
|
# the play profile) 23.5, the other game files 3.4, the rest ~1.3 s each)
|
|
gen = os.path.join(work, "sa", "gen")
|
|
|
|
def cost(name, mb=None):
|
|
if mb is None:
|
|
try:
|
|
mb = os.path.getsize(os.path.join(gen, name)) / (1 << 20)
|
|
except OSError:
|
|
mb = 0
|
|
return 23.5 * mb if name.startswith("code_hot") else 3.4 * mb + 1 if name.startswith("code_") else 1.3
|
|
|
|
said = [] # (build.py's lines about how its compile and its links went: compile_alone_failed)
|
|
|
|
def compiled(line):
|
|
if line.startswith("compiled in ") or "link FAILED" in line:
|
|
said.append(line)
|
|
m = re.match(r"(\d+) files, ", line)
|
|
if m:
|
|
try:
|
|
ours = [f for f in os.listdir(gen) if f.startswith("code_") and f.endswith(".c")]
|
|
except OSError:
|
|
ours = []
|
|
done["of"] = sum(cost(f) for f in ours) + max(0, int(m.group(1)) - len(ours)) * 1.3
|
|
else:
|
|
m = re.match(r"\s+(\S+)\s+(\d+) MB\s+\d+s", line)
|
|
if m:
|
|
done["n"] += cost(m.group(1))
|
|
say("progress", "%.3f" % min(1.0, done["n"] / max(1.0, done["of"])))
|
|
if line.startswith("@@file "):
|
|
parts = line.split(" ")
|
|
if len(parts) == 4 and parts[3] in ("new", "again"):
|
|
try:
|
|
seconds = float(parts[2])
|
|
except ValueError:
|
|
return
|
|
if seconds > 0: # (a module a cache held is not a measurement of this PC)
|
|
run_files[parts[1]] = seconds
|
|
elif line.startswith("@@plan "):
|
|
parts = line.split(" ")
|
|
if len(parts) == 3 and parts[1].isdigit() and parts[2].isdigit():
|
|
plan[:] = [int(parts[1]), int(parts[2])]
|
|
try:
|
|
run(PY + ["nx2/build.py", "--standalone", "--jobs", str(a.jobs)], env, compiled)
|
|
except SystemExit:
|
|
# (hookshot) a compile or link that failed forgets the lift: the next build translates the game again. A lifted file cut short
|
|
# (an antivirus, a full disk, a stopped run: "g_module_b" for g_module_base, issue 1197) or lifted C another builder left (20
|
|
# undefined game functions at the link, issue 1176) otherwise failed the same way at every try, the key still matching
|
|
key_file = os.path.join(work, "sa", "gen", LIFT_KEY)
|
|
if os.path.exists(key_file):
|
|
os.remove(key_file)
|
|
# (and its compiled objects go too: a cached runtime object from another version linked with every object "compiled in 0s"
|
|
# and a64_dirty, a64_icall, a64_lookup, recomp_cntpct undefined, at every try (issue 1197, Linux, v2.0.10))
|
|
# (Triforce: but not when the compile alone failed, before any link: compile_alone_failed)
|
|
if compile_alone_failed(said):
|
|
print("the build failed in its compile: the files that did compile are kept, and the next build translates the game "
|
|
"again and compiles only the rest", flush=True)
|
|
else:
|
|
for sub in (os.path.join(work, "objcache"), os.path.join(work, "sa", "build", "obj")):
|
|
shutil.rmtree(sub, ignore_errors=True)
|
|
print("the build failed: the next one translates and compiles the game again (nothing of this one is reused)", flush=True)
|
|
prompts.stop()
|
|
raise
|
|
compile_seconds = time.time() - compile_started
|
|
|
|
say("stage", "finish", "Putting the game folder together")
|
|
built = os.path.join(work, "sa", "build")
|
|
# (the new files are copied in beside the old ones as <name>.new and swapped in only once all are complete: a build stopped while
|
|
# copying, or a full disk, left a half-written 460 MB Triforce.exe and no game to play. The old game stays playable until the swap)
|
|
# (Windows) Triforce.exe in the folder is the game's launcher: the builder's small setup, which opens the installed builder as a
|
|
# launcher for this folder (Play, and Update left of it when a newer release is out: builder/setup/setup.c launch mode), or starts
|
|
# the game when no builder is installed. The game and its DLLs are in bin\; its files (image.bin, romfs, saves, settings, caches)
|
|
# stay in the folder (host/hos_paths.h). Without the setup (a development build) the game stays Triforce.exe in the folder
|
|
staged = []
|
|
# (Linux) the same launcher, as a script, when this is a packaged builder (the AppImage sets HOOKSHOT_TOOLS). A source
|
|
# checkout leaves hookshot in the folder, as a development build on Windows does when the setup exe is not beside it
|
|
launcher = os.path.join(os.path.dirname(os.path.abspath(ROOT)), "HookshotBuilder-Setup.exe")
|
|
if LINUX:
|
|
script = os.path.join(ROOT, "builder", "linux", "hookshot-folder")
|
|
launcher = script if os.environ.get("HOOKSHOT_TOOLS") and os.path.isfile(script) else None
|
|
else:
|
|
launcher = launcher if os.path.isfile(launcher) else None
|
|
game_dir = os.path.join(out, "bin") if launcher else out
|
|
os.makedirs(game_dir, exist_ok=True)
|
|
ffmpeg_dlls = [] if LINUX else sorted(f for f in os.listdir(built) if f.endswith(".dll") and f.startswith(("av", "sw")))
|
|
for n in BESIDE + ffmpeg_dlls:
|
|
src = os.path.join(built, n)
|
|
if os.path.exists(src):
|
|
staged.append((src, os.path.join(game_dir, n)))
|
|
staged.append((os.path.join(work, "sa", "image", "image.bin"), os.path.join(out, "image.bin")))
|
|
if launcher:
|
|
staged.append((launcher, os.path.join(out, BESIDE[0])))
|
|
for src, dst in staged:
|
|
try:
|
|
if os.path.exists(dst + ".new"):
|
|
os.remove(dst + ".new")
|
|
shutil.copy2(src, dst + ".new")
|
|
except PermissionError:
|
|
fail("Could not stage the new build at %s. Check that %s is readable and the staged file and output folder are writable, "
|
|
"close any program using that file, then retry. No installed files were replaced; the compiled source is still available."
|
|
% (dst + ".new", src))
|
|
for src, dst in staged:
|
|
try:
|
|
os.replace(dst + ".new", dst) # (linux: a running copy keeps its own file; windows: a running exe cannot be replaced)
|
|
except PermissionError:
|
|
fail("Could not replace %s. Close any running copy of Triforce and check that the file and folder are writable, then retry. "
|
|
"Staged replacements remain beside the installed files; the compiled source remains at %s." % (dst, src))
|
|
if launcher: # (a folder built before the launcher: its libraries left the folder for bin\; hookshot is the launcher now)
|
|
stale = [] if LINUX else sorted(f for f in os.listdir(out) if f.endswith(".dll") and f.startswith(("av", "sw")))
|
|
for n in BESIDE[1:] + stale + ["Triforce.map", "hookshot.map", "hookshot-game.exe"]:
|
|
try:
|
|
if os.path.isfile(os.path.join(out, n)):
|
|
os.remove(os.path.join(out, n))
|
|
except OSError:
|
|
pass
|
|
drop_old_program(out)
|
|
place_ngx_runtime(out)
|
|
place_dlssg_runtime(out)
|
|
place_fsr_runtime(out, work, os.environ.get("HOOKSHOT_FSR") == "1")
|
|
# every shader of the game, for its first start to translate (gpu/vk/vk_shader.cpp seed_open). Without it the game still runs, compiling them
|
|
# as it meets them: not worth stopping a finished build for
|
|
if run(shader_census_cmd(romfs, os.path.join(out, "triforce_shaders.bin")), check=False, output=sys.stderr):
|
|
print("the shader list was not made: the game will compile its shaders as it plays", flush=True)
|
|
# (hookshot) the button prompts of other controllers (host/prompts.c): the game's icon font redrawn per controller, with fontTools.
|
|
# Without it the game shows its own icons. (Triforce: drawn since the game files were read, or kept: class Prompts)
|
|
prompts.finish()
|
|
icon = os.path.join(ROOT, "assets", "triforce.png")
|
|
if LINUX and os.path.exists(icon): # (the window's icon: host/posix/display_sdl.c)
|
|
copy_writable(icon, os.path.join(out, "triforce.png")) # (a second build overwrites the first's)
|
|
seed_pipelines(out)
|
|
ship_state_list(out)
|
|
ship_triforce_assets(out)
|
|
language = os.path.join(out, "language.txt") # (the same text as host/nn_applet.c writes)
|
|
with open(language, "w", encoding="utf-8", newline="\n") as f:
|
|
f.write("# The game's language. Change the line below to one of the codes listed here, save, and start the game again.\n\n%s\n\n" % a.language)
|
|
f.writelines("# %-8s %s\n" % l for l in LANGUAGES)
|
|
# (what the game's update offer reads, host/hos_update.c: which release these tools are, and where the builder that made the folder is)
|
|
tag = ""
|
|
for version_name in ("version.txt", "VERSION"): # (VERSION: the AppImage, which has no version.txt)
|
|
version = os.path.join(ROOT, version_name)
|
|
if os.path.isfile(version):
|
|
with open(version, encoding="utf-8") as f:
|
|
tag = f.readline().strip()
|
|
if tag:
|
|
break
|
|
if tag:
|
|
with open(os.path.join(out, "triforce_version.txt"), "w", encoding="utf-8", newline="\n") as f:
|
|
f.write("version=%s\n" % tag)
|
|
if os.environ.get("HOOKSHOT_BUILDER_EXE"):
|
|
f.write("builder=%s\n" % os.environ["HOOKSHOT_BUILDER_EXE"])
|
|
# (the Python builder: builder is its pythonw.exe, and these the arguments before --update, host/hos_update.c)
|
|
if os.environ.get("HOOKSHOT_BUILDER_ARGS"):
|
|
f.write("builder_args=%s\n" % os.environ["HOOKSHOT_BUILDER_ARGS"])
|
|
# (Triforce) what code patches this game is built with, for the next build and for the game's start (host/mods.c)
|
|
_code_mods.write_record(out, code_patches)
|
|
# (the game's code as read: written again by every build, so not left behind in the work folder. The translated code (sa\gen)
|
|
# and the compiled objects stay: the next build skips the lift when nothing it reads changed (#428) and compiles only what
|
|
# changed; the builder says how much stays and lets it go. Kept in a development checkout)
|
|
if os.path.normcase(work) != os.path.normcase(os.path.abspath(os.path.join(ROOT, "work"))):
|
|
shutil.rmtree(exefs, ignore_errors=True)
|
|
# (players asked for the space back: the translated code (1.5 GB) goes once the game is ready, on Windows and Linux alike. The
|
|
# compiled objects of this build stay (~1 GB; those of earlier builds go): without them every update compiled all ~600 files
|
|
# again, an hour and more on many PCs (issue 1287, "updates rebuild everything"), where the update changes a few. The next
|
|
# update translates again (minutes) and compiles only what changed. The builder's Free up space lets them go too.
|
|
# HOOKSHOT_KEEP_BUILD_FILES=1 keeps everything, as before)
|
|
if os.environ.get("HOOKSHOT_KEEP_BUILD_FILES", "0") in ("", "0"):
|
|
kept = prune_object_cache(work)
|
|
for sub in ("sa", "probe"):
|
|
shutil.rmtree(os.path.join(work, sub), ignore_errors=True)
|
|
print("the build files in %s were deleted but this build's compiled objects (%d MB), so that an update compiles only what "
|
|
"changed (HOOKSHOT_KEEP_BUILD_FILES=1 keeps all)" % (work, kept >> 20), flush=True)
|
|
total_seconds = time.time() - STARTED
|
|
previous = record_times(work, run_files, compile_seconds, total_seconds, modules=plan[1],
|
|
kind="update" if a.update else "build")
|
|
print("the build took %s, compiled in %s%s" % (clock(total_seconds), clock(compile_seconds),
|
|
"" if previous is None else " (the last one %s)" % clock(previous)), flush=True)
|
|
say("times", "%.0f" % total_seconds, "%.0f" % compile_seconds, "-1" if previous is None else "%.0f" % previous)
|
|
# (where the time went, for a report of a slow build and for measuring a change to the build itself)
|
|
print("the build's steps: %s" % stage_times(time.time()), flush=True)
|
|
# (after the times: they stay comparable from build to build, without a GPU's minutes that can be switched off)
|
|
skipped = prepare_skipped(a)
|
|
if skipped:
|
|
not_prepared(skipped)
|
|
else:
|
|
prepare_shaders([os.path.join(game_dir, BESIDE[0]), "--prepare-shaders"], out)
|
|
say("done", out)
|
|
|
|
|
|
def cli():
|
|
# (an error nobody foresaw is said in the window, not as "The game files could not be read" or "something went wrong" with the reason
|
|
# only in a console nobody sees: the traceback's last lines go to Show details, its last line to the message)
|
|
with contextlib.ExitStack() as stack:
|
|
try:
|
|
for stream in (sys.stdout, sys.stderr):
|
|
try:
|
|
stream.reconfigure(encoding="utf-8", errors="replace") # (-I ignores PYTHONIOENCODING: a path's letters crashed a print)
|
|
except (AttributeError, ValueError):
|
|
pass
|
|
a = parse_args()
|
|
stack.enter_context(build_log(a.log))
|
|
main(a)
|
|
except SystemExit as e:
|
|
# (fail() exits with a number: it already said @@error. A library's SystemExit (toolchain.need) carries its message as
|
|
# text, which would reach only stderr; the builder app does not show that, and said "The game files could not be read")
|
|
if isinstance(e.code, str):
|
|
say("error", e.code)
|
|
sys.exit(1)
|
|
raise
|
|
except BaseException as e:
|
|
import traceback
|
|
print(traceback.format_exc(), flush=True)
|
|
say("error", "The builder stopped on an error it did not expect: %s: %s. Please send the text under Show details to the "
|
|
"Triforce issues page (github.com/ruckus7382sfgsgdf/CBeta0.2/issues)." % (type(e).__name__, str(e)[:300]))
|
|
sys.exit(1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
cli()
|