Triforce v0.2.0

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
ruckus7382sfgsgdf
2026-11-10 19:32:48 -06:00
commit 34b3683441
1730 changed files with 554853 additions and 0 deletions

140
scripts/changelog.py Normal file
View File

@@ -0,0 +1,140 @@
"""Write the next entry of changelog.txt in the Factorio mod changelog format.
python3 scripts/changelog.py <version> [<last-tag>]
semantic-release calls this from its prepare step with the version being released and the last release's tag. The
commits after that tag are read from git and the entry is put at the top of changelog.txt. Only Conventional Commits
that make a release are listed, so the changelog says what the version rules say. HOOKSHOT_RELEASE_DATE=YYYY-MM-DD
fixes the date, for a release job that runs again.
A prerelease (`0.2.0-rc.1`) changes nothing: Factorio's parser accepts only `major.minor.patch` versions, and two
release candidates share one base version. The final release gets the entry, with every commit since the last stable
tag.
The file stays in the format Factorio's changelog parser expects: a 99-dash separator before each version, Version:
and Date: lines, section names indented two spaces, bullets four, no tabs, no trailing spaces, newest entry first. See
docs/versioning.md.
"""
import argparse
import datetime
import os
import re
import shutil
import subprocess
import sys
import tempfile
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
CHANGELOG = os.path.join(ROOT, "changelog.txt")
SEPARATOR = "-" * 99
# (the order sections appear in; only the ones with commits are written)
SECTIONS = ("Major Features", "Minor Features", "Features", "Changes", "Bugfixes", "Optimizations",
"Graphics", "Sounds", "Balancing", "Scripting", "Modding", "Locale", "Info")
# (what a released commit becomes; docs, style, test, build, ci and chore are not listed)
SECTIONS_BY_TYPE = {"feat": "Features", "fix": "Bugfixes", "perf": "Optimizations", "revert": "Bugfixes",
"refactor": "Changes"}
HEADER = re.compile(r"^(?P<type>[A-Za-z][A-Za-z0-9-]*)(?:\((?P<scope>[^)\r\n]*)\))?(?P<breaking>!)?: (?P<text>.+)$")
BREAKING = re.compile(r"^BREAKING[ -]CHANGES?:", re.MULTILINE)
VERSION = re.compile(r"^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$")
def commits(since=None):
"""(subject, body) for every commit after `since`; the order does not matter, the entry is grouped by section."""
rev = "%s..HEAD" % since if since else "HEAD"
r = subprocess.run(["git", "-C", ROOT, "log", "--no-merges", "--format=%H%x1f%s%x1f%b%x1e", rev],
capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=120)
if r.returncode:
raise SystemExit("git log %s failed: %s" % (rev, r.stderr.strip()))
out = []
for record in r.stdout.split("\x1e"):
fields = record.strip("\n").split("\x1f", 2)
if len(fields) == 3:
out.append((fields[1].strip(), fields[2]))
return out
def bullet(scope, text):
"""One Factorio bullet: a capital first letter and a full stop, as the format uses."""
text = " ".join(text.split())
if not text:
return ""
text = text[:1].upper() + text[1:]
if text[-1] not in ".!?":
text += "."
if scope:
scope = scope[:1].upper() + scope[1:]
return "%s: %s" % (scope, text)
return text
def date_line(value=None):
value = value or os.environ.get("HOOKSHOT_RELEASE_DATE", "").strip()
day = datetime.date.fromisoformat(value) if value else datetime.date.today()
return "%02d. %02d. %04d" % (day.day, day.month, day.year)
def entry(version, since=None, date=None):
found = {section: [] for section in SECTIONS}
for subject, body in commits(since):
m = HEADER.match(subject)
if not m:
continue
text = bullet((m.group("scope") or "").strip(), m.group("text"))
if not text:
continue
if m.group("breaking") or BREAKING.search(body):
found["Changes"].append("Breaking: " + text)
elif m.group("type").lower() in SECTIONS_BY_TYPE:
found[SECTIONS_BY_TYPE[m.group("type").lower()]].append(text)
lines = [SEPARATOR, "Version: %s" % version, "Date: %s" % date_line(date)]
for section in SECTIONS:
# (Factorio rejects an exact duplicate in one section, so a repeated subject is listed once)
entries = list(dict.fromkeys(found[section]))
if entries:
lines.append(" %s:" % section)
lines.extend(" - %s" % one for one in entries)
if len(lines) == 3: # (no commit followed the convention: say the least that is true)
lines.extend((" Changes:", " - Release %s." % version))
return "\n".join(lines) + "\n"
def write(version, since=None, date=None):
if "-" in version:
print("changelog: %s is a prerelease; changelog.txt is not changed" % version)
return
try:
with open(CHANGELOG, encoding="utf-8") as f:
old = f.read()
except FileNotFoundError:
old = ""
if "Version: %s" % version in old.splitlines(): # (a retried job must not add the version twice)
print("changelog: %s is already in changelog.txt" % version)
return
content = entry(version, since, date) + old.lstrip("\n")
temporary = None
try:
with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", newline="\n", delete=False,
dir=os.path.dirname(CHANGELOG), prefix=".changelog-", suffix=".tmp") as f:
temporary = f.name
f.write(content)
if os.path.exists(CHANGELOG):
shutil.copymode(CHANGELOG, temporary)
os.replace(temporary, CHANGELOG)
finally:
if temporary and os.path.exists(temporary):
os.unlink(temporary)
def main():
ap = argparse.ArgumentParser(description="Add the next version's entry to changelog.txt.")
ap.add_argument("version", help="the version being released, without the v (0.2.0)")
ap.add_argument("since", nargs="?", default="", help="the last release's tag (v0.1.0); every commit if omitted")
ap.add_argument("--date", help="YYYY-MM-DD; default today, or HOOKSHOT_RELEASE_DATE")
a = ap.parse_args()
if not VERSION.match(a.version):
ap.error("not a semantic version: %s" % a.version)
write(a.version, a.since or None, a.date)
if __name__ == "__main__":
main()

115
scripts/crit.py Normal file
View File

@@ -0,0 +1,115 @@
"""The frame's critical path from an all-thread sample (host/mkw_glue.c HOS_SAMPLE_ALL): every 2 ms each guest thread
and renderer thread, "<ms> <name>|<frames>" (module+offset, leaf first).
python -I scripts/crit.py work/runs/<run>_all.txt [top]
A thread is running at an instant when its leaf is in hookshot.exe or hookshot_vk.dll (on Linux hookshot or
hookshot_vk.so: game code, the runtime, the renderer), blocked otherwise (the leaf in ntdll, win32u, KERNELBASE, libc,
the driver). RX_MAPS=<folder>: another build's linker maps, or on Linux the game folder (its binaries' symbols). Prints each thread's running share;
then for the instants when no game thread runs (the frame waits on something outside the game's CPU work): what each
thread waits in and what the renderer's threads do; then for the instants when the main thread is blocked: how many
game threads run and the functions they are in.
"""
import collections, os, sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import prof # noqa: E402
RUN_MODS = prof.RUN_MODS
def main():
path = sys.argv[1]
top = int(sys.argv[2]) if len(sys.argv) > 2 else 25
maps = prof.load_symbols()
cache = {}
def name(frame):
if frame in cache:
return cache[frame]
mod, _, off = frame.rpartition("+")
r = mod
if mod in maps:
addrs, names = maps[mod]
import bisect
i = bisect.bisect_right(addrs, int(off, 16)) - 1
if i >= 0:
r = prof.module_name(mod) + ":" + prof.demangle(names[i])
cache[frame] = r
return r
ticks = collections.defaultdict(dict) # ms -> thread -> (running, frames named)
for line in open(path, encoding="utf-8", errors="replace"):
head, _, stack = line.rstrip("\n").partition("|")
ms, _, thread = head.partition(" ")
if not stack or not ms.isdigit():
continue
frames = stack.split(";")
leafmod = frames[0].rpartition("+")[0]
ticks[int(ms)][thread] = (leafmod in RUN_MODS, [name(f) for f in frames])
n = len(ticks)
print("%d instants" % n)
running = collections.Counter()
seen = collections.Counter()
for t in ticks.values():
for th, (r, _) in t.items():
seen[th] += 1
running[th] += r
print("-- running share by thread")
for th, c in sorted(running.items(), key=lambda kv: -kv[1])[:30]:
print(" %5.1f%% %s" % (100.0 * c / seen[th], th))
def is_game(th):
return not th.startswith("mkw_vk")
def guest_frame(frames):
"""the innermost game function (f_...) and the import it called, if any"""
imp = next((f for f in frames if ":imp_" in f), None)
fn = next((f for f in frames if ":f_" in f), None)
return (imp.split(":imp_")[1][:48] + " <- " if imp else "") + (fn or frames[-1])
idle = [t for t in ticks.values() if not any(r for th, (r, _) in t.items() if is_game(th))]
print("-- no game thread running: %.1f%% of instants" % (100.0 * len(idle) / max(1, n)))
waits = collections.Counter()
vk = collections.Counter()
for t in idle:
for th, (r, frames) in t.items():
if th in ("sead::MainThread", "Presentation_Thread") or th.startswith("BaseProcExecutor0"):
waits["%s: %s" % (th, guest_frame(frames))] += 1
if not is_game(th):
vk["%s %s: %s" % (th, "runs" if r else "waits", frames[0] if not r else
next((f for f in frames if f.startswith("hookshot_vk:")), frames[0]))] += 1
for k, c in waits.most_common(top):
print(" %5.1f%% %s" % (100.0 * c / max(1, len(idle)), k))
print(" (renderer threads then)")
for k, c in vk.most_common(top):
print(" %5.1f%% %s" % (100.0 * c / max(1, len(idle)), k))
mw = [t for t in ticks.values() if "sead::MainThread" in t and not t["sead::MainThread"][0]]
print("-- main thread blocked: %.1f%% of instants" % (100.0 * len(mw) / max(1, n)))
hist = collections.Counter(sum(1 for th, (r, _) in t.items() if r and is_game(th)) for t in mw)
print(" game threads running then: " + ", ".join("%d: %.0f%%" % (k, 100.0 * v / len(mw)) for k, v in sorted(hist.items())))
mwhere = collections.Counter(guest_frame(t["sead::MainThread"][1]) for t in mw)
print(" main waits in:")
for k, c in mwhere.most_common(8):
print(" %5.1f%% %s" % (100.0 * c / len(mw), k))
fns = collections.Counter()
for t in mw:
for th, (r, frames) in t.items():
if r and is_game(th):
fns[next((f for f in frames if ":f_" in f), frames[0])] += 1
print(" game functions running then (innermost f_, share of those instants):")
for k, c in fns.most_common(top):
print(" %5.1f%% %s" % (100.0 * c / len(mw), k))
leaves = collections.Counter()
for t in mw:
for th, (r, frames) in t.items():
if r and is_game(th):
leaves[frames[0]] += 1
print(" leaves running then:")
for k, c in leaves.most_common(top):
print(" %5.1f%% %s" % (100.0 * c / len(mw), k))
if __name__ == "__main__":
main()

136
scripts/framelog.py Normal file
View File

@@ -0,0 +1,136 @@
"""The flashes in a frame log (RX_VK_FRAMELOG, gpu/vk/vk_replay.cpp): capture samples, 320x180.
Timestamps sample command recording before GPU submission.
python -I scripts/framelog.py <file> [out.png]
python -I scripts/framelog.py <file> --inspect
A frame is a flash where a part of the picture goes brighter (or darker) than BOTH its neighbours by more than they differ
from each other, with the brightness of that part back within reach the next frame. Blocks of 20x20, so that an edge moving
past counts little; a block must jump in brightness, not only in detail. Prints the events by size, with whether the frame was
the game's or a made one, and writes the worst as rows of (before, frame, after).
Only consecutive sequence numbers form a triplet: dropped records cannot establish a one-frame flash. Inspect a
completed, stable capture; --inspect validates its headers without NumPy. Sequence numbers may wrap at uint32;
nonadvancing headers, partial records and captures shorter than three frames are rejected.
"""
import argparse
import mmap
from pathlib import Path
import struct
from typing import NamedTuple
W, H = 320, 180
SIZE = 16 + W * H * 4
MASK = (1 << 32) - 1
class CaptureInfo(NamedTuple):
timestamps: list[int]
sequences: list[int]
kinds: list[int]
missing: int
centres: list[int]
def triplet_centres(sequences):
"""Only stored consecutive frames can establish a one-frame brightness jump."""
return [i for i in range(1, len(sequences) - 1)
if ((sequences[i] - sequences[i - 1]) & MASK) == 1
and ((sequences[i + 1] - sequences[i]) & MASK) == 1]
def capture_info(path):
size = Path(path).stat().st_size
count, tail = divmod(size, SIZE)
if tail:
raise ValueError("truncated record %d: %d bytes, expected %d" % (count + 1, tail, SIZE))
if count < 3:
raise ValueError("need at least 3 frames for flash analysis; got %d" % count)
timestamps, sequences, kinds = [], [], []
missing = 0
with open(path, "rb") as file, mmap.mmap(file.fileno(), 0, access=mmap.ACCESS_READ) as raw:
for i in range(count):
t, s, k = struct.unpack_from("<QII", raw, i * SIZE)
if k not in (0, 1):
raise ValueError("invalid frame kind %d at record %d (expected 0 or 1)" % (k, i + 1))
if timestamps and t <= timestamps[-1]:
raise ValueError("timestamp does not advance at record %d" % (i + 1))
if sequences:
delta = (s - sequences[-1]) & MASK
if not delta or delta >= (1 << 31):
raise ValueError("sequence does not advance at record %d" % (i + 1))
missing += delta - 1
timestamps.append(t), sequences.append(s), kinds.append(k)
return CaptureInfo(timestamps, sequences, kinds, missing, triplet_centres(sequences))
def blocks(a):
"""luma means of 20x20 blocks"""
l = a[..., 0] * 0.3 + a[..., 1] * 0.6 + a[..., 2] * 0.1
return l[:H // 20 * 20, :].reshape(H // 20, 20, W // 20, 20).mean(axis=(1, 3))
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("path", type=Path)
parser.add_argument("out", nargs="?", help="write the worst events to this PNG")
parser.add_argument("--inspect", action="store_true", help="validate metadata and report eligible triplets without NumPy")
args = parser.parse_args()
path, out = args.path, args.out
try:
info = capture_info(path)
except (OSError, ValueError, struct.error) as error:
parser.error(str(error))
n = len(info.sequences)
print(n, "frames")
eligible = set(info.centres)
print("flash triplets: %d eligible, %d skipped across sequence gaps; missing %d frames" % (
len(eligible), n - 2 - len(eligible), info.missing))
if args.inspect:
return 0
try:
import numpy as np
except ImportError:
parser.error("image analysis requires NumPy; --inspect validates capture metadata without it")
raw = np.memmap(path, dtype=np.uint8, mode="r")
ts = np.array(info.timestamps, dtype=np.uint64)
kinds = np.array(info.kinds)
d = np.diff(ts) / 1e6
print("capture samples apart: median %.1f ms, 99%% %.1f, worst %.1f; made %d of %d; missing %d" % (
np.median(d), np.percentile(d, 99), d.max(), kinds.sum(), n, info.missing))
frame = lambda i: np.asarray(raw[i * SIZE + 16:(i + 1) * SIZE]).reshape(H, W, 4)[..., :3].astype(np.float32)
prev = blocks(frame(0))
cur = blocks(frame(1))
events = []
for i in range(1, n - 1):
nxt = blocks(frame(i + 1))
if i in eligible:
up = np.minimum(cur - prev, cur - nxt)
down = np.minimum(prev - cur, nxt - cur)
span = np.abs(prev - nxt)
for sign, jump in ((1, up), (-1, down)):
hit = (jump > 12) & (jump > span * 1.5)
c = int(hit.sum())
if c:
events.append((float(jump[hit].sum()), i, sign, c, float(jump.max())))
prev, cur = cur, nxt
events.sort(reverse=True)
tot = {0: 0, 1: 0}
for e in events:
tot[int(kinds[e[1]])] += 1
print("events: %d on the game's frames, %d on made frames" % (tot[0], tot[1]))
for size, i, sign, c, peak in events[:24]:
print("frame %d (%s) %s %d blocks, peak %.0f, at %.2f s" % (i, "made" if kinds[i] else "game", "brighter" if sign > 0 else "darker",
c, peak, (ts[i] - ts[0]) / 1e9))
if out and events:
from PIL import Image
rows = events[:8]
im = Image.new("RGB", (W * 3, H * len(rows)))
for r, e in enumerate(rows):
for k in range(3):
im.paste(Image.fromarray(frame(e[1] - 1 + k).astype(np.uint8)), (k * W, r * H))
im.save(out)
if __name__ == "__main__":
raise SystemExit(main())

202
scripts/fsr3_shaders.py Normal file
View File

@@ -0,0 +1,202 @@
"""Compile AMD's FSR 3.1 upscaler passes into the SPIR-V headers gpu/vk uses.
The FidelityFX SDK ships each pass as GLSL that includes the SDK's own GPU headers, so the eleven
variants below are the same eight passes compiled with different options. They are checked in like
every other shader in gpu/vk/shaders, so a clone builds without glslang; this script regenerates
them, and --check proves the checked-in files still match their sources (tests/test_fsr3_shaders.py).
python -I -B scripts/fsr3_shaders.py # write gpu/vk/shaders/fsr3_*_comp_spv.h
python -I -B scripts/fsr3_shaders.py --check # compare instead of writing
"""
import argparse
import difflib
import shutil
import subprocess
import sys
import tempfile
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
SDK = ROOT / "third_party/fidelityfx-sdk"
GPU = SDK / "include/FidelityFX/gpu"
PASSES = SDK / "shaders/fsr3upscaler"
FI_PASSES = SDK / "shaders/frameinterpolation"
OF_PASSES = SDK / "shaders/opticalflow"
OUT = ROOT / "gpu/vk/shaders"
# The option set the renderer records these passes under. Each one is a promise about the inputs:
# the colour is HDR, the motion is low-resolution and not jittered, and the depth is reversed. They
# have to match what gpu/vk/vk_fsr3.cpp hands the passes, which is why they live in one place.
OPTIONS = [
"-DFFX_GPU=1",
"-DFFX_GLSL=1",
"-DFFX_FSR3UPSCALER_OPTION_UPSAMPLE_USE_LANCZOS_TYPE=2",
"-DFFX_FSR3UPSCALER_OPTION_REPROJECT_USE_LANCZOS_TYPE=0",
"-DFFX_FSR3UPSCALER_OPTION_HDR_COLOR_INPUT=1",
"-DFFX_FSR3UPSCALER_OPTION_LOW_RESOLUTION_MOTION_VECTORS=1",
"-DFFX_FSR3UPSCALER_OPTION_JITTERED_MOTION_VECTORS=0",
"-DFFX_FSR3UPSCALER_OPTION_INVERTED_DEPTH=1",
]
# name, the pass GLSL it compiles, extra defines. The _nowave variants are for devices without the
# subgroup operations the single-pass downsampler prefers.
VARIANTS = [
("prepare_inputs", "prepare_inputs", []),
("luma_pyramid", "luma_pyramid", []),
("luma_pyramid_nowave", "luma_pyramid", ["-DFFX_SPD_NO_WAVE_OPERATIONS=1"]),
("shading_change_pyramid", "shading_change_pyramid", []),
("shading_change_pyramid_nowave", "shading_change_pyramid", ["-DFFX_SPD_NO_WAVE_OPERATIONS=1"]),
("shading_change", "shading_change", []),
("prepare_reactivity", "prepare_reactivity", []),
("luma_instability", "luma_instability", []),
("accumulate", "accumulate", []),
("accumulate_sharpen", "accumulate", ["-DFFX_FSR3UPSCALER_OPTION_APPLY_SHARPENING=1"]),
("rcas", "rcas", []),
]
# (hookshot) Frame interpolation and the optical flow it runs on, the same arrangement: AMD ships
# each pass as GLSL over the SDK's own GPU headers, and the renderer records them as SPIR-V headers
# beside the upscaler's. The options are the same promises about the same inputs -- the depth is
# reversed and the motion is at the render extent -- because the inputs are the ones the upscaler is
# already given (gpu/vk/vk_fsr3fi.cpp).
FI_OPTIONS = [
"-DFFX_GPU=1",
"-DFFX_GLSL=1",
"-DFFX_FRAMEINTERPOLATION_OPTION_INVERTED_DEPTH=1",
"-DFFX_FRAMEINTERPOLATION_OPTION_LOW_RESOLUTION_MOTION_VECTORS=1",
"-DFFX_FRAMEINTERPOLATION_OPTION_JITTERED_MOTION_VECTORS=0",
]
OF_OPTIONS = ["-DFFX_GPU=1", "-DFFX_GLSL=1"]
# (name, the pass file, extra defines). The file is named outright rather than composed: AMD's
# optical-flow v5 passes end "_pass_v5.glsl" and the interpolation pass itself is just
# "ffx_frameinterpolation_pass.glsl", so a composed name is wrong for four of the eighteen.
FI_VARIANTS = [
("fi_setup", "ffx_frameinterpolation_setup_pass.glsl", []),
("fi_reconstruct_previous_depth", "ffx_frameinterpolation_reconstruct_previous_depth_pass.glsl", []),
("fi_reconstruct_and_dilate", "ffx_frameinterpolation_reconstruct_and_dilate_pass.glsl", []),
("fi_game_motion_vector_field", "ffx_frameinterpolation_game_motion_vector_field_pass.glsl", []),
("fi_optical_flow_vector_field", "ffx_frameinterpolation_optical_flow_vector_field_pass.glsl", []),
("fi_disocclusion_mask", "ffx_frameinterpolation_disocclusion_mask_pass.glsl", []),
("fi_game_vector_field_inpainting_pyramid",
"ffx_frameinterpolation_compute_game_vector_field_inpainting_pyramid_pass.glsl", []),
("fi_inpainting_pyramid", "ffx_frameinterpolation_compute_inpainting_pyramid_pass.glsl", []),
("fi_interpolate", "ffx_frameinterpolation_pass.glsl", []),
("fi_inpainting", "ffx_frameinterpolation_inpainting_pass.glsl", []),
]
OF_VARIANTS = [
("of_prepare_luma", "ffx_opticalflow_prepare_luma_pass.glsl", []),
("of_luminance_pyramid", "ffx_opticalflow_compute_luminance_pyramid_pass.glsl", []),
("of_generate_scd_histogram", "ffx_opticalflow_generate_scd_histogram_pass.glsl", []),
("of_compute_scd_divergence", "ffx_opticalflow_compute_scd_divergence_pass.glsl", []),
("of_compute_optical_flow_v5", "ffx_opticalflow_compute_optical_flow_advanced_pass_v5.glsl", []),
("of_filter_optical_flow_v5", "ffx_opticalflow_filter_optical_flow_pass_v5.glsl", []),
("of_scale_optical_flow_v5", "ffx_opticalflow_scale_optical_flow_advanced_pass_v5.glsl", []),
]
def compile_family(glslang, name, source, extra, out_path, target_env, options, includes,
variable=True):
"""One frame-interpolation or optical-flow pass. The upscaler's compile_one is kept as it is:
its passes are composed by name and its option set is the upscaler's own."""
cmd = [glslang, "-V", "--quiet", "-S", "comp"]
for d in includes:
cmd.append("-I" + str(d))
cmd += [*options, *extra]
if variable:
cmd += ["--variable-name", f"FSR3_{name.upper()}_COMP_SPV"]
cmd += ["-o", str(out_path), str(source), "--target-env", target_env]
subprocess.run(cmd, check=True)
def compile_one(glslang, name, pass_name, extra, out_path, target_env, variable=True):
"""Compile one variant. With `variable`, glslang writes a C header; without it, binary SPIR-V,
which is what spirv-val reads."""
defines = list(OPTIONS)
if not any("APPLY_SHARPENING" in d for d in extra):
defines.append("-DFFX_FSR3UPSCALER_OPTION_APPLY_SHARPENING=0")
cmd = [glslang, "-V", "--quiet", "-S", "comp",
"-I" + str(GPU), "-I" + str(GPU / "fsr3upscaler"),
*defines, *extra]
if variable:
cmd += ["--variable-name", f"FSR3_{name.upper()}_COMP_SPV"]
cmd += ["-o", str(out_path),
str(PASSES / f"ffx_fsr3upscaler_{pass_name}_pass.glsl"),
"--target-env", target_env]
subprocess.run(cmd, check=True)
def strip_generator(text):
"""The header without glslang's own version comment.
glslang writes its version into the first line as `// 1116.2.0`, so two glslang builds produce
byte-different headers from identical SPIR-V. Comparing the whole text made this check fail on
a CI runner whose glslang was one release behind the one that generated the checked-in files --
a version bump reported as a source mismatch, which is the kind of red that teaches people to
ignore red. The words after it are the module, and those are what this is about.
"""
return [line for line in text.splitlines()
if not line.strip().startswith("// ") or not line.strip()[3:].replace(".", "").isdigit()]
def main():
p = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
p.add_argument("--glslang", default=None, help="glslangValidator; found on PATH by default")
p.add_argument("--spirv-val", default=None, help="spirv-val, to validate each module")
p.add_argument("--target-env", default="spirv1.3")
p.add_argument("--check", action="store_true", help="compare with the checked-in headers")
a = p.parse_args()
glslang = a.glslang or shutil.which("glslangValidator") or shutil.which("glslang")
if not glslang:
print("glslangValidator not found; pass --glslang", file=sys.stderr)
return 2
bad = []
with tempfile.TemporaryDirectory(prefix="fsr3-shaders-") as tmp:
# the upscaler's eleven, then frame interpolation's ten and optical flow's seven
families = [(name, None, pass_name, extra, None, None) for name, pass_name, extra in VARIANTS]
families += [(name, FI_PASSES / f, None, extra, FI_OPTIONS,
[GPU, GPU / "frameinterpolation", GPU / "opticalflow"])
for name, f, extra in FI_VARIANTS]
families += [(name, OF_PASSES / f, None, extra, OF_OPTIONS, [GPU, GPU / "opticalflow"])
for name, f, extra in OF_VARIANTS]
for name, source, pass_name, extra, options, includes in families:
header = OUT / f"fsr3_{name}_comp_spv.h"
made = Path(tmp) / header.name
if source is None:
compile_one(glslang, name, pass_name, extra, made, a.target_env)
else:
compile_family(glslang, name, source, extra, made, a.target_env, options, includes)
if a.spirv_val:
spv = Path(tmp) / f"{name}.spv"
if source is None:
compile_one(glslang, name, pass_name, extra, spv, a.target_env, variable=False)
else:
compile_family(glslang, name, source, extra, spv, a.target_env, options,
includes, variable=False)
subprocess.run([a.spirv_val, str(spv)], check=True)
if a.check:
have = header.read_text() if header.exists() else ""
want = made.read_text()
if strip_generator(have) != strip_generator(want):
bad.append(name)
if not have:
print(f"{header.name}: missing", file=sys.stderr)
else:
diff = difflib.unified_diff(have.splitlines(), want.splitlines(),
"checked in", "regenerated", n=0, lineterm="")
print("\n".join(list(diff)[:12]), file=sys.stderr)
else:
shutil.copyfile(made, header)
print(f"wrote {header.relative_to(ROOT)} ({header.stat().st_size} bytes)")
if bad:
print(f"{len(bad)} header(s) do not match their source: {', '.join(bad)}", file=sys.stderr)
return 1
print(f"fsr3 shaders: {len(VARIANTS) + len(FI_VARIANTS) + len(OF_VARIANTS)} {'match' if a.check else 'written'}")
return 0
if __name__ == "__main__":
sys.exit(main())

View File

@@ -0,0 +1,82 @@
"""Who reports problems: the GPU vendor, card class and OS named in the issue forms (docs/workflow.md, "Reach").
python -I scripts/hardware_report.py [--limit 1000] [--state all|open|closed]
Reads the issues with the gh CLI (logged in) and counts the "Graphics card and driver version" and "CPU, RAM and operating
system" answers. It counts reports, not players: hardware where things break shows up more. Standard library only.
"""
import argparse
import collections
import json
import re
import subprocess
import sys
REPO = "NotRexed/Hookshot"
def section(body: str, name: str) -> str:
m = re.search(r"### " + re.escape(name) + r"[^\n]*\n+(.*?)(\n### |\Z)", body or "", re.S)
return m.group(1).strip() if m else ""
def classify(gpu: str):
g = gpu.lower()
if re.search(r"nvidia|rtx|gtx|geforce", g):
cls = ("RTX 50" if re.search(r"rtx ?50\d0", g) else "RTX 40" if re.search(r"rtx ?40\d0", g) else
"RTX 30" if re.search(r"rtx ?30\d0", g) else "RTX 20" if re.search(r"rtx ?20\d0", g) else
"GTX 10/16 (no mesh shaders)" if re.search(r"gtx ?1[06]\d0", g) else "NVIDIA other")
return "NVIDIA", cls
if re.search(r"amd|radeon|\brx ?\d|rdna|780m|680m|890m|860m|8060s|z1|steam deck|legion|ally", g):
cls = ("AMD APU / handheld" if re.search(r"780m|680m|890m|860m|8060s|z1|steam deck|legion|ally|apu|vega", g) else
"RX 9000" if re.search(r"9070|9060", g) else "RX 7000" if re.search(r"7900|7800|7700|7600", g) else
"RX 6000" if re.search(r"6[5-9]\d0|6600|6650|6700|6800|6900", g) else
"RX 5000 or older (no mesh shaders)" if re.search(r"5[5-7]00|rx ?5[78]0|rx ?4[78]0", g) else "AMD other")
return "AMD", cls
if re.search(r"intel|arc\b|iris|uhd", g):
return "Intel", "Intel"
return "unknown", "unknown"
def os_of(text: str) -> str:
t = text.lower()
if re.search(r"linux|ubuntu|arch|fedora|steamos|bazzite|cachy|nixos|mint|pop!?_os|steam deck", t):
return "Linux / SteamOS"
if re.search(r"windows|win ?1[01]", t):
return "Windows"
return "unknown"
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
ap.add_argument("--limit", type=int, default=1000)
ap.add_argument("--state", default="all", choices=("all", "open", "closed"))
a = ap.parse_args()
out = subprocess.run(["gh", "issue", "list", "-R", REPO, "--state", a.state, "--limit", str(a.limit),
"--json", "number,title,body"], capture_output=True, text=True, encoding="utf-8")
if out.returncode:
print(out.stderr.strip() or "gh failed", file=sys.stderr)
return 1
issues = json.loads(out.stdout)
vendor, cls, oses = collections.Counter(), collections.Counter(), collections.Counter()
n = 0
for i in issues:
gpu = section(i["body"], "Graphics card")
sysinfo = section(i["body"], "CPU, RAM and operating system")
if not gpu and not sysinfo:
continue
n += 1
v, c = classify(gpu + " " + i["title"])
vendor[v] += 1
cls[c] += 1
oses[os_of(sysinfo + " " + gpu + " " + i["title"])] += 1
print(f"{len(issues)} issues read, {n} with hardware fields")
for title, counter in (("vendor", vendor), ("card class", cls), ("OS", oses)):
print(f"\n{title}:")
for k, v in counter.most_common():
print(f" {k:36s} {v:5d} {100 * v / max(n, 1):5.1f}%")
return 0
if __name__ == "__main__":
sys.exit(main())

53
scripts/hotlist.py Normal file
View File

@@ -0,0 +1,53 @@
"""Hot lifted functions from an all-thread sample (HOS_SAMPLE_ALL, scripts/crit.py): for each sample of a game thread
running in hookshot.exe (on Linux hookshot), the innermost lifted function (f_...) is charged. Writes the functions that make up the
first <share> of those samples, hottest first, one a line, for the lift's A64_HOT_FUNCS (their FP ops call the
helpers' inline fast paths; with A64_HOT_CHUNK=1 they are compiled together in code_hot.c).
python -I scripts/hotlist.py work/runs/<run>_all.txt <out> [share 0.85] [max 1500]
"""
import bisect, collections, os, sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import prof # noqa: E402
maps = prof.load_symbols()
game = next((m for m in prof.GAME_MODS if m in maps), None)
if not game:
sys.exit("no hookshot.map (Windows) or hookshot binary (Linux) in %s: set RX_MAPS" % os.environ.get("RX_MAPS", prof.BUILD))
addrs, names = maps[game]
share = float(sys.argv[3]) if len(sys.argv) > 3 else 0.85
cap = int(sys.argv[4]) if len(sys.argv) > 4 else 1500
c = collections.Counter()
total = 0
for line in open(sys.argv[1], encoding="utf-8", errors="replace"):
head, _, st = line.rstrip("\n").partition("|")
if not st or head.split(" ", 1)[-1].startswith("mkw_vk"):
continue
frames = st.split(";")
if not frames[0].startswith(game + "+"):
continue
for f in frames:
mod, _, off = f.rpartition("+")
if mod != game:
continue
i = bisect.bisect_right(addrs, int(off, 16)) - 1
n = names[i] if i >= 0 else ""
if n.startswith("f_"):
c[n] += 1
total += 1
break
out, acc = [], 0
for n, k in c.most_common():
if acc >= share * total or len(out) >= cap:
break
out.append(n)
acc += k
# (the game's Zstd decoder: the job loop's f_80115bf0 -> f_80115c80 tree. It runs while areas load and stream, which a play
# profile seldom samples, so it fell to the cold -O1 chunks. At the hot flags a whole decode takes 16% less time and the
# code is no bigger; the functions match Unicorn on every register and written page either way. Listed whatever the profile says)
ZSTD_DECODER = ("f_80115c80 f_80109790 f_80113880 f_80116490 f_801099a0 f_8010b160 f_8010ffc0 f_801102e0 f_80110860 "
"f_801117a0 f_80112880 f_80113080 f_801156e0 f_80115880 f_8010ad60 f_834ab610 f_834abda0 f_834ab4a0 "
"f_834ac0b0 f_834ac110 f_834ac2a0 f_8572f040").split()
out += [n for n in ZSTD_DECODER if n not in out]
open(sys.argv[2], "w").write("".join(n + "\n" for n in out))
print("%d samples in %d functions; %d functions cover %.0f%%" % (total, len(c), len(out), 100.0 * acc / max(1, total)))

155
scripts/linux/appimage.sh Executable file
View File

@@ -0,0 +1,155 @@
#!/bin/bash
# The Linux AppImage: Hookshot Builder with everything it needs to build the game on the player's PC. No game data and
# nothing made from it is in it: the game is built from the player's own copy, as on Windows.
#
# scripts/linux/deps.sh (first: the compiler, Python, FFmpeg, SDL2, headers into build-linux/deps)
# scripts/linux/appimage.sh -> build-linux/Hookshot-<version>-x86_64.AppImage
# The same image builds the game and, with --play, starts the one it saved.
#
# Built on the oldest system it is for (CI: Ubuntu 22.04): the sysroot it carries is that system's C library, so the
# game it builds runs on glibc 2.35 and newer. work/ (hot_funcs.txt, hot_order.txt, hookshot_pieces.prof,
# image/chunks.txt: addresses and counts from playing, no game code) is carried along when it is there.
set -euo pipefail
cd "$(dirname "$0")/../.."
ROOT=$PWD
B=$ROOT/build-linux
DEPS=${DEPS:-$B/deps}
APPDIR=$B/AppDir
VERSION=${HOOKSHOT_VERSION:-$(git describe --tags --always --match "v[0-9]*" 2>/dev/null || echo dev)}
export EMSDK=$DEPS/emsdk RX_CPM=$DEPS/cpm
PY=$DEPS/python/bin/python3
# ---- the Vulkan back end, built here once (it needs no game) ----
VKW=$B/vk-work
rm -rf "$VKW/sa/build/hookshot_vk.so"
OOT_WORK=$VKW RX_MARCH=x86-64-v3 RX_VK_ONLY=1 RX_RELEASE=1 "$PY" -I nx2/build.py --standalone --jobs "${JOBS:-$(nproc)}"
VK=$VKW/sa/build/hookshot_vk.so
[ -f "$VK" ] || { echo "hookshot_vk.so was not built"; exit 1; }
# ---- the tools' native helpers, built now: the AppImage is read-only. a64emit with its C++ runtime inside (the one this system
# builds with is newer than older systems have), and the stamp build_emit.py checks, so it is not built again ----
E=third_party/a64-kit/lift
"$EMSDK/upstream/bin/clang++" -std=c++20 -O2 -shared -fPIC -static-libstdc++ -static-libgcc -Wno-everything \
-o "$E/liba64emit.so" "$E/a64emit.cpp"
"$PY" -I -c "import hashlib, sys; f = ['$E/a64emit.cpp', '$E/suyu/arm64_to_c.h']; open('$E/liba64emit.so.sha1', 'w').write(hashlib.sha1(b''.join(open(x, 'rb').read() for x in f)).hexdigest())"
"$PY" -I third_party/a64-kit/lift/build_emit.py
"$PY" -I -c "import sys; sys.path.insert(0, 'nx2'); import zbic; zbic.lib()"
rm -rf "$APPDIR"
mkdir -p "$APPDIR/usr/share/hookshot" "$APPDIR/usr/lib/hookshot"
T=$APPDIR/usr/share/hookshot
L=$APPDIR/usr/lib/hookshot
# ---- the tools (the sources the player's build compiles, and the scripts) ----
cp -a nx2 host scripts LICENSE readme.md assets "$T/"
printf '%s
' "$VERSION" > "$T/VERSION" # (no git history in here: nx2/version.py reads the release name from it)
mkdir -p "$T/builder" "$T/gpu/render" "$T/gpu/vk" "$T/third_party"
cp -a builder/linux "$T/builder/"
cp -a gpu/render/*.h "$T/gpu/render/"
# (every gpu/vk header the host's sources include: v2.0.6 shipped without oot_cheat_items.h and no Linux build finished, issue 990.
# tests/test_appimage_headers.py checks that this covers what host/ includes)
for h in $(grep -rhoE 'gpu/vk/[A-Za-z0-9_]+[.]h' host | sort -u); do if [ -f "$h" ]; then cp -a "$h" "$T/gpu/vk/"; fi; done
cp -a third_party/opus "$T/third_party/"
cp -a third_party/kenney_input "$T/third_party/"
# (NVIDIA DLSS, when ticked in the builder: scripts/ngx_sdk.py and the Vulkan back end's sources, compiled with HOOKSHOT_NGX_SDK)
mkdir -p "$T/scripts"
cp scripts/ngx_sdk.py "$T/scripts/"
cp -a gpu/vk gpu/compat gpu/third_party "$T/gpu/"
cp -a third_party/imgui third_party/zstd third_party/fonts "$T/third_party/"
mkdir -p "$T/third_party/zbic" "$T/third_party/a64-kit"
cp -a third_party/zbic/zbic.so third_party/zbic/LICENSE third_party/zbic/PROVENANCE.md "$T/third_party/zbic/"
cp -a third_party/a64-kit/* "$T/third_party/a64-kit/"
rm -f "$T/third_party/a64-kit/tools/lld-link.exe"
if [ -d work ]; then
mkdir -p "$T/work/image"
for f in hot_funcs.txt hot_order.txt hookshot_pieces.prof pipeline_states.bin; do [ -f "work/$f" ] && cp "work/$f" "$T/work/"; done
[ -f work/image/chunks.txt ] && cp work/image/chunks.txt "$T/work/image/"
fi
find "$T" -name __pycache__ -prune -exec rm -rf {} +
# ---- what it builds with ----
cp -a "$DEPS/python" "$APPDIR/usr/python"
cp "$DEPS/tk/libtcl9tk9.0.so" "$DEPS/tk/libXft.so.2" "$DEPS/tk/libXrender.so.1" "$APPDIR/usr/python/lib/"
mkdir -p "$L/emsdk"
cp -a "$DEPS/emsdk/upstream" "$L/emsdk/"
cp -a "$DEPS/ffmpeg" "$L/ffmpeg"
cp "$VK" "$L/hookshot_vk.so"
cp "$DEPS/sdl2/libSDL2-2.0.so.0" "$L/libSDL2-2.0.so.0"
for d in "$DEPS"/cpm/*/*; do
mkdir -p "$T/cpm/${d#"$DEPS/cpm/"}"
cp -a "$d/include" "$T/cpm/${d#"$DEPS/cpm/"}/"
done
cp -a "$DEPS/licenses" "$T/licenses" # (LLVM, FFmpeg, dav1d, SDL2: what this AppImage carries besides Hookshot)
# ---- the licences of the third-party parts compiled into hookshot_vk.so (tests/test_appimage_licenses.py) ----
mkdir -p "$T/licenses/imgui" "$T/licenses/zstd" "$T/licenses/libertinus"
cp third_party/imgui/LICENSE.txt third_party/imgui/PROVENANCE.md "$T/licenses/imgui/"
cp third_party/zstd/LICENSE third_party/zstd/PROVENANCE.md "$T/licenses/zstd/"
cp third_party/fonts/libertinus/OFL.txt third_party/fonts/PROVENANCE.md "$T/licenses/libertinus/"
# ---- end of the licences ----
# the sysroot: this system's C library (headers, start files, the libraries to link against) and GCC's runtime pieces
S=$L/sysroot
mkdir -p "$S/usr/lib/x86_64-linux-gnu" "$S/usr/lib/gcc/x86_64-linux-gnu" "$S/lib64"
ln -s usr/lib "$S/lib"
cp -a /usr/include "$S/usr/"
rm -rf "$S/usr/include/c++" "$S/usr/include/x86_64-linux-gnu/c++" "$S/usr/include/llvm"* "$S/usr/include/clang"*
M=/usr/lib/x86_64-linux-gnu
for f in crt1.o crti.o crtn.o Scrt1.o rcrt1.o libc.so libc.so.6 libc_nonshared.a libm.so libm.so.6 libm.a libmvec.so libmvec.so.1 \
libmvec.a libpthread.so.0 libpthread.a libdl.so.2 libdl.a librt.so.1 librt.a libutil.a libgcc_s.so.1 ld-linux-x86-64.so.2; do
# System aliases can have absolute targets (e.g. libmvec.so). Carry their bytes;
# update parts refuse absolute-target symlinks, and must not use the host's library.
[ -e "$M/$f" ] && cp -aL "$M/$f" "$S/usr/lib/x86_64-linux-gnu/" || true
done
cp -aL /lib64/ld-linux-x86-64.so.2 "$S/lib64/"
GCC=$(ls -d /usr/lib/gcc/x86_64-linux-gnu/*/ | sort -V | tail -1)
GV=$(basename "$GCC")
G=$S/usr/lib/gcc/x86_64-linux-gnu/$GV
mkdir -p "$G" "$S/usr/include/c++" "$S/usr/include/x86_64-linux-gnu/c++"
cp -aL "/usr/include/c++/$GV" "$S/usr/include/c++/"
cp -aL "/usr/include/x86_64-linux-gnu/c++/$GV" "$S/usr/include/x86_64-linux-gnu/c++/"
for f in crtbegin.o crtbeginS.o crtbeginT.o crtend.o crtendS.o libgcc.a libgcc_eh.a libgcc_s.so libstdc++.a libstdc++fs.a libstdc++exp.a; do
[ -e "$GCC/$f" ] && cp -aL "$GCC/$f" "$G/" || true
done
# (libgcc_s.so is a linker script naming /lib/x86_64-linux-gnu/libgcc_s.so.1: found in the sysroot through lib -> usr/lib)
# ---- check: the bundled compiler builds and links a program against the sysroot, and it runs ----
cat > "$B/t.c" <<'EOF'
#include <pthread.h>
#include <stdio.h>
#include <math.h>
static void *f(void *p) { return p; }
int main(void) { pthread_t t; pthread_create(&t, 0, f, 0); pthread_join(t, 0); printf("ok %g\n", sqrt(2.0)); return 0; }
EOF
"$L/emsdk/upstream/bin/clang" --sysroot="$S" -fuse-ld=lld -O2 -march=x86-64-v3 "$B/t.c" -o "$B/t" -lpthread -lm
"$B/t"
cat > "$B/t.cpp" <<'EOF'
#include <algorithm>
#include <filesystem>
#include <format>
#include <string>
extern "C" int t_cpp(int v) { return (int)std::format("{}", v).size(); }
EOF
"$L/emsdk/upstream/bin/clang++" --sysroot="$S" -fuse-ld=lld -std=c++20 -O2 -march=x86-64-v3 -shared -fPIC -static-libstdc++ -static-libgcc \
"$B/t.cpp" -o "$B/t.so" -ldl -lpthread -lm
# ---- the AppDir's own files ----
cp builder/linux/AppRun "$APPDIR/AppRun"
cp builder/linux/hookshot-builder.desktop "$APPDIR/"
# (RecompLauncher reads this from inside the AppImage, at /recomplauncher.json)
cp recomplauncher.json "$APPDIR/recomplauncher.json"
cp assets/hookshot.png "$APPDIR/hookshot.png"
mkdir -p "$APPDIR/usr/share/icons/hicolor/256x256/apps"
cp assets/hookshot.png "$APPDIR/usr/share/icons/hicolor/256x256/apps/hookshot.png"
# (the update list, beside AppRun and not inside a hashed part: hookshot-linux.txt names each part's checksum)
# (the parts are served from the linux-parts release of the repository this is built in, where
# scripts/linux/publish_parts.sh uploads them: without --base the list named NotRexed/Hookshot's release, which a fork's
# CI does not upload to, so a fork's update list named parts that were not there)
python3 scripts/linux/make_parts.py "$APPDIR" "$VERSION" "$B/parts" \
--base "https://github.com/${GITHUB_REPOSITORY:-NotRexed/Hookshot}/releases/download/linux-parts"
OUT=$B/Hookshot-$VERSION-x86_64.AppImage
rm -f "$OUT"
ARCH=x86_64 APPIMAGE_EXTRACT_AND_RUN=1 "$DEPS/appimagetool" --runtime-file "$DEPS/runtime-x86_64" --comp zstd "$APPDIR" "$OUT"
echo "$OUT: $(du -h "$OUT" | cut -f1)"

View File

@@ -0,0 +1,51 @@
#!/bin/bash
# Game-free: a player's build started from what the AppImage carries, not from the repository. CI's AppImage job packages
# the builder but never builds a game with it, so a file the AppImage leaves out was found by every player instead: #1459
# made nx2/build.py write the menu's fonts from third_party/fonts at every build start, the AppImage does not carry that
# folder, and every Linux build stopped at LibertinusSans-Regular.ttf while CI stayed green (2026-11-10, #1567).
#
# scripts/linux/check_player_build.sh [<Hookshot-...-x86_64.AppImage>] (default: the newest in build-linux/)
#
# The image is extracted, as its runtime does, and run with AppRun's own environment (the AppImage's Python, compiler,
# sysroot, FFmpeg and the prebuilt Vulkan back end, HOOKSHOT_VK_PREBUILT) in an otherwise empty one, so nothing of the
# checkout or of CI's caches stands in for what the image lacks:
# - AppRun --cli --help: nx2/make_game.py, as the builder window starts it;
# - nx2/build.py --standalone --dry-run, as make_game.py runs it: its whole start-up up to the compile, which is where
# the game's lifted code is first needed. Every other file the compile would read must be in the image.
# Neither may write into the image's tree: on a player's PC it is read-only.
set -euo pipefail
cd "$(dirname "$0")/../.."
IMAGE=${1:-$(ls -t build-linux/Hookshot-*-x86_64.AppImage 2>/dev/null | head -1 || true)}
[ -n "$IMAGE" ] && [ -f "$IMAGE" ] || { echo "no AppImage to check (scripts/linux/appimage.sh makes one)"; exit 1; }
IMAGE=$(readlink -f "$IMAGE")
TMP=$(mktemp -d)
trap 'rm -rf "$TMP"' EXIT
(cd "$TMP" && "$IMAGE" --appimage-extract >/dev/null)
R=$TMP/squashfs-root
T=$R/usr/share/hookshot
mkdir -p "$TMP/home" "$TMP/work"
# (AppRun's environment, from AppRun itself: its lines before it starts Python, sourced with $0 naming it, so HERE is the image)
sed '/^PY=/,$d' "$R/AppRun" > "$TMP/apprun-env.sh"
in_image() {
env -i PATH=/usr/bin:/bin HOME="$TMP/home" LANG=C.UTF-8 bash -c '. "$1"; shift; exec "$@"' "$R/AppRun" "$TMP/apprun-env.sh" "$@"
}
touch "$TMP/before"
sleep 1 # (so a file written below is newer than the stamp)
echo "== AppRun --cli --help"
in_image "$R/AppRun" --cli --help > "$TMP/help.txt"
grep -q -- "--nup" "$TMP/help.txt" || { cat "$TMP/help.txt"; echo "nx2/make_game.py did not start"; exit 1; }
echo "== nx2/build.py --standalone --dry-run (as nx2/make_game.py runs it, with no game)"
# (-B: no __pycache__ written into the image's tree. make_game.py's other settings: nx2/make_game.py, the compile's run())
(cd "$T" && in_image env OOT_WORK="$TMP/work" RX_RELEASE=1 A64_HOT_CHUNK=1 PYTHONIOENCODING=utf-8 \
"$R/usr/python/bin/python3" -I -B -u -X utf8 nx2/build.py --standalone --jobs 1 --dry-run)
# (Python's own __pycache__ aside: AppRun runs Python with -I, which writes it where it can, and does without where it cannot)
written=$(find "$R" -type f -newer "$TMP/before" ! -path "*/__pycache__/*" -print)
if [ -n "$written" ]; then
echo "a player's build wrote into the AppImage's tree, which is read-only on a player's PC:"
echo "$written" | head -20
exit 1
fi
echo "a player's build starts from the AppImage's tree"

208
scripts/linux/deps.sh Executable file
View File

@@ -0,0 +1,208 @@
#!/bin/bash
# What the Linux AppImage carries, fetched and built into build-linux/deps (or $DEPS): each part once, kept while its
# version stays the same. Run on the oldest system the AppImage is for (CI: Ubuntu 22.04, glibc 2.35).
#
# emsdk/upstream/ LLVM's clang and lld (the layout third_party/a64-kit/tools/toolchain.py finds through EMSDK):
# the player's PC compiles the game with them. The play profile (work/hookshot_pieces.prof) wants
# LLVM 23 or newer.
# python/ CPython (python-build-standalone) with lz4, numpy, capstone, unicorn and zstandard: the tools
# (and nx2/prompt_requirements.txt's, which draw the button prompts)
# tk/ libtcl9tk9.0.so built with Xft, with libXft and libXrender: put over python/'s, which draws only
# X core fonts (under XWayland one 13 px bitmap font, whatever the size asked for or the screen)
# ffmpeg/ FFmpeg's libavcodec, libswscale, libswresample, libavutil, static and LGPL, with dav1d for OoT's
# AV1 movies (VP8 / VP9 too) - linked into the game
# sdl2/ libSDL2-2.0.so.0: the window, sound and controllers, placed beside the game
# cpm/ Vulkan-Headers, SPIRV-Headers and VulkanMemoryAllocator, in the CPM cache layout nx2/build.py reads
# appimagetool the AppImage packer
# runtime-x86_64 the AppImage runtime it puts in front of the image (pinned: appimagetool's own download of the
# 'continuous' one, at packing time, fails whenever GitHub answers 504)
set -euo pipefail
cd "$(dirname "$0")/../.."
DEPS=${DEPS:-$PWD/build-linux/deps}
DL=$DEPS/downloads
mkdir -p "$DEPS" "$DL"
JOBS=$(nproc)
LLVM_VER=23.1.5
PY_TAG=20261026
PY_VER=3.12.16
TCLTK_VER=9.0.4 # (python-build-standalone's Tcl and Tk: its tk9.0 scripts stay, beside this libtcl9tk9.0.so)
TCL_SHA256=d0aed49230bc02a65c1e0229e65f34590a4b037ec40d546f32573b467f7551ea
TK_SHA256=d7a146d2917eb8b5cc95276dbf0e3d03c7464d2b19c1675357857c989301dbb4
SDL_VER=2.32.10
FFMPEG_VER=9.0.2
DAV1D_VER=1.5.4
VK_HEADERS=v1.4.345
SPIRV_HEADERS=vulkan-sdk-1.4.341.0
VMA=v3.3.0
APPIMAGETOOL=1.9.1
APPIMAGE_RUNTIME=20251108
APPIMAGE_RUNTIME_SHA256=2fca8b443c92510f1483a883f60061ad09b46b978b2631c807cd873a47ec260d
# (the SHA-256 of each release download: a part whose download differs stops the script. Change them with the versions.
# GitHub's and GitLab's tag archives (dav1d, the headers) are made on demand and not pinned here)
LLVM_SHA256=f72edb0b36580ece7f52e23858f00c18fad4e87395ea7ad05d7e55c39f8fc78a
PY_SHA256=ce4f6d0a6cfdb3bed6e44c2f9affd85943c76a42e25ad3a259002f31d6fb8a20
FFMPEG_SHA256=8c3850283eb25fa026482078a04051e0be17347b09ef81a0849bec15a96e002e
SDL_SHA256=5f5993c530f084535c65a6879e9b26ad441169b3e25d789d83287040a9ca5165
APPIMAGETOOL_SHA256=ed4ce84f0d9caff66f50bcca6ff6f35aae54ce8135408b3fa33abfc3cb384eb0
CURL=(curl -fL --retry 5 --retry-delay 10 --proto '=https')
# A GitHub release link (github.com/O/R/releases/download/TAG/NAME) can answer 504 for a long while: the same file is
# then fetched through the GitHub API (with $GH_TOKEN when it is set).
fetch_github_api() { # url out
[[ $1 =~ ^https://github\.com/([^/]+/[^/]+)/releases/download/([^/]+)/([^/]+)$ ]] || return 1
local repo=${BASH_REMATCH[1]} tag=${BASH_REMATCH[2]} name=${BASH_REMATCH[3]} auth=() id
[ -n "${GH_TOKEN:-}" ] && auth=(-H "Authorization: Bearer $GH_TOKEN")
id=$("${CURL[@]}" -sS "${auth[@]}" "https://api.github.com/repos/$repo/releases/tags/$tag" |
python3 -c 'import json,sys; print(next(a["id"] for a in json.load(sys.stdin)["assets"] if a["name"] == sys.argv[1]))' "$name") ||
return 1
echo "release link failed: fetching $name through the GitHub API"
"${CURL[@]}" "${auth[@]}" -H "Accept: application/octet-stream" -o "$2" "https://api.github.com/repos/$repo/releases/assets/$id"
}
fetch() { # url file [sha256: checked on every run, the download removed when it differs]
if [ ! -f "$DL/$2" ]; then
"${CURL[@]}" -o "$DL/$2.part" "$1" || fetch_github_api "$1" "$DL/$2.part"
mv -f "$DL/$2.part" "$DL/$2"
fi
if [ -n "${3:-}" ] && ! echo "$3 $DL/$2" | sha256sum -c --quiet -; then
rm -f "$DL/$2"
echo "$2: not the expected download (SHA-256 differs)"
exit 1
fi
}
done_for() { [ -f "$DEPS/.$1" ] && [ "$(cat "$DEPS/.$1")" = "$2" ]; }
mark() { echo "$2" > "$DEPS/.$1"; }
# ---- LLVM: clang, lld and clang's own headers, nothing else ----
if ! done_for llvm "$LLVM_VER"; then
fetch "https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VER/LLVM-$LLVM_VER-Linux-X64.tar.xz" llvm.tar.xz "$LLVM_SHA256"
rm -rf "$DEPS/emsdk" "$DEPS/llvm-x"
mkdir -p "$DEPS/llvm-x"
tar -xJf "$DL/llvm.tar.xz" -C "$DEPS/llvm-x" --strip-components=1 --wildcards \
'*/bin/clang' '*/bin/clang++' '*/bin/clang-[0-9]*' '*/bin/lld' '*/bin/ld.lld' '*/bin/llvm-profdata' '*/lib/clang/*'
U=$DEPS/emsdk/upstream
mkdir -p "$U/bin" "$U/lib"
mv "$DEPS/llvm-x/bin/"* "$U/bin/"
mv "$DEPS/llvm-x/lib/clang" "$U/lib/clang"
rm -rf "$DEPS/llvm-x"
# (the runtimes: only the profile and builtins libraries; no sanitizers)
find "$U"/lib/clang/*/lib -type f ! -name 'libclang_rt.profile*.a' ! -name 'libclang_rt.builtins*.a' -delete
strip "$U/bin/clang-"[0-9]* "$U/bin/lld" "$U/bin/llvm-profdata" 2>/dev/null || true
"$U/bin/clang" --version | head -1
[ -f "$U/lib/clang/${LLVM_VER%%.*}/include/stddef.h" ] || { echo "clang's own headers are missing"; exit 1; }
mark llvm "$LLVM_VER"
fi
# ---- Python and the tools' modules ----
PY_MARK="$PY_VER-$PY_TAG-$(sha256sum requirements.txt nx2/prompt_requirements.txt | sha256sum | cut -c1-12)" # (a new pin installs again)
if ! done_for python "$PY_MARK"; then
fetch "https://github.com/astral-sh/python-build-standalone/releases/download/$PY_TAG/cpython-$PY_VER+$PY_TAG-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz" python.tar.gz "$PY_SHA256"
rm -rf "$DEPS/python"
tar -xzf "$DL/python.tar.gz" -C "$DEPS"
"$DEPS/python/bin/python3" -m pip install --no-cache-dir --only-binary :all: --disable-pip-version-check -q \
-r requirements.txt
rm -rf "$DEPS/python/lib/python3.12/test" "$DEPS/python/lib/python3.12/idlelib" "$DEPS/python/lib/python3.12/ensurepip"
find "$DEPS/python" -name __pycache__ -prune -exec rm -rf {} +
mark python "$PY_MARK"
fi
# ---- Tk with Xft (the builder window's fonts; Tcl is built only for Tk's configure and stub library) ----
if ! done_for tk "$TCLTK_VER"; then
fetch "https://downloads.sourceforge.net/project/tcl/Tcl/$TCLTK_VER/tcl$TCLTK_VER-src.tar.gz" "tcl$TCLTK_VER.tar.gz"
fetch "https://downloads.sourceforge.net/project/tcl/Tcl/$TCLTK_VER/tk$TCLTK_VER-src.tar.gz" "tk$TCLTK_VER.tar.gz"
printf '%s %s\n' "$TCL_SHA256" "$DL/tcl$TCLTK_VER.tar.gz" "$TK_SHA256" "$DL/tk$TCLTK_VER.tar.gz" | sha256sum -c -
rm -rf "$DEPS/src-tcl" "$DEPS/src-tk" "$DEPS/tk"
mkdir -p "$DEPS/src-tcl" "$DEPS/src-tk" "$DEPS/tk"
tar -xzf "$DL/tcl$TCLTK_VER.tar.gz" -C "$DEPS/src-tcl" --strip-components=1
tar -xzf "$DL/tk$TCLTK_VER.tar.gz" -C "$DEPS/src-tk" --strip-components=1
(cd "$DEPS/src-tcl/unix" && ./configure --enable-shared --with-system-libtommath=no && make -j"$JOBS" binaries)
# (no rpath: libXft and libXrender are found beside it through _tkinter's $ORIGIN/../..)
(cd "$DEPS/src-tk/unix" && ./configure --enable-shared --with-tcl="$DEPS/src-tcl/unix" --enable-xft --disable-libcups \
--disable-xss --disable-zipfs --disable-rpath && make -j"$JOBS" binaries)
cp "$DEPS/src-tk/unix/libtcl9tk9.0.so" "$DEPS/tk/"
strip --strip-unneeded "$DEPS/tk/libtcl9tk9.0.so"
readelf -d "$DEPS/tk/libtcl9tk9.0.so" | grep -q 'libXft\.so\.2' || { echo "Tk was built without Xft"; exit 1; }
cp -L /usr/lib/x86_64-linux-gnu/libXft.so.2 /usr/lib/x86_64-linux-gnu/libXrender.so.1 "$DEPS/tk/"
cp "$DEPS/src-tk/license.terms" "$DEPS/tk/Tk-license.terms"
rm -rf "$DEPS/src-tcl" "$DEPS/src-tk"
mark tk "$TCLTK_VER"
fi
# ---- dav1d and FFmpeg (static, position independent: linked into the game's executable) ----
P=$DEPS/ffmpeg
if ! done_for ffmpeg "$FFMPEG_VER-$DAV1D_VER"; then
rm -rf "$P" "$DEPS/src-dav1d" "$DEPS/src-ffmpeg"
fetch "https://code.videolan.org/videolan/dav1d/-/archive/$DAV1D_VER/dav1d-$DAV1D_VER.tar.gz" dav1d.tar.gz
mkdir -p "$DEPS/src-dav1d" && tar -xzf "$DL/dav1d.tar.gz" -C "$DEPS/src-dav1d" --strip-components=1
(cd "$DEPS/src-dav1d" && meson setup build --prefix="$P" --libdir=lib --buildtype=release --default-library=static \
-Db_staticpic=true -Denable_tools=false -Denable_tests=false -Denable_examples=false && ninja -C build install)
fetch "https://ffmpeg.org/releases/ffmpeg-$FFMPEG_VER.tar.xz" ffmpeg.tar.xz "$FFMPEG_SHA256"
mkdir -p "$DEPS/src-ffmpeg" && tar -xJf "$DL/ffmpeg.tar.xz" -C "$DEPS/src-ffmpeg" --strip-components=1
(cd "$DEPS/src-ffmpeg" && PKG_CONFIG_PATH="$P/lib/pkgconfig" ./configure --prefix="$P" --pkg-config-flags=--static \
--enable-static --disable-shared --enable-pic --disable-programs --disable-doc --disable-network --disable-autodetect \
--disable-everything --disable-avformat --disable-avfilter --disable-avdevice \
--enable-avcodec --enable-swscale --enable-swresample --enable-avutil \
--enable-libdav1d --enable-decoder=vp8,vp9,libdav1d --enable-parser=vp8,vp9,av1 \
&& make -j"$JOBS" && make install)
rm -rf "$DEPS/src-dav1d" "$DEPS/src-ffmpeg" "$P/share" "$P/lib/pkgconfig"
mark ffmpeg "$FFMPEG_VER-$DAV1D_VER"
fi
# ---- SDL2 (its X11, Wayland and sound back ends load their libraries at run time) ----
if ! done_for sdl2 "$SDL_VER"; then
fetch "https://github.com/libsdl-org/SDL/releases/download/release-$SDL_VER/SDL2-$SDL_VER.tar.gz" sdl2.tar.gz "$SDL_SHA256"
rm -rf "$DEPS/src-sdl2" "$DEPS/sdl2"
mkdir -p "$DEPS/src-sdl2" && tar -xzf "$DL/sdl2.tar.gz" -C "$DEPS/src-sdl2" --strip-components=1
cmake -S "$DEPS/src-sdl2" -B "$DEPS/src-sdl2/build" -G Ninja -DCMAKE_BUILD_TYPE=Release -DSDL_SHARED=ON -DSDL_STATIC=OFF \
-DSDL_TEST=OFF -DSDL_X11=ON -DSDL_WAYLAND=ON -DSDL_PULSEAUDIO=ON -DSDL_PIPEWIRE=ON -DSDL_ALSA=ON
ninja -C "$DEPS/src-sdl2/build"
mkdir -p "$DEPS/sdl2"
cp -L "$DEPS/src-sdl2/build/libSDL2-2.0.so.0" "$DEPS/sdl2/libSDL2-2.0.so.0"
strip "$DEPS/sdl2/libSDL2-2.0.so.0"
cp "$DEPS/src-sdl2/LICENSE.txt" "$DEPS/sdl2/LICENSE.txt"
rm -rf "$DEPS/src-sdl2"
mark sdl2 "$SDL_VER"
fi
# ---- the Vulkan back end's headers ----
if ! done_for cpm "$VK_HEADERS-$SPIRV_HEADERS-$VMA"; then
rm -rf "$DEPS/cpm"
get_headers() { # repo tag name
fetch "https://github.com/$1/archive/refs/tags/$2.tar.gz" "$3-$2.tar.gz"
mkdir -p "$DEPS/cpm/$3/$2"
tar -xzf "$DL/$3-$2.tar.gz" -C "$DEPS/cpm/$3/$2" --strip-components=1
}
get_headers KhronosGroup/Vulkan-Headers "$VK_HEADERS" vulkanheaders
get_headers KhronosGroup/SPIRV-Headers "$SPIRV_HEADERS" spirv-headers
get_headers GPUOpen-LibrariesAndSDKs/VulkanMemoryAllocator "$VMA" vulkanmemoryallocator
mark cpm "$VK_HEADERS-$SPIRV_HEADERS-$VMA"
fi
# ---- the licenses of what the AppImage carries (Python's and its modules' are in python/) ----
if ! done_for licenses "$LLVM_VER-$FFMPEG_VER-$DAV1D_VER-$SDL_VER-$TCLTK_VER"; then
L=$DEPS/licenses
rm -rf "$L" && mkdir -p "$L"
curl -fsSL -o "$L/LLVM-LICENSE.TXT" "https://raw.githubusercontent.com/llvm/llvm-project/llvmorg-$LLVM_VER/llvm/LICENSE.TXT"
curl -fsSL -o "$L/FFmpeg-COPYING.LGPLv2.1" "https://raw.githubusercontent.com/FFmpeg/FFmpeg/n$FFMPEG_VER/COPYING.LGPLv2.1"
curl -fsSL -o "$L/FFmpeg-LICENSE.md" "https://raw.githubusercontent.com/FFmpeg/FFmpeg/n$FFMPEG_VER/LICENSE.md"
curl -fsSL -o "$L/dav1d-COPYING" "https://code.videolan.org/videolan/dav1d/-/raw/$DAV1D_VER/COPYING"
cp "$DEPS/sdl2/LICENSE.txt" "$L/SDL2-LICENSE.txt"
cp "$DEPS/tk/Tk-license.terms" "$L/Tk-license.terms"
cp /usr/share/doc/libxft2/copyright "$L/libXft-copyright"
cp /usr/share/doc/libxrender1/copyright "$L/libXrender-copyright"
mark licenses "$LLVM_VER-$FFMPEG_VER-$DAV1D_VER-$SDL_VER-$TCLTK_VER"
fi
# ---- appimagetool ----
if ! done_for appimagetool "$APPIMAGETOOL"; then
fetch "https://github.com/AppImage/appimagetool/releases/download/$APPIMAGETOOL/appimagetool-x86_64.AppImage" appimagetool "$APPIMAGETOOL_SHA256"
cp "$DL/appimagetool" "$DEPS/appimagetool" && chmod +x "$DEPS/appimagetool"
mark appimagetool "$APPIMAGETOOL"
fi
if ! done_for runtime "$APPIMAGE_RUNTIME"; then
fetch "https://github.com/AppImage/type2-runtime/releases/download/$APPIMAGE_RUNTIME/runtime-x86_64" "runtime-x86_64-$APPIMAGE_RUNTIME"
echo "$APPIMAGE_RUNTIME_SHA256 $DL/runtime-x86_64-$APPIMAGE_RUNTIME" | sha256sum -c -
cp "$DL/runtime-x86_64-$APPIMAGE_RUNTIME" "$DEPS/runtime-x86_64"
mark runtime "$APPIMAGE_RUNTIME"
fi
echo "deps in $DEPS: $(du -sh --exclude=downloads "$DEPS" | cut -f1)"

117
scripts/linux/make_parts.py Normal file
View File

@@ -0,0 +1,117 @@
"""The parts a Linux update downloads (builder/linux/linux_update.py), from an AppDir.
python3 scripts/linux/make_parts.py <AppDir> <tag> <out folder> [--base <url>]
Each directory that changes on its own is one zip: the tools (every release), Python, clang, FFmpeg, the sysroot, and
the two libraries beside them. A zip is deterministic, so a directory whose files did not change has the same SHA-256
next time and the builder copies the one it already has. The zips are uploaded to the linux-parts release; the version's
own release carries hookshot-linux.txt, which names them.
The list is not inside a zip (it names the zips' checksums).
"""
import argparse
import hashlib
import os
import sys
import zipfile
sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
"builder", "linux"))
from part_links import SYSROOT_PART, is_absolute_link, relative_sysroot_link # noqa: E402
# (dest under the AppDir, and None for the whole directory or the file names when the directory holds other parts too)
PARTS = (
("usr/share/hookshot", None),
("usr/python", None),
("usr/lib/hookshot/emsdk", None),
("usr/lib/hookshot/ffmpeg", None),
("usr/lib/hookshot/sysroot", None),
("usr/lib/hookshot", ("hookshot_vk.so", "libSDL2-2.0.so.0")),
)
def _entries(root, dest, files):
base = os.path.join(root, dest)
if files is not None:
return [(name, os.path.join(base, name)) for name in files]
out = []
for dirpath, dirnames, names in os.walk(base, followlinks=False):
rel = os.path.relpath(dirpath, base)
real = []
for name in sorted(dirnames):
link = os.path.join(dirpath, name)
arc = name if rel == os.curdir else rel.replace(os.sep, "/") + "/" + name
if os.path.islink(link):
out.append((arc, link))
else:
real.append(name)
dirnames[:] = real
for name in sorted(names):
full = os.path.join(dirpath, name)
arc = name if rel == os.curdir else rel.replace(os.sep, "/") + "/" + name
out.append((arc, full))
return out
def write_zip(path, root, dest, files):
entries = _entries(root, dest, files)
with zipfile.ZipFile(path + ".new", "w", zipfile.ZIP_DEFLATED, compresslevel=6) as zipped:
for arc, full in entries:
info = zipfile.ZipInfo(arc, date_time=(2026, 1, 1, 0, 0, 0))
info.compress_type = zipfile.ZIP_DEFLATED
info.create_system = 3
if os.path.islink(full):
info.external_attr = 0o120777 << 16
link_to = os.readlink(full)
if dest == SYSROOT_PART and is_absolute_link(link_to):
link_to = relative_sysroot_link(os.path.join(root, dest), full, link_to)
zipped.writestr(info, link_to, compresslevel=6)
continue
info.external_attr = (0o755 if os.access(full, os.X_OK) else 0o644) << 16
with open(full, "rb") as handle:
zipped.writestr(info, handle.read(), compresslevel=6)
os.replace(path + ".new", path)
digest = hashlib.sha256()
with open(path, "rb") as handle:
while True:
block = handle.read(1 << 20)
if not block:
break
digest.update(block)
return digest.hexdigest(), os.path.getsize(path)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("appdir")
parser.add_argument("tag")
parser.add_argument("out")
parser.add_argument("--base", default="https://github.com/NotRexed/Hookshot/releases/download/linux-parts")
args = parser.parse_args()
root = os.path.abspath(args.appdir)
if not os.path.isfile(os.path.join(root, "AppRun")):
sys.exit("not an AppDir: %s" % root)
os.makedirs(args.out, exist_ok=True)
base = args.base.rstrip("/")
lines = ["tag %s" % args.tag]
for dest, files in PARTS:
if not os.path.isdir(os.path.join(root, dest)):
sys.exit("the AppDir has no %s" % dest)
leaf = dest.replace("/", "-")
path = os.path.join(args.out, "HookshotLinux-%s.zip" % leaf)
digest, size = write_zip(path, root, dest, files)
named = os.path.join(args.out, "HookshotLinux-%s-%s.zip" % (leaf, digest[:12]))
os.replace(path, named)
lines.append("part %s %d %s %s/%s" % (digest, size, dest, base, os.path.basename(named)))
lines.append("tree" if files is None else "\n".join("file %s" % name for name in files))
print("%-32s %5d MB %s" % (dest, size >> 20, digest[:16]), flush=True)
text = "\n".join(lines) + "\n"
with open(os.path.join(args.out, "hookshot-linux.txt"), "w", encoding="utf-8", newline="\n") as handle:
handle.write(text)
# (beside AppRun, not inside a hashed part: the list names those checksums)
with open(os.path.join(root, "hookshot-linux.txt"), "w", encoding="utf-8", newline="\n") as handle:
handle.write(text)
if __name__ == "__main__":
main()

15
scripts/linux/ngx-sdk.sh Executable file
View File

@@ -0,0 +1,15 @@
#!/bin/bash
# ngx-sdk.sh: NVIDIA's DLSS SDK files for a local Linux build with NVIDIA DLSS (HOOKSHOT_NGX_SDK): scripts/ngx_sdk.py, which
# says what it fetches and asks for NVIDIA's license (--accept-nvidia-license: accepted without the question).
#
# Usage: scripts/linux/ngx-sdk.sh [--accept-nvidia-license] [<sdk folder>] [<runtime folder>]
# <sdk folder>: default build-linux/ngx-sdk; <runtime folder>: libnvidia-ngx-dlss.so copied there too (a game folder's
# ngx/, or $XDG_DATA_HOME/hookshot/ngx; RX_VK_DLSS_DIR names another)
# Then build with HOOKSHOT_NGX_SDK=<sdk folder>.
set -euo pipefail
ROOT=$(cd "$(dirname "$0")/../.." && pwd)
accept=()
if [ "${1:-}" = --accept-nvidia-license ]; then accept=(--accept-nvidia-license); shift; fi
args=(--platform linux "${accept[@]}")
[ -n "${2:-}" ] && args+=(--runtime-dir "$2")
exec "${PYTHON:-python3}" -I "$ROOT/scripts/ngx_sdk.py" "${args[@]}" ${1:+"$1"}

47
scripts/linux/publish_parts.sh Executable file
View File

@@ -0,0 +1,47 @@
#!/bin/bash
# Upload the Linux builder's parts (scripts/linux/make_parts.py) to the linux-parts release, which is not "latest".
# A zip whose name is already there is left (its checksum is in the name). hookshot-linux.txt itself goes on the
# version release, with the AppImage. Run from CI on a tag, with GH_TOKEN and GITHUB_REPOSITORY.
set -euo pipefail
ROOT=$(cd "$(dirname "$0")/../.." && pwd)
PARTS=${1:-$ROOT/build-linux/parts}
REPO=${GITHUB_REPOSITORY:?}
TOKEN=${GH_TOKEN:?}
API="https://api.github.com/repos/$REPO"
AUTH=(-H "Authorization: Bearer $TOKEN" -H "Accept: application/vnd.github+json" -H "User-Agent: Hookshot")
J=$(mktemp) # (the release's JSON: a private file, not a fixed name in /tmp)
trap 'rm -f "$J"' EXIT
code=$(curl -s -o "$J" -w '%{http_code}' "${AUTH[@]}" "$API/releases/tags/linux-parts")
if [ "$code" = "404" ]; then
curl -fsSL "${AUTH[@]}" -X POST "$API/releases" -d "$(python3 - <<'PY'
import json, os
print(json.dumps({
"tag_name": "linux-parts",
"target_commitish": os.environ["GITHUB_SHA"],
"name": "Linux builder parts",
"body": "Parts Hookshot's Linux builder downloads. A file's name is its checksum; a part that did not change is not uploaded again. This release is not a version of the game.",
"prerelease": True,
"make_latest": "false",
}))
PY
)" > "$J"
fi
python3 - "$J" <<'PY'
import json, sys
json.load(open(sys.argv[1]))["upload_url"]
PY
upload=$(python3 -c 'import json, sys; print(json.load(open(sys.argv[1]))["upload_url"].split("{")[0])' "$J")
have=$(python3 -c 'import json, sys; print("\n".join(a["name"] for a in json.load(open(sys.argv[1])).get("assets", [])))' "$J")
for zip in "$PARTS"/HookshotLinux-*.zip; do
[ -f "$zip" ] || continue
name=$(basename "$zip")
if printf '%s\n' "$have" | grep -qx "$name"; then
echo "already on linux-parts: $name"
continue
fi
echo "uploading $name"
curl -fsSL --retry 5 --retry-delay 10 -X POST "${AUTH[@]}" -H "Content-Type: application/zip" \
--data-binary @"$zip" "$upload?name=$name" >/dev/null
done
echo "parts published"

View File

@@ -0,0 +1,73 @@
"""The Python Hookshot Builder's download: one zip with python.org's own Python (python.exe and pythonw.exe, signed by the Python
Software Foundation), Qt for Python's DLLs (signed by The Qt Company), the builder's code (builder/py) and the .bat that starts it.
Nothing in it is an executable of Hookshot's own.
python -I scripts/make_py_builder.py --python <python folder> --qt <PySide6 site folder> --tag <tag> <out zip>
--python: a python.org Windows embeddable Python of the version the build tools use (its pythonw.exe starts the window without a
console). --qt: a folder holding PySide6 and shiboken6 (the two wheels unpacked). Only the parts of Qt the window uses are taken:
QtCore, QtGui, QtWidgets, the Windows platform plugin and the icon and image plugins (~40 MB of the wheel's ~200)."""
import argparse
import os
import zipfile
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
TOP = "Hookshot Builder"
QT_KEEP_DLLS = ("Qt6Core.dll", "Qt6Gui.dll", "Qt6Widgets.dll", "Qt6Svg.dll", "pyside6.abi3.dll", "msvcp140.dll", "msvcp140_1.dll",
"msvcp140_2.dll", "vcruntime140.dll", "vcruntime140_1.dll", "concrt140.dll")
QT_KEEP_PY = ("QtCore.pyd", "QtGui.pyd", "QtWidgets.pyd", "__init__.py", "_config.py", "_git_pyside_version.py")
QT_PLUGINS = ("platforms/qwindows.dll", "imageformats/qico.dll", "imageformats/qjpeg.dll", "imageformats/qsvg.dll",
"styles/qmodernwindowsstyle.dll")
BAT = ('@echo off\r\n'
'rem Hookshot Builder: python.org\'s pythonw.exe runs the builder (app\\hookshot_builder_start.py) with no console window.\r\n'
'start "" "%~dp0python\\pythonw.exe" -I "%~dp0app\\hookshot_builder_start.py" %*\r\n')
def add_tree(z, src, dst, keep=lambda rel: True):
n = 0
for d, ds, fs in os.walk(src):
ds[:] = sorted(x for x in ds if x != "__pycache__")
for f in sorted(fs):
full = os.path.join(d, f)
rel = os.path.relpath(full, src).replace(os.sep, "/")
if keep(rel):
z.write(full, "%s/%s/%s" % (TOP, dst, rel))
n += 1
return n
def main():
p = argparse.ArgumentParser()
p.add_argument("--python", required=True)
p.add_argument("--qt", required=True)
p.add_argument("--tag", required=True)
p.add_argument("out")
a = p.parse_args()
for need in ("python.exe", "pythonw.exe"):
if not os.path.isfile(os.path.join(a.python, need)):
raise SystemExit("%s has no %s (use python.org's Windows embeddable package)" % (a.python, need))
pyside, shiboken = os.path.join(a.qt, "PySide6"), os.path.join(a.qt, "shiboken6")
if not os.path.isdir(pyside) or not os.path.isdir(shiboken):
raise SystemExit("%s has no PySide6 and shiboken6" % a.qt)
def qt_keep(rel):
if "/" not in rel:
return rel in QT_KEEP_DLLS or rel in QT_KEEP_PY
return rel.startswith("plugins/") and rel[len("plugins/"):] in QT_PLUGINS
with zipfile.ZipFile(a.out + ".new", "w", zipfile.ZIP_DEFLATED, compresslevel=9) as z:
z.writestr("%s/Hookshot Builder.bat" % TOP, BAT)
n = add_tree(z, a.python, "python")
n += add_tree(z, pyside, "qt/PySide6", qt_keep)
n += add_tree(z, shiboken, "qt/shiboken6")
py = os.path.join(ROOT, "builder", "py")
n += add_tree(z, py, "app", lambda rel: rel not in ("gen_strings.py", "shots.py"))
z.writestr("%s/app/VERSION" % TOP, a.tag + "\n")
for lic in ("LICENSE",):
z.write(os.path.join(ROOT, lic), "%s/%s" % (TOP, lic))
os.replace(a.out + ".new", a.out)
print("%s: %d files, %d MB" % (a.out, n, os.path.getsize(a.out) >> 20))
if __name__ == "__main__":
main()

122
scripts/make_setup_parts.py Normal file
View File

@@ -0,0 +1,122 @@
"""The parts HookshotBuilder-Setup.exe downloads (builder/setup/setup.c), from a Hookshot Builder package folder.
python -I scripts/make_setup_parts.py <package folder> <tag> <out folder> [--base <url the parts are served from>]
[--reuse <an earlier release's hookshot-setup.txt>]
A release is split by how often each part changes, so an update downloads only what changed: the builder's own code (a few MB)
in every release, the .NET runtime, Python, emsdk, the CPM packages and third_party only when they change. llvm-mingw comes from
its official release. Each zip is deterministic (sorted names, fixed dates and attributes): a part whose files did not change
has the same SHA-256 in the next release, and the setup takes it from its download cache instead of downloading it again.
Writes <out>/hookshot-setup.txt and one HookshotBuilder-<part>-<sha256 prefix>.zip per part (upload the ones the builder-parts
release lacks). Scripts the build does not use (.ps1, .bat, .js,
.html: antivirus machine-learning guesses fire on them) are left out.
--reuse: a part whose tools/<name> folder the package lacks is taken from that list as it is (its line, with its URL), so a
package made in CI (.github/workflows/windows-builder.yml) carries the Python, emsdk, CPM and wgpu-native parts the maintainer
made. Without the folder and without the part in that list it is an error.
"""
import argparse
import hashlib
import os
import sys
import zipfile
SKIP_EXT = {".ps1", ".psm1", ".bat", ".cmd", ".js", ".vbs", ".html", ".htm", ".hta", ".wsf", ".pyc"}
SKIP_DIRS = {"__pycache__"}
TOOLS_PARTS = ("python", "emsdk", "cpm", "third_party", "wgpu-native") # (tools/<name>: each its own part)
LLVM = ("552191b5754c49a5a20723276a6fc56541ea1922c4c50b7b669022333fc1b40f", 191037046,
"llvm-mingw-20261104-ucrt-x86_64",
"https://github.com/mstorsjo/llvm-mingw/releases/download/20261104/llvm-mingw-20261104-ucrt-x86_64.zip")
def files_under(root, top):
out = []
for d, ds, fs in os.walk(os.path.join(root, top)):
ds[:] = sorted(x for x in ds if x not in SKIP_DIRS)
for f in fs:
if os.path.splitext(f)[1].lower() not in SKIP_EXT:
out.append(os.path.relpath(os.path.join(d, f), root).replace(os.sep, "/"))
return out
def write_zip(path, root, names):
"""Deterministic: the same files give the same bytes (sorted, a fixed date, fixed attributes)"""
with zipfile.ZipFile(path + ".new", "w", zipfile.ZIP_DEFLATED, compresslevel=6) as z:
for n in sorted(names):
info = zipfile.ZipInfo("HookshotBuilder/" + n, date_time=(2026, 1, 1, 0, 0, 0))
info.compress_type = zipfile.ZIP_DEFLATED
info.external_attr = 0o644 << 16
with open(os.path.join(root, n.replace("/", os.sep)), "rb") as f:
z.writestr(info, f.read(), compresslevel=6)
os.replace(path + ".new", path)
h = hashlib.sha256()
with open(path, "rb") as f:
while b := f.read(1 << 20):
h.update(b)
return h.hexdigest(), os.path.getsize(path)
def main():
p = argparse.ArgumentParser()
p.add_argument("package")
p.add_argument("tag")
p.add_argument("out")
p.add_argument("--base", help="where the parts are served from (default: this repository's builder-parts release)")
p.add_argument("--reuse", help="an earlier hookshot-setup.txt: the parts the package has no folder for, from it")
a = p.parse_args()
root = os.path.abspath(a.package)
if not os.path.isfile(os.path.join(root, "HookshotBuilder.exe")):
sys.exit("not a Hookshot Builder package folder: %s" % root)
# (the parts live in one release of their own, named by their checksum: a part that did not change is not uploaded again, and a
# release itself holds only HookshotBuilder.exe (the setup) and the list it reads)
base = (a.base or "https://github.com/NotRexed/Hookshot/releases/download/builder-parts").rstrip("/")
os.makedirs(a.out, exist_ok=True)
with open(os.path.join(root, "tools", "version.txt"), "w", newline="\n") as f:
f.write(a.tag + "\n")
top = sorted(os.listdir(root))
builder_files = [n for n in top if os.path.isfile(os.path.join(root, n)) and
(n.startswith("HookshotBuilder") or n in ("LICENSE", "README.txt"))] # (HookshotBuilder-Setup.exe too: Repair runs it)
runtime_files = [n for n in top if os.path.isfile(os.path.join(root, n)) and n not in builder_files]
runtime_dirs = [n for n in top if os.path.isdir(os.path.join(root, n)) and n != "tools"]
tools = sorted(os.listdir(os.path.join(root, "tools")))
parts = {
"app": builder_files + [n for t in tools if t not in TOOLS_PARTS and t != "llvm-mingw"
for n in (["tools/" + t] if os.path.isfile(os.path.join(root, "tools", t)) else files_under(root, "tools/" + t))],
"dotnet": runtime_files + [n for d in runtime_dirs for n in files_under(root, d)],
}
for t in TOOLS_PARTS:
if os.path.isdir(os.path.join(root, "tools", t)):
parts[t] = files_under(root, "tools/" + t)
reuse = {}
if a.reuse:
with open(a.reuse, encoding="utf-8") as f:
for line in f.read().splitlines():
w = line.split()
if len(w) == 7 and w[0] == "part" and w[6].rsplit("/", 1)[-1].startswith("HookshotBuilder-"):
reuse[w[6].rsplit("/", 1)[-1][len("HookshotBuilder-"):].rsplit("-", 1)[0]] = line
lines = ["tag %s" % a.tag]
for name in ("app", "dotnet") + TOOLS_PARTS:
if not parts.get(name):
if a.reuse and name in TOOLS_PARTS:
if name not in reuse:
sys.exit("no tools/%s in the package and no %s part in %s" % (name, name, a.reuse))
lines.append(reuse[name])
print("%-12s (as in %s)" % (name, os.path.basename(a.reuse)), flush=True)
continue
zp = os.path.join(a.out, "HookshotBuilder-%s.zip" % name)
sha, size = write_zip(zp, root, parts[name])
named = os.path.join(a.out, "HookshotBuilder-%s-%s.zip" % (name, sha[:12]))
os.replace(zp, named)
zp = named
lines.append("part %s %d . - - %s/%s" % (sha, size, base, os.path.basename(zp)))
print("%-12s %5d files %5d MB %s" % (name, len(parts[name]), size >> 20, sha[:16]), flush=True)
lines.append("part %s %d HookshotBuilder/tools %s llvm-mingw %s" % LLVM)
lines.append("run HookshotBuilder/HookshotBuilder.exe")
with open(os.path.join(a.out, "hookshot-setup.txt"), "w", newline="\n") as f:
f.write("\n".join(lines) + "\n")
if __name__ == "__main__":
main()

207
scripts/ngx_sdk.py Normal file
View File

@@ -0,0 +1,207 @@
"""NVIDIA's DLSS SDK files for a local build with NVIDIA DLSS Super Resolution (HOOKSHOT_NGX_SDK, gpu/vk/vk_dlss.cpp).
NVIDIA's files are not part of Hookshot and are never committed or bundled: they are under NVIDIA's own license (the DLSS
SDK's LICENSE.txt, fetched beside them; read it before using them). This fetches them from NVIDIA's DLSS repository at one
pinned release, each file checked against its SHA-256, into <sdk folder>:
- LICENSE.txt and the SDK's headers (include/), the same for every platform;
- the library the build links (lib/): libnvsdk_ngx.a on Linux, nvsdk_ngx_s.lib (the /MT one) on Windows;
- the DLSS runtime the driver loads (runtime/): libnvidia-ngx-dlss.so.<version> on Linux, nvngx_dlss.dll on Windows.
The NGX core itself comes with NVIDIA's driver (libnvidia-ngx.so on Linux; _nvngx.dll on Windows, which is loaded from the
driver's own folder and is never to be put beside the game).
Usage: python -I scripts/ngx_sdk.py [--accept-nvidia-license] [--platform linux|windows] [--game <game folder>]
[--runtime-dir <folder>] [--build-only] [<sdk folder>]
<sdk folder>: default build-linux/ngx-sdk on Linux, work/ngx-sdk on Windows
--game: the DLSS runtime copied into <game folder>/ngx too, where the game looks first (nx2/make_game.py does the same for
the folder it builds when HOOKSHOT_NGX_SDK is set)
--runtime-dir: the runtime copied into that folder instead (Linux: $XDG_DATA_HOME/hookshot/ngx; Windows:
%LOCALAPPDATA%\\Hookshot\\ngx. RX_VK_DLSS_DIR names another when the game runs)
--build-only: only what the build reads, not DLSS's runtime (CI's link check)
--accept-nvidia-license: NVIDIA's license accepted without the question (scripts, the Hookshot Builder's DLSS tick, CI)
Then build with HOOKSHOT_NGX_SDK=<sdk folder>.
"""
import argparse
import hashlib
import os
import shutil
import sys
import urllib.request
COMMIT = "9981c6350dc58c0ce21f024fd65e31e851c31689" # (v310.9.16)
BASE = "https://raw.githubusercontent.com/NVIDIA/DLSS/" + COMMIT
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
LICENSE_URL = "https://github.com/NVIDIA/DLSS/blob/%s/LICENSE.txt" % COMMIT
# (path in NVIDIA's repository, its SHA-256): what every platform's build reads
COMMON = [
("LICENSE.txt", "d4216e39ebef5f9b50a6712ebb37beeb5379862a67733a9999c651f21592aaf0"),
("include/nvsdk_ngx_defs_dlssd.h", "2e97fe1595b1334f71b51724f2cb67496e8c296a7a2eec5606da53055882e716"),
("include/nvsdk_ngx_defs_dlssg.h", "c29765ab9e0125f9d724ae9811cc2abe03738d17486ec8eee80d78a1504549e1"),
("include/nvsdk_ngx_defs.h", "ea23f33497cd274860d1c25a97644fce807dcb0037c594547203343103fad03e"),
("include/nvsdk_ngx_defs_vk.h", "0a24d0861ace7d6b9362a67b7f08bea1b33ea123c5ce730b19133de8a891d031"),
("include/nvsdk_ngx.h", "dc38e7467cf415379c9d12ae1b6e4a494c453ed92720fb53e92aecb523e7b848"),
("include/nvsdk_ngx_helpers_cuda.h", "f2851a76107bdf4fbcb7b261f3c35573c312d8ecb4bc18fb02d9e2cec7705042"),
("include/nvsdk_ngx_helpers_d3d.h", "ec75224f36ed6580baaf250fa83620405ad81a1d98c2c436654a0c9cf6a6b8ba"),
("include/nvsdk_ngx_helpers_dlssd_cuda.h", "7e35cde214d905346a487b9a28c05d35e1c55b480e237fe3de69580a7e332730"),
("include/nvsdk_ngx_helpers_dlssd_d3d.h", "6df9d02c6f47feee3afc8d12340a9f554a7ddce4ddacb1b6a13a42b26c2c6293"),
("include/nvsdk_ngx_helpers_dlssd.h", "a25fdda925b479dfe869593aeba311ef08793da54e0391f2fe2fe756cbef5532"),
("include/nvsdk_ngx_helpers_dlssd_vk.h", "11df8fbda532a97fe867c8ef2b40058b49b7898183710f3468e55b1eedb8f40b"),
("include/nvsdk_ngx_helpers_dlssg_d3d.h", "6335ebcb95de39a361e87fe0257d6ce0723e5d8deaa15db3b35afacf2b3f4220"),
("include/nvsdk_ngx_helpers_dlssg.h", "6881d21419c40c78ec948b326902ece09fc112b0223144d51c4905f3fced7a1f"),
("include/nvsdk_ngx_helpers_dlssg_vk.h", "b371d90b9f4372b9a415e68a0afbdc2f31b7b552e219a9637d848749bc965316"),
("include/nvsdk_ngx_helpers.h", "5bcbadfe7478b802cf6d3aca4dc5ddd7d0889b99726e69c63f9e9bd555f44471"),
("include/nvsdk_ngx_helpers_vk.h", "c192bff72138f12f770db48e12c1d8f712dfc81ea0e8e055ebb27d8dd1b31623"),
("include/nvsdk_ngx_loader.h", "abdf5f66a692620538357d29fdf78e3e2a677ee8140b7a638962ecb766585e75"),
("include/nvsdk_ngx_params_dlssd.h", "d41e3ba8f1d72583b55273092c99bf426dc4a0d46de827229e25d5b1325a370e"),
("include/nvsdk_ngx_params_dlssg.h", "c24fa5aa68ffa2808c828b75cfa32843f1606f80a145f84055ac450d7e6c74ad"),
("include/nvsdk_ngx_params.h", "943bc8cc5cdae03b6303016fbad3183636f2335ae27a2d18776798c3b4efabbc"),
("include/nvsdk_ngx_standalone_common.h", "38413c3338b0aea0202c07da95d426bfee7080a42913823fa2d9dd585b087b53"),
("include/nvsdk_ngx_standalone_cuda.h", "b23436ed5040051ac8e81554c9af080f2463c941435bbc3b82222e5431a09499"),
("include/nvsdk_ngx_vk.h", "2d364ce7132881eb669e9498fd570d74cba563b1fbebc82c235f8e1ad2dd8b6d"),
]
# each platform's library (into lib/) and DLSS runtime (into runtime/)
PLATFORMS = {
"linux": {
"library": ("lib/Linux_x86_64/libnvsdk_ngx.a", "d3c50bb0d9624fc35e502a8e929b8fc1e2b505017ebdb84c0d5b627e11a235df"),
"runtime": ("lib/Linux_x86_64/rel/libnvidia-ngx-dlss.so.310.9.16",
"30982c937afed85c96b8d313323de9a650a2eec6de1bb0f714b21750f4ce98f6"),
},
"windows": {
# (the static NGX loader built against the static C runtime (/MT); llvm-mingw links it with gpu/vk/vk_ngx_msvc_rt.cpp)
"library": ("lib/Windows_x86_64/x64/nvsdk_ngx_s.lib", "7b9a5239c1145af5f645ab472aa066f9ea79db36ac819562312aae69d65f3765"),
"runtime": ("lib/Windows_x86_64/rel/nvngx_dlss.dll", "d00b5b0ab29c3ca101b70c3a5ecb3a01b7ef33766a3383a28a18623083e6123e"),
},
}
NOTICE = """\
NVIDIA DLSS: this downloads NVIDIA's DLSS SDK (release v310.9.16) from github.com/NVIDIA/DLSS.
NVIDIA's files are NVIDIA's, not Hookshot's, and are under NVIDIA's own license, the NVIDIA RTX SDKs license:
%s
(saved as LICENSE.txt beside the files). Using them means accepting it. Hookshot never includes or redistributes them:
they stay on this PC, in the SDK folder and the game folder's ngx folder. Never share a game folder built with them.""" % LICENSE_URL
def host_platform():
"""The platform a build here is for: Windows on Windows or with RX_TARGET=windows (a cross build), else Linux"""
return "windows" if os.name == "nt" or os.environ.get("RX_TARGET", "").lower() == "windows" else "linux"
def destination(path):
"""Where a file of NVIDIA's repository goes in the SDK folder (a relative path)"""
if path.startswith("lib/"):
return ("runtime/" if "/rel/" in path else "lib/") + path.rsplit("/", 1)[1]
return path
def files(platform, runtime=True):
"""[(path in NVIDIA's repository, SHA-256, path in the SDK folder)] for one platform; runtime=False: what a build reads only"""
spec = PLATFORMS[platform]
return [(p, s, destination(p)) for p, s in COMMON + [spec["library"]] + ([spec["runtime"]] if runtime else [])]
def library(sdk, platform):
"""The library a build with this SDK folder links, for that platform"""
return os.path.join(sdk, *destination(PLATFORMS[platform]["library"][0]).split("/"))
def runtime(sdk, platform):
"""The DLSS runtime in this SDK folder, for that platform"""
return os.path.join(sdk, *destination(PLATFORMS[platform]["runtime"][0]).split("/"))
def runtime_name(platform):
return PLATFORMS[platform]["runtime"][0].rsplit("/", 1)[1]
def sha256(path):
h = hashlib.sha256()
with open(path, "rb") as f:
for block in iter(lambda: f.read(1 << 20), b""):
h.update(block)
return h.hexdigest()
TIMEOUT = 60 # (seconds without an answer, or between two reads, before a download gives up: a stalled one ends the build)
def fetch_one(path, digest, dst, opener=urllib.request.urlopen):
"""One file into dst, unless it is there already with that SHA-256. The download goes to dst.part and is moved in only
when its SHA-256 is the expected one; otherwise it is deleted and ValueError says which"""
if os.path.isfile(dst) and sha256(dst) == digest:
return False
os.makedirs(os.path.dirname(dst) or ".", exist_ok=True)
tmp = dst + ".part"
try:
with opener(BASE + "/" + path, timeout=TIMEOUT) as r, open(tmp, "wb") as f:
shutil.copyfileobj(r, f, 1 << 20)
if sha256(tmp) != digest:
raise ValueError("ngx-sdk: %s is not the file expected (SHA-256): not used" % path)
os.replace(tmp, dst)
finally:
if os.path.exists(tmp):
os.remove(tmp)
return True
def fetch(sdk, platform, opener=urllib.request.urlopen, say=print, runtime=True):
"""Every file of that platform into the SDK folder (runtime=False: not DLSS's runtime)"""
for path, digest, rel in files(platform, runtime):
if fetch_one(path, digest, os.path.join(sdk, *rel.split("/")), opener):
say("ngx-sdk: %s" % rel)
def install_runtime(sdk, platform, folder):
"""The SDK folder's DLSS runtime copied into folder (made when missing): where the game finds it"""
src = runtime(sdk, platform)
dst = os.path.join(folder, runtime_name(platform))
os.makedirs(folder, exist_ok=True)
if not (os.path.isfile(dst) and os.path.getsize(dst) == os.path.getsize(src) and sha256(dst) == sha256(src)):
shutil.copyfile(src, dst + ".part")
os.replace(dst + ".part", dst)
return dst
def accepted(flag, ask=input, interactive=None):
"""NVIDIA's license, accepted: the flag, or a typed yes when there is someone to ask"""
if flag:
return True
if not (sys.stdin.isatty() if interactive is None else interactive):
return False
try:
return ask("Do you accept NVIDIA's license? Type yes to go on: ").strip().lower() in ("y", "yes")
except EOFError:
return False
def main(argv=None):
p = argparse.ArgumentParser(description=__doc__.split("\n")[0])
p.add_argument("sdk", nargs="?")
p.add_argument("--platform", choices=sorted(PLATFORMS), default=host_platform())
p.add_argument("--game", help="a game folder: the DLSS runtime into its ngx folder")
p.add_argument("--runtime-dir", help="the DLSS runtime into this folder")
p.add_argument("--accept-nvidia-license", action="store_true")
p.add_argument("--build-only", action="store_true", help="not DLSS's runtime: only what the build reads")
a = p.parse_args(argv)
if a.build_only and (a.game or a.runtime_dir):
p.error("--build-only fetches no runtime for --game or --runtime-dir")
sdk = os.path.abspath(a.sdk or os.path.join(ROOT, "build-linux" if a.platform == "linux" else "work", "ngx-sdk"))
print(NOTICE, flush=True)
if not accepted(a.accept_nvidia_license):
print("ngx-sdk: NVIDIA's license was not accepted (--accept-nvidia-license, or yes when asked): nothing downloaded",
file=sys.stderr, flush=True)
return 2
try:
fetch(sdk, a.platform, runtime=not a.build_only)
except (OSError, ValueError) as e:
print("ngx-sdk: %s" % e, file=sys.stderr, flush=True)
return 1
print("ngx-sdk: the SDK's headers and library in %s (NVIDIA's license: %s)" % (sdk, os.path.join(sdk, "LICENSE.txt")),
flush=True)
for folder in ([os.path.join(a.game, "ngx")] if a.game else []) + ([a.runtime_dir] if a.runtime_dir else []):
print("ngx-sdk: the DLSS runtime in %s" % install_runtime(sdk, a.platform, folder), flush=True)
print("ngx-sdk: build with HOOKSHOT_NGX_SDK=%s" % sdk, flush=True)
return 0
if __name__ == "__main__":
sys.exit(main())

409
scripts/oot_saves.py Normal file
View File

@@ -0,0 +1,409 @@
"""Copy Ocarina of Time saves between Hookshot and the emulators and ports that run the game: they all keep the same
files, so a copy is all it takes.
python -I scripts/oot_saves.py list
python -I scripts/oot_saves.py import <name> [--profile ID] [--keep-settings] [--dry-run] [--force]
python -I scripts/oot_saves.py export <name> [--profile ID] [--keep-settings] [--dry-run] [--force]
python -I scripts/oot_saves.py import --from DIR (a save from any folder into Hookshot)
python -I scripts/oot_saves.py export --to DIR (Hookshot's save into any folder)
import copies into Hookshot, export copies out of it. <name> is one of EMULATORS below: lichii, frizti, orchid,
pokedinner, eden, eglantine or nx2navi (to-lichii and to-hookshot still work: they are export lichii and import
lichii). --from/--to name the other side's folder instead of where it is looked for: the save folder itself (the one
with account.sav and slot_*), or a folder above it (an emulator's data folder, a portable user folder, NX2navi's game
folder), which is searched for the game's saves. --hookshot DIR does the same for Hookshot's side: its data folder
(Linux: $XDG_DATA_HOME/hookshot, default ~/.local/share/hookshot) or, on Windows, its game folder (the one with
image.bin and user\\), which has to be given there. The side being overwritten is copied first to
<backup>/<import or export>-<name>-<YYYYmmdd-HHMMSS> (Linux: $XDG_DATA_HOME/oot-save-backups, Windows:
%LOCALAPPDATA%\\Hookshot\\oot-save-backups; --backup-dir changes it). Close every game first: --force skips that check.
Standard library only; the guide is docs/saves.md.
"""
import argparse
import collections
import datetime
import os
import re
import shutil
import stat
import subprocess
import sys
TITLE = "0400486024DEC000"
NO_ACCOUNT = "0000000000000000"
WINDOWS = os.name == "nt"
Program = collections.namedtuple("Program", "name display folders processes aliases")
# Everything that runs the game and where it keeps its data, one entry each. folders: per OS, where to look, in order
# ({data} = $XDG_DATA_HOME or ~/.local/share, {appdata} = %APPDATA%, {localappdata} = %LOCALAPPDATA%, {home} = the
# home folder); the saves are then found under it by LAYOUTS. processes: lower case, without .exe, for the
# running-game check. The sources are in docs/saves.md.
EDEN = {"linux": ["{data}/eden"], "windows": ["{appdata}/eden"]}
EMULATORS = [
Program("lichii", "Lichii", EDEN, ["lichii"], []),
Program("frizti", "Frizti (Fritchii)", EDEN, ["lichii", "fritchii"], ["fritchii"]),
Program("orchid", "Orchid", EDEN, ["orchid"], []),
Program("pokedinner", "PokeDinner", EDEN, ["eglantine"], []),
Program("eden", "Eden", EDEN, ["eden"], []),
Program("eglantine", "Eglantine", {"linux": ["{data}/eglantine"], "windows": ["{appdata}/eglantine"]},
["eglantine"], []),
Program("nx2navi", "NX2navi", {"linux": ["{home}/Games/Ocarina"],
"windows": ["{localappdata}/Programs/NX2navi/game", "C:/Games/Ocarina"]},
["nx2navi", "ocarina"], []),
]
HOOKSHOT = Program("hookshot", "Hookshot", {"linux": ["{data}/hookshot"], "windows": []}, ["hookshot"], [])
FOLDER = Program("folder", "the folder", {"linux": [], "windows": []}, [], [])
# Where the save folders sit under a data folder. "<id>" is one folder per user profile (its name is the id shown).
# A folder deeper down the same path is accepted too. The first layout that finds a profile wins.
LAYOUTS = (
("nand", "user", "save", "account", "<id>", TITLE, "0"), # Eden's newer layout: used first when it exists
("nand", "user", "save", NO_ACCOUNT, "<id>", TITLE), # Eden and its forks (Lichii, Eglantine, ...)
("profile", "save", "<id>"), # NX2navi's game folder
("user", "save", "<id>"), # Hookshot's game folder on Windows
("save", "<id>"), # Hookshot's data folder
)
class Fail(Exception):
pass
def data_home():
return os.environ.get("XDG_DATA_HOME") or os.path.join(os.path.expanduser("~"), ".local", "share")
def default_backup_dir():
if WINDOWS:
base = os.environ.get("LOCALAPPDATA") or os.path.join(os.path.expanduser("~"), "AppData", "Local")
return os.path.join(base, "Hookshot", "oot-save-backups")
return os.path.join(data_home(), "oot-save-backups")
def folders(program):
"""program's data folders on this OS, in the order they are tried."""
home = os.path.expanduser("~")
values = {"data": data_home(), "home": home,
"appdata": os.environ.get("APPDATA") or os.path.join(home, "AppData", "Roaming"),
"localappdata": os.environ.get("LOCALAPPDATA") or os.path.join(home, "AppData", "Local")}
return [os.path.normpath(t.format(**values)) for t in program.folders["windows" if WINDOWS else "linux"]]
def subdirs(path):
try:
return sorted(n for n in os.listdir(path) if os.path.isdir(os.path.join(path, n)))
except OSError:
return []
def slots(path):
return [n for n in subdirs(path) if n.startswith("slot_")]
def is_save_folder(path):
return os.path.isfile(os.path.join(path, "account.sav")) or bool(slots(path))
def walk(root, parts, key):
if not parts:
yield key, root
return
for n in subdirs(root):
if parts[0] == "<id>":
yield from walk(os.path.join(root, n), parts[1:], n)
elif n.lower() == parts[0].lower():
yield from walk(os.path.join(root, n), parts[1:], key)
def find_saves(root):
"""The game's save folders under root, as {profile id: path}; root may be the save folder itself."""
norm = os.path.normpath(os.path.abspath(root))
title_folder = os.path.basename(norm).upper() == TITLE
if is_save_folder(norm) or title_folder and "0" not in subdirs(norm):
key = norm
while os.path.basename(key).upper() in (TITLE, "0") and os.path.dirname(key) != key:
key = os.path.dirname(key)
return {os.path.basename(key) or norm: norm}
found = {}
for layout in LAYOUTS:
idpos = layout.index("<id>")
for start in range(len(layout)):
if TITLE in layout and TITLE not in layout[start:] and not title_folder:
continue # (below the title id folder, only the title id folder itself is a place to start)
key = None
if start > idpos: # (root is at or below the <id> folder: the id is a folder above it)
up = norm
for _ in range(start - idpos - 1):
up = os.path.dirname(up)
key = os.path.basename(up)
for k, path in walk(norm, layout[start:], key):
# A folder named by the layout (the title id folder) counts as soon as it is there: the game may not
# have saved yet. A profile folder counts only when it holds a save (Eden's profile folders do not).
if (layout[-1] != "<id>" or is_save_folder(path)) and k not in found and path not in found.values():
found[k] = path
return found
def locate(program, override, flag):
"""(folder, {profile id: save folder}) for program: override when given, else the first of its folders with saves."""
if override is not None:
if not os.path.isdir(override):
raise Fail("no folder at %s" % override)
found = find_saves(override)
if not found:
raise Fail("no Ocarina of Time save under %s (a folder with account.sav and slot_*, or %s under an "
"emulator's nand\\user\\save): start the game there once and save" % (override, TITLE))
return override, found
tried = folders(program)
for d in tried:
found = find_saves(d) if os.path.isdir(d) else {}
if found:
return d, found
if program is HOOKSHOT and not tried:
raise Fail("Hookshot's saves are in the user folder of your game folder on Windows: pass the game folder with "
"--hookshot (for example --hookshot C:\\Games\\Hookshot)")
raise Fail("no %s save found (looked in %s): start the game in %s once and save, or pass %s <folder>"
% (program.display, ", ".join(tried) or "no default folder", program.display, flag))
def pick(found, name, flag, wanted=None):
if wanted is not None:
for k, v in found.items():
if k.lower() == wanted.lower():
return k, v
raise Fail("%s has no profile %s; it has: %s" % (name, wanted, ", ".join(sorted(found))))
if len(found) > 1:
lines = "".join("\n %s %s" % (k, found[k]) for k in sorted(found))
raise Fail("%s has Ocarina of Time saves in %d profiles, so pick one with %s:%s" % (name, len(found), flag, lines))
return next(iter(found.items()))
def pending_commit(save):
"""NX2navi writes a save to profile/save-tx first and moves it into profile/save/<account> after: while something is
waiting there, that is the save the game loads next, not the account folder."""
parent = os.path.dirname(os.path.normpath(save))
tx = os.path.join(os.path.dirname(parent), "save-tx")
if os.path.basename(parent).lower() == "save" and os.path.isdir(tx):
for _, _, files in os.walk(tx):
if files:
return tx
return None
def process_names():
"""{process name: what it belongs to} for every program in the tables."""
names = collections.OrderedDict()
for program in [HOOKSHOT] + EMULATORS:
for p in program.processes:
names[p] = names[p] + " or " + program.display if p in names else program.display
return names
def running_games():
"""What is running among Hookshot and the emulators in EMULATORS."""
known = process_names()
names = set()
if WINDOWS:
out = subprocess.run(["tasklist", "/FO", "CSV", "/NH"], capture_output=True, text=True).stdout
for line in out.splitlines():
image = line.split('","')[0].strip('"').lower()
if image.endswith(".exe") and image[:-4] in known:
names.add(known[image[:-4]])
return sorted(names)
appimage = re.compile(r"^(%s)[-_.].*\.appimage$" % "|".join(map(re.escape, known)))
for pid in filter(str.isdigit, os.listdir("/proc")):
try:
with open("/proc/%s/cmdline" % pid, "rb") as f:
argv0 = f.read().split(b"\0")[0].decode("utf-8", "replace")
with open("/proc/%s/comm" % pid, encoding="utf-8", errors="replace") as f:
comm = f.read().strip()
except OSError:
continue
for candidate in (comm, os.path.basename(argv0)):
low = candidate.lower()
m = appimage.match(low)
if low in known:
names.add(known[low])
elif m:
names.add(known[m.group(1)])
return sorted(names)
def describe(path):
out = []
for s in slots(path):
p = os.path.join(path, s, "progress.sav")
if os.path.isfile(p):
when = datetime.datetime.fromtimestamp(os.path.getmtime(p)).strftime("%Y-%m-%d %H:%M")
out.append("%s (saved %s)" % (s, when))
else:
out.append(s)
return out
def show(title, folder, found):
print("%s: %s" % (title, folder))
for k in sorted(found):
print(" %s %s" % (k, found[k]))
for line in describe(found[k]) or ["(no slots)"]:
print(" " + line)
if pending_commit(found[k]):
print(" (a save is still waiting in %s: start and quit NX2navi before copying)" % pending_commit(found[k]))
def list_all(args):
try:
show("Hookshot", *locate(HOOKSHOT, args.hookshot, "--hookshot"))
except Fail as e:
print("Hookshot: %s" % e)
groups = collections.OrderedDict()
for program in EMULATORS:
groups.setdefault(tuple(folders(program)), []).append(program.display)
for tried, displays in groups.items():
title = " / ".join(displays) + (" (one shared folder)" if len(displays) > 1 else "")
shown = False
for d in tried:
found = find_saves(d) if os.path.isdir(d) else {}
if found:
show(title, d, found)
shown = True
if not shown:
print("%s: nothing in %s" % (title, ", ".join(tried) or "(no default folder)"))
if args.folder:
show("Folder", *locate(FOLDER, args.folder, "--from"))
def refuse_links(path):
"""A save from someone else is only files and folders: a link in it would copy what it points to (any file or folder
of this PC) into the save"""
def link(p):
# (any reparse point on Windows: a junction is not a symbolic link, and os.path.isjunction is only in 3.12 and
# newer; copytree walks into a junction as into a folder)
try:
st = os.lstat(p)
except OSError:
return False
return stat.S_ISLNK(st.st_mode) or bool(getattr(st, "st_file_attributes", 0) & 0x400)
if link(path):
raise Fail("%s is a link: a save holds only files and folders, so nothing was copied" % path)
for root, dirs, files in os.walk(path):
for n in dirs + files:
if link(os.path.join(root, n)):
raise Fail("%s is a link: a save holds only files and folders, so nothing was copied" % os.path.join(root, n))
def copy_saves(src, dst, backup, keep_settings, dry_run):
target_path = os.path.normcase(os.path.realpath(dst))
backup_path = os.path.normcase(os.path.realpath(backup))
try:
inside_target = os.path.commonpath((target_path, backup_path)) == target_path
except ValueError:
inside_target = False # Windows paths can be on different drives.
if inside_target:
raise Fail("choose a --backup-dir outside the target save folder (%s)" % dst)
names = slots(src)
if not names:
raise Fail("%s has no slot_* folders: nothing to copy" % src)
for s in names:
refuse_links(os.path.join(src, s))
for f in ("account.sav", "option.sav"):
if os.path.lexists(os.path.join(src, f)):
refuse_links(os.path.join(src, f))
files = [f for f in ("account.sav", "option.sav") if os.path.isfile(os.path.join(src, f))]
if keep_settings and "option.sav" in files:
files.remove("option.sav")
print("%s %s -> %s" % ("Would back up" if dry_run else "Backing up", dst, backup))
print("%s %d slot(s) (%s) and %s from %s to %s" % ("Would copy" if dry_run else "Copying", len(names), ", ".join(names),
", ".join(files) or "no loose files", src, dst))
if keep_settings:
print("option.sav kept as it is on the target")
if dry_run:
print("Dry run: nothing was changed")
return
os.makedirs(os.path.dirname(backup), exist_ok=True)
shutil.copytree(dst, backup)
for s in slots(dst):
shutil.rmtree(os.path.join(dst, s))
for s in names: # (links were refused above; copied as links, never followed, all the same)
shutil.copytree(os.path.join(src, s), os.path.join(dst, s), symlinks=True)
for f in files:
shutil.copy2(os.path.join(src, f), os.path.join(dst, f), follow_symlinks=False)
print("Done. Backup of the old %s in %s" % (dst, backup))
def program_named(name):
for program in EMULATORS:
if name.lower() in [program.name] + program.aliases:
return program
raise Fail("unknown emulator %s: use one of %s, or --from/--to <folder>"
% (name, ", ".join(p.name for p in EMULATORS)))
def run(args):
if args.command == "list":
list_all(args)
return
other = program_named(args.name) if args.name else FOLDER
if other is FOLDER and not args.folder:
raise Fail("%s what? Name the emulator (%s) or the folder (%s <folder>)"
% (args.command, ", ".join(p.name for p in EMULATORS), "--from" if args.command == "import" else "--to"))
flag = "--from" if args.command == "import" else "--to"
_, hs = locate(HOOKSHOT, args.hookshot, "--hookshot")
_, em = locate(other, args.folder, flag)
hs_id, hs_path = pick(hs, "Hookshot", "--hookshot <the user id folder>")
em_id, em_path = pick(em, other.display, "--profile <id>", args.profile)
src, dst = (em_path, hs_path) if args.command == "import" else (hs_path, em_path)
if os.path.realpath(src) == os.path.realpath(dst):
raise Fail("%s is both the source and the target" % src)
if not args.force:
tx = pending_commit(em_path)
if tx:
raise Fail("NX2navi still has a save waiting in %s, which it loads instead of %s: start NX2navi and quit "
"it from the game, then copy (or pass --force)" % (tx, em_path))
if not args.dry_run and not args.force:
busy = running_games()
if busy:
raise Fail("close %s first (or pass --force): a running game can write its save again and undo the copy"
% " and ".join(busy))
stamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
backup = base = os.path.join(args.backup_dir or default_backup_dir(), "%s-%s-%s" % (args.command, other.name, stamp))
n = 1
while os.path.exists(backup): # (two copies in the same second)
n += 1
backup = "%s-%d" % (base, n)
copy_saves(src, dst, backup, args.keep_settings, args.dry_run)
def main(argv=None):
p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0], prog="oot_saves.py")
p.add_argument("command", choices=["list", "import", "export", "to-lichii", "to-hookshot"],
help="import: into Hookshot; export: out of Hookshot; to-lichii, to-hookshot: export/import lichii")
p.add_argument("name", nargs="?", help="the emulator: %s" % ", ".join(e.name for e in EMULATORS))
p.add_argument("--from", dest="from_dir", metavar="DIR", help="import from this folder (a save, or one to search)")
p.add_argument("--to", dest="to_dir", metavar="DIR", help="export to this folder (a save, or one to search)")
p.add_argument("--lichii", metavar="DIR", help=argparse.SUPPRESS) # (the old name of --from/--to)
p.add_argument("--hookshot", metavar="DIR", help="Hookshot's data folder (Windows: the game folder)")
p.add_argument("--profile", metavar="ID", help="the emulator's profile, when several have saves")
p.add_argument("--backup-dir", metavar="DIR", help="where the backup of the overwritten side goes")
p.add_argument("--keep-settings", action="store_true", help="leave the target's option.sav (the game's settings) alone")
p.add_argument("--dry-run", action="store_true", help="say what would happen and change nothing")
p.add_argument("--force", action="store_true",
help="copy even if a game seems to be running, or NX2navi has a save waiting")
args = p.parse_args(argv)
if args.command in ("to-lichii", "to-hookshot"):
if args.name:
p.error("%s takes no name" % args.command)
args.command, args.name = ("export" if args.command == "to-lichii" else "import"), "lichii"
if args.command == "import" and args.to_dir or args.command == "export" and args.from_dir:
p.error("import reads --from, export writes --to")
args.folder = args.from_dir or args.to_dir or args.lichii
try:
run(args)
except Fail as e:
print("error: %s" % e, file=sys.stderr)
return 1
return 0
if __name__ == "__main__":
sys.exit(main())

90
scripts/probe.py Normal file
View File

@@ -0,0 +1,90 @@
"""Which pass made a flash: reads a probe_<frame>.txt (vk_draw.cpp RX_VK_PROBE) and aligns the frames before, at and
after it by (kind, program, target, nth use), then prints the entries of the flash frame in order whose colour is far
from both neighbours', decoded by format. The comparison skips decoded NaNs.
The first reported entry is the pass that went wrong.
python -I scripts/probe.py work/runs/<run>/probe_<frame>.txt [ratio 4] [lines 25]
"""
import collections, math, struct, sys
def half(h):
return struct.unpack("<e", struct.pack("<H", h))[0]
def uf(v, mbits):
e, m = v >> mbits, v & ((1 << mbits) - 1)
return (m / (1 << mbits)) * 2.0 ** -14 if e == 0 else 1e30 if e == 31 else (1 + m / (1 << mbits)) * 2.0 ** (e - 15)
def decode(fmt, w):
if fmt == 122: # B10G11R11
return [uf(w[0] & 2047, 6), uf((w[0] >> 11) & 2047, 6), uf(w[0] >> 22, 5)]
if fmt == 123: # E5B9G9R9
s = 2.0 ** ((w[0] >> 27) - 24)
return [(w[0] & 511) * s, ((w[0] >> 9) & 511) * s, ((w[0] >> 18) & 511) * s]
if fmt == 97: # R16G16B16A16_SFLOAT
return [half(w[0] & 0xffff), half(w[0] >> 16), half(w[1] & 0xffff)]
if fmt == 83: # R16G16_SFLOAT
return [half(w[0] & 0xffff), half(w[0] >> 16)]
if fmt == 76: # R16_SFLOAT
return [half(w[0] & 0xffff)]
if fmt == 100: # R32_SFLOAT
return list(struct.unpack("<f", struct.pack("<I", w[0])))
if fmt == 109: # R32G32B32A32_SFLOAT
return list(struct.unpack("<4f", struct.pack("<4I", *w)))
return [(w[0] >> k) & 255 for k in (0, 8, 16, 24)]
def key_rows(rows):
seen, out = collections.Counter(), {}
for r in rows:
k = (r[2], r[3], r[4])
out[k + (seen[k],)] = r
seen[k] += 1
return out
frames = collections.OrderedDict()
with open(sys.argv[1], encoding="utf-8") as capture:
for line_number, line in enumerate(capture, 1):
r = line.split()
if not r:
continue
if len(r) < 11:
raise SystemExit(f"Probe capture line {line_number} is incomplete. Capture a new probe_*.txt file.")
frames.setdefault(r[0], []).append(r)
if len(frames) != 3:
raise SystemExit(f"Probe capture expected 3 frames, found {len(frames)}. "
"Capture a new probe_*.txt file with the frame before, the flash, and the frame after.")
a, b, c = [key_rows(v) for v in frames.values()]
ratio = float(sys.argv[2]) if len(sys.argv) > 2 else 4.0
lines = int(sys.argv[3]) if len(sys.argv) > 3 else 25
def val(r):
return decode(int(r[6]), [int(x, 16) for x in r[7:11]])
def far(x, y):
for p, q in zip(x, y):
if math.isnan(p) or math.isnan(q):
continue
hi, lo = max(abs(p), abs(q)), min(abs(p), abs(q))
if hi > 1e-3 and hi > lo * ratio:
return True
return False
print("frames", list(frames), {k: len(v) for k, v in frames.items()}, "unmatched at the flash:", sum(1 for k in b if k not in a or k not in c))
told = 0
for k, r in b.items():
if k not in a or k not in c:
continue
va, vb, vc = val(a[k]), val(r), val(c[k])
if far(va, vb) and far(vc, vb):
print("%4s %s prog %s %s %s f%s: %s | %s | %s" % (r[1], r[2], r[3][:8], r[4], r[5], r[6], ["%.4g" % x for x in va], ["%.4g" % x for x in vb], ["%.4g" % x for x in vc]))
told += 1
if told >= lines:
break
print("%d entries far from both neighbours" % told)

151
scripts/prof.py Normal file
View File

@@ -0,0 +1,151 @@
"""Offline profile of a thread sampled with HOS_SAMPLE + HOS_SAMPLE_FILE (host/mkw_glue.c, mkw-recomp work/prof.py): each line is a stack of
module+offset frames, leaf first. Frames in hookshot_vk.dll / hookshot.exe are named from the linker maps beside them;
on Linux, frames in hookshot_vk.so / hookshot from the binaries' own symbols (nm). RX_MAPS=<folder>: another build's
maps or binaries (on Linux, the game folder).
python -I scripts/prof.py work/runs/<run>/stacks.txt [top] [skip] [only]
(skip: the first fraction of the samples, e.g. 0.5; only: samples with that function in their stack, e.g.
bench_replay)
Prints the functions by inclusive share (in the stack at all) and by self share (the leaf), and the leaf modules.
"""
import bisect, collections, itertools, os, re, stat, subprocess, sys
from contextlib import ExitStack
from tempfile import TemporaryFile
BUILD = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "work", "sa", "build")
def load_map(path, base):
syms = []
pat = re.compile(r"^\s*0001:[0-9a-f]+\s+(\S+)\s+([0-9a-f]{16})\s", re.I)
with open(path, encoding="utf-8", errors="replace") as f:
for line in f:
m = pat.match(line)
if m:
syms.append((int(m.group(2), 16) - base, m.group(1)))
syms.sort()
return [a for a, _ in syms], [n for _, n in syms]
# The modules whose frames are Hookshot's own code (the game and the runtime, the renderer): Windows, then Linux
GAME_MODS = ("hookshot.exe", "hookshot")
VK_MODS = ("hookshot_vk.dll", "hookshot_vk.so")
RUN_MODS = GAME_MODS + VK_MODS
def load_elf(path):
"""An ELF's function symbols by their offset in the file's address space, which is the offset from the load address
that a sample of a position-independent executable or a shared object gives (binutils' nm, demangled)"""
out = subprocess.run(["nm", "--defined-only", "-C", path], capture_output=True, text=True, encoding="utf-8",
errors="replace").stdout
syms = []
for line in out.splitlines():
parts = line.split(" ", 2)
if len(parts) == 3 and parts[1] in ("t", "T", "w", "W") and len(parts[0]) >= 8:
syms.append((int(parts[0], 16), parts[2].split("(")[0]))
syms.sort()
return [a for a, _ in syms], [n for _, n in syms]
def load_symbols(folder=None):
"""{module: (addresses, names)} for the modules found in folder (RX_MAPS, else the build): the linker maps of the
Windows build, the binaries of the Linux one"""
folder = folder or os.environ.get("RX_MAPS", BUILD)
maps = {}
for mod in ("hookshot_vk.dll", "hookshot.exe"):
mp = os.path.join(folder, mod[:-4] + ".map")
if os.path.exists(mp):
maps[mod] = load_map(mp, 0x180000000 if mod.endswith(".dll") else 0x140000000)
for mod in ("hookshot_vk.so", "hookshot"):
b = os.path.join(folder, mod)
if os.path.isfile(b):
try:
maps[mod] = load_elf(b)
except OSError: # (no nm: frames stay module+offset)
pass
return maps
def module_name(mod):
"""hookshot_vk / hookshot, whichever system the sample came from"""
return mod.split(".")[0]
def demangle(n):
# ?name@ns@@... -> ns::name (enough to read)
m = re.match(r"\?([^@]+)@([^@]*)@", n)
if m and not n.startswith("??"):
return "%s::%s" % (m.group(2), m.group(1)) if m.group(2) else m.group(1)
m = re.match(r"\?\?\$?([^@]+)@", n)
return m.group(1) if m else n
def profile_lines(path, skip):
"""Keep the old line-based slice without holding the capture in memory."""
with open(path, encoding="utf-8", errors="replace") as f, ExitStack() as files:
start = 0
if skip:
# Windows can report a pipe as seekable, even though it cannot be replayed.
if stat.S_ISREG(os.fstat(f.fileno()).st_mode) and f.seekable():
count = sum(1 for _ in f)
else:
# A pipe can only be read once. Save its lines on disk to keep memory bounded.
spool = files.enter_context(TemporaryFile(mode="w+t", encoding="utf-8", newline="\n"))
count = 0
for count, line in enumerate(f, 1):
spool.write(line)
f = spool
start = int(count * skip)
if start < 0:
start = max(0, count + start)
else:
start = min(start, count)
f.seek(0)
yield from itertools.islice(f, start, None)
def main():
path = sys.argv[1]
top = int(sys.argv[2]) if len(sys.argv) > 2 else 40
skip = float(sys.argv[3]) if len(sys.argv) > 3 else 0.0
only = sys.argv[4] if len(sys.argv) > 4 else None
maps = load_symbols()
def name(frame):
mod, _, off = frame.rpartition("+")
if mod in maps:
addrs, names = maps[mod]
i = bisect.bisect_right(addrs, int(off, 16)) - 1
if i >= 0:
return module_name(mod) + ":" + demangle(names[i])
return mod
incl, self_, leafmod = collections.Counter(), collections.Counter(), collections.Counter()
n = 0
for line in profile_lines(path, skip):
frames = line.strip().split(";")
if not frames or not frames[0]:
continue
named = [name(f) for f in frames]
if only and not any(only in f for f in named):
continue
n += 1
self_[named[0]] += 1
leafmod[frames[0].rpartition("+")[0]] += 1
for f in set(named):
incl[f] += 1
print("%d samples" % n)
print("-- leaf modules")
for k, v in leafmod.most_common(10):
print("%6.1f%% %s" % (100.0 * v / n, k))
print("-- inclusive")
for k, v in incl.most_common(top):
print("%6.1f%% %s" % (100.0 * v / n, k))
print("-- self")
for k, v in self_.most_common(top):
print("%6.1f%% %s" % (100.0 * v / n, k))
if __name__ == "__main__":
main()

268
scripts/release_parts.py Normal file
View File

@@ -0,0 +1,268 @@
"""What the release workflows need from earlier releases, and the Hookshot Builder package folder made in CI
(.github/workflows/windows-builder.yml, linux-appimage.yml). No game data is involved: the parts hold the builder, its tools
and the play profile (addresses and counts made on the development PC).
python -I scripts/release_parts.py previous <out list> --repo <owner/name> [--repo ...] [--skip-tag <tag>]
the newest stable release's hookshot-setup.txt (of the first repository that has one; not --skip-tag's, the release
being made); prints that release's tag. Exits 1 when no repository has one.
python -I scripts/release_parts.py profile <setup list> <work folder>
the play profile (tools/work of the list's app part) into <work folder>
python -I scripts/release_parts.py llvm-mingw <folder> [--name]
llvm-mingw as scripts/make_setup_parts.py names it (its official release, SHA-256 checked) unpacked into <folder>;
prints its folder. --name: only prints its name (a cache key)
python -I scripts/release_parts.py package <out folder> --publish <dotnet publish folder> --setup <setup exe>
[--list <setup list>] [--python-packages <requirements>]
the package folder make_setup_parts.py splits: the published builder, the setup, LICENSE, the tracked gpu, host, nx2
and scripts under tools/, tools/work (the profile) and tools/third_party: the tracked files and, from the list's
third_party part, the prebuilt files the repository does not carry (a64emit.dll, lld-link.exe, SDL2.dll, SDL3.dll,
zbic.dll; rebuilt on the player's PC when their sources' stamps differ). Python, emsdk, CPM and wgpu-native are not
in it: make_setup_parts.py --reuse takes their parts from the list. --python-packages: when the list's Python part
lacks one of those pinned packages, the part goes into tools/python with them installed (by this Python's pip, for
the part's version), and make_setup_parts.py makes a new Python part of it
GitHub's API is read with $GH_TOKEN when it is set. Downloads are retried (GitHub answers 5xx now and then). Progress goes to
stderr: what a command prints on stdout is its answer.
"""
import argparse
import hashlib
import importlib.util
import json
import os
import shutil
import subprocess
import sys
import tempfile
import time
import urllib.error
import urllib.request
import zipfile
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
CACHE = os.path.join(os.environ.get("RUNNER_TEMP") or tempfile.gettempdir(), "release-parts")
# (the prebuilt files of the third_party part: what it holds and git does not)
PREBUILT_EXT = (".dll", ".lib", ".exe", ".sha1")
def get(url, api=False, tries=5):
if not url.startswith("https://"): # (only https: the token is never sent over plain http, nor a file: link read)
raise ValueError("not an https link: " + url)
headers = {"User-Agent": "hookshot-ci"}
token = None
if api:
headers["Accept"] = "application/vnd.github+json"
token = os.environ.get("GH_TOKEN")
for i in range(tries):
try:
req = urllib.request.Request(url, headers=headers)
if token: # (not sent again on a redirect, which may lead to another host: urllib copies plain headers)
req.add_unredirected_header("Authorization", "Bearer " + token)
with urllib.request.urlopen(req, timeout=120) as r:
return r.read()
except urllib.error.HTTPError as e:
if e.code < 500 or i == tries - 1:
raise
except (urllib.error.URLError, TimeoutError):
if i == tries - 1:
raise
print(" %s: retrying in 10 s" % url, file=sys.stderr, flush=True)
time.sleep(10)
def previous(a):
for repo in a.repo:
try:
releases = json.loads(get("https://api.github.com/repos/%s/releases?per_page=30" % repo, api=True))
except urllib.error.HTTPError as e:
print("%s: %s" % (repo, e), file=sys.stderr)
continue
for r in releases:
if r["draft"] or r["prerelease"] or r["tag_name"] == a.skip_tag:
continue
url = next((x["browser_download_url"] for x in r["assets"] if x["name"] == "hookshot-setup.txt"), None)
if url:
with open(a.out, "wb") as f:
f.write(get(url))
print(r["tag_name"])
return 0
print("no release with a hookshot-setup.txt in %s" % ", ".join(a.repo), file=sys.stderr)
return 1
def part_lines(path):
with open(path, encoding="utf-8") as f:
return [l.split() for l in f.read().splitlines() if l.startswith("part ")]
def fetch_part(sha, size, url):
"""A part downloaded once into the cache, its size and SHA-256 checked"""
os.makedirs(CACHE, exist_ok=True)
path = os.path.join(CACHE, url.rsplit("/", 1)[-1])
if not (os.path.isfile(path) and os.path.getsize(path) == size and sha256(path) == sha):
print("downloading %s (%d MB)" % (url, size >> 20), file=sys.stderr, flush=True)
data = get(url)
if len(data) != size or hashlib.sha256(data).hexdigest() != sha:
sys.exit("%s: not the size and SHA-256 the list gives" % url)
with open(path + ".new", "wb") as f:
f.write(data)
os.replace(path + ".new", path)
return path
def sha256(path):
h = hashlib.sha256()
with open(path, "rb") as f:
while b := f.read(1 << 20):
h.update(b)
return h.hexdigest()
def named_part(list_path, name):
for w in part_lines(list_path):
if w[6].rsplit("/", 1)[-1].startswith("HookshotBuilder-%s-" % name):
return fetch_part(w[1], int(w[2]), w[6])
return None
def extract(zip_path, prefix, dest, keep=lambda rel: True):
"""The zip's members under prefix (HookshotBuilder/...), into dest without that prefix; how many"""
n = 0
with zipfile.ZipFile(zip_path) as z:
for m in z.infolist():
if m.is_dir() or not m.filename.startswith(prefix):
continue
rel = m.filename[len(prefix):]
if ".." in rel.split("/") or not keep(rel):
continue
out = os.path.join(dest, *rel.split("/"))
# (a name with a drive or a root, such as "C:/x" on Windows, would leave dest: such members are skipped)
root = os.path.realpath(dest)
if os.path.commonpath([os.path.realpath(out), root]) != root:
continue
os.makedirs(os.path.dirname(out), exist_ok=True)
with z.open(m) as src, open(out, "wb") as f:
shutil.copyfileobj(src, f)
n += 1
return n
def profile(a):
app = named_part(a.list, "app")
if not app:
sys.exit("no app part in %s" % a.list)
n = extract(app, "HookshotBuilder/tools/work/", a.work)
print("%d profile files into %s" % (n, a.work))
return 0
def llvm_spec():
spec = importlib.util.spec_from_file_location("make_setup_parts", os.path.join(ROOT, "scripts", "make_setup_parts.py"))
m = importlib.util.module_from_spec(spec)
spec.loader.exec_module(m)
return m.LLVM # (sha256, size, top folder in the zip, url)
def llvm_mingw(a):
sha, size, name, url = llvm_spec()
if a.name:
print(name)
return 0
folder = os.path.join(a.folder, name)
if not os.path.isfile(os.path.join(folder, "bin", "clang.exe")):
z = fetch_part(sha, size, url)
with zipfile.ZipFile(z) as zf:
zf.extractall(a.folder)
print(folder)
return 0
def tracked(*paths):
r = subprocess.run(["git", "ls-files", "-z", "--"] + list(paths), cwd=ROOT, check=True, stdout=subprocess.PIPE)
return [f for f in r.stdout.decode("utf-8").split("\0") if f]
def copy(src, dst):
os.makedirs(os.path.dirname(dst), exist_ok=True)
shutil.copyfile(src, dst)
def package(a):
out = os.path.abspath(a.out)
if os.path.exists(out):
shutil.rmtree(out)
shutil.copytree(a.publish, out)
copy(a.setup, os.path.join(out, "HookshotBuilder-Setup.exe")) # (the builder's Repair runs it)
copy(os.path.join(ROOT, "LICENSE"), os.path.join(out, "LICENSE"))
tools = os.path.join(out, "tools")
for f in tracked("gpu", "host", "nx2", "scripts", "third_party"):
copy(os.path.join(ROOT, f), os.path.join(tools, *f.split("/")))
if a.list:
if not profile(argparse.Namespace(list=a.list, work=os.path.join(tools, "work"))) == 0:
return 1
tp = named_part(a.list, "third_party")
if tp:
known = set(tracked("third_party"))
prebuilt = lambda rel: rel.startswith("third_party/") and rel not in known and rel.lower().endswith(PREBUILT_EXT)
n = extract(tp, "HookshotBuilder/tools/", tools, keep=prebuilt)
print("%d prebuilt third_party files from %s" % (n, os.path.basename(tp)))
if a.python_packages and python_packages(a.list, a.python_packages, tools) != 0:
return 1
else:
print("no setup list: the package has no play profile and no prebuilt third_party files")
print("package folder: %s" % out)
return 0
def python_packages(list_path, requirements, tools):
"""The list's Python part with requirements' pins (name==version) installed into tools/python, if it lacks one: by this
Python's pip, for the part's version and win_amd64 (the part has no pip)"""
with open(requirements, encoding="utf-8") as f:
pins = [p for p in (l.split("#")[0].strip() for l in f.read().splitlines()) if p]
part = named_part(list_path, "python")
if not part:
sys.exit("no python part in %s" % list_path)
with zipfile.ZipFile(part) as z:
names = [m.split("/") for m in z.namelist()]
have = {m[-2].lower() for m in names if len(m) > 1 and m[-2].endswith(".dist-info")}
missing = [p for p in pins if "%s-%s.dist-info" % tuple(x.strip().lower().replace("-", "_") for x in p.split("==")) not in have]
if not missing:
print("the Python part has %s" % ", ".join(pins))
return 0
dll = [m[-1] for m in names if m[:-1] == ["HookshotBuilder", "tools", "python"] and m[-1].lower().startswith("python3")
and m[-1][7:-4].isdigit()]
if len(dll) != 1:
sys.exit("%s: no pythonXY.dll to give the version" % os.path.basename(part))
version = "3." + dll[0][7:-4]
py = os.path.join(tools, "python")
n = extract(part, "HookshotBuilder/tools/python/", py)
print("%d files of %s (Python %s) into %s, with %s" % (n, os.path.basename(part), version, py, ", ".join(missing)))
subprocess.run([sys.executable, "-m", "pip", "install", "--no-deps", "--only-binary", ":all:", "--no-compile", "--no-cache-dir",
"--disable-pip-version-check", "-q", "--platform", "win_amd64", "--python-version", version,
"--implementation", "cp", "--target", os.path.join(py, "Lib", "site-packages")] + missing, check=True)
return 0
def main():
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
sub = p.add_subparsers(dest="cmd", required=True)
s = sub.add_parser("previous")
s.add_argument("out")
s.add_argument("--repo", action="append", required=True)
s.add_argument("--skip-tag", default="")
s = sub.add_parser("profile")
s.add_argument("list")
s.add_argument("work")
s = sub.add_parser("llvm-mingw")
s.add_argument("folder", nargs="?", default="")
s.add_argument("--name", action="store_true")
s = sub.add_parser("package")
s.add_argument("out")
s.add_argument("--publish", required=True)
s.add_argument("--setup", required=True)
s.add_argument("--list")
s.add_argument("--python-packages")
a = p.parse_args()
return {"previous": previous, "profile": profile, "llvm-mingw": llvm_mingw, "package": package}[a.cmd](a)
if __name__ == "__main__":
sys.exit(main())

21
scripts/release_upload.sh Executable file
View File

@@ -0,0 +1,21 @@
#!/bin/bash
# A tag's GitHub release with these files attached. The Linux AppImage and Windows builder workflows both call it: the
# first to get there makes the release (from the tag, with GitHub's generated notes), the other adds its files to it, so
# the two never race to make two releases. A release candidate (a tag with -rc: v0.2.0-rc.1) is a prerelease and never
# "latest". Needs gh, with GH_TOKEN (contents: write) and GH_REPO set. No game data is involved.
#
# scripts/release_upload.sh <tag> <file> ...
set -euo pipefail
tag=$1
shift
if [[ $tag == *-rc* ]]; then
kind=(--prerelease --latest=false)
else
kind=(--latest)
fi
if ! gh release view "$tag" >/dev/null 2>&1; then
# (the other workflow can make it between the view and the create: its release is then the one used)
gh release create "$tag" --verify-tag --title "$tag" --generate-notes "${kind[@]}" ||
gh release view "$tag" >/dev/null
fi
gh release upload "$tag" --clobber "$@"

54
scripts/run-vk.ps1 Normal file
View File

@@ -0,0 +1,54 @@
# A test run of the standalone build (mkw-recomp's work/run-vk.ps1, for hookshot.exe): the Vulkan back end
# (RX_BACKEND=vulkan), headless unless -Window (then it never takes focus), High priority (-Priority), muted unless
# -Sound, screenshots every -ShotEvery presents into work\runs\<name>\shots. Extra environment: -Env "A=1;B=2".
#
# Guards (the PC is the user's, and one game runs at a time): no run while Rainbow Six Siege is open, nor while
# another session's mkw.exe or hookshot.exe runs (-Force overrides both); the game is stopped when its working set passes
# -MaxGB. Delete work\runs\<name>\shots once the shots are looked at: they reach gigabytes.
param([int]$Seconds = 120, [string]$Name = "vk", [int]$ShotEvery = 300, [string]$Env = "", [switch]$Window,
[int]$MaxGB = 10, [switch]$Force, [switch]$Sound, [string]$Romfs = "", [string]$Priority = "High")
$ErrorActionPreference = "Stop"
$root = Split-Path -Parent $PSScriptRoot
if (-not $Force) {
if (Get-Process RainbowSix, RainbowSix_BE, RainbowSix_Vulkan -ErrorAction SilentlyContinue) {
"a game is running (Rainbow Six): no test run now"; exit 2
}
$other = Get-Process mkw, hookshot -ErrorAction SilentlyContinue
if ($other) { "another game run is going ($(($other | ForEach-Object { "$($_.Name).exe $($_.Id)" }) -join ', ')): one at a time"; exit 2 }
}
if (-not $Romfs) { # (the dump's romfs, from work\dump.txt: nx2/dump.py --set)
$cfg = Join-Path $root "work\dump.txt"
if (-not (Test-Path $cfg)) { "no -Romfs and no work\dump.txt (python -I nx2/dump.py --dump <folder> --set)"; exit 2 }
$dump = (Get-Content $cfg | Where-Object { $_ -and -not $_.StartsWith("#") } | Select-Object -First 1).Trim()
$Romfs = Join-Path $dump "romfs"
}
$exe = Join-Path $root "work\sa\build\hookshot.exe"
$image = Join-Path $root "work\sa\image\image.bin"
$user = Join-Path $root "work\sa\user"
$out = Join-Path $root "work\runs\$Name"
$shots = Join-Path $out "shots"
New-Item -ItemType Directory -Force $out, $user, $shots | Out-Null
$env:RX_BACKEND = "vulkan"
if ($Window) { Remove-Item Env:HOS_HEADLESS -ErrorAction SilentlyContinue; $env:HOS_WINDOW_NOACTIVATE = "1" } else { $env:HOS_HEADLESS = "1" }
$env:RX_VK_SHOTS = $shots
$env:RX_VK_SHOT_EVERY = "$ShotEvery"
# (test runs are silent: the user is often in a call; -Sound plays it)
if ($Sound) { Remove-Item Env:HOS_AUDIO -ErrorAction SilentlyContinue } else { $env:HOS_AUDIO = "0" }
foreach ($kv in ($Env -split ";")) { if ($kv -match "^([^=]+)=(.*)$") { Set-Item -Path ("Env:" + $matches[1]) -Value $matches[2] } }
$t0 = Get-Date
$p = Start-Process -FilePath $exe -ArgumentList @("`"$image`"", "--romfs", "`"$Romfs`"", "--user", "`"$user`"") `
-WorkingDirectory $out -PassThru -NoNewWindow `
-RedirectStandardOutput (Join-Path $out "stdout.txt") -RedirectStandardError (Join-Path $out "log.txt")
try { $p.PriorityClass = $Priority } catch {} # (High by default: the user, 2026-11-04; -Priority BelowNormal to yield)
$peak = 0
$exited = $false
for ($t = 0; $t -lt $Seconds; $t += 2) {
if ($p.WaitForExit(2000)) { $exited = $true; break }
try { $p.Refresh(); $mb = [int]($p.WorkingSet64 / 1MB) } catch { $mb = 0 }
if ($mb -gt $peak) { $peak = $mb }
if ($mb -gt $MaxGB * 1024) { "memory guard: $mb MB, stopped"; break }
}
if (-not $exited) { Stop-Process -Id $p.Id -Force -ErrorAction SilentlyContinue }
"peak working set: $peak MB"
$secs = [int]((Get-Date) - $t0).TotalSeconds
"run '$Name': $secs s, " + $(if ($exited) { "exited with $($p.ExitCode)" } else { "stopped after $Seconds s" })

226
scripts/validate_spirv.py Normal file
View File

@@ -0,0 +1,226 @@
"""The SPIR-V Hookshot's translator kept in hookshot_programs.bin, checked offline before a driver sees it again.
Each stage goes through the Vulkan SDK's spirv-val, and through the device limits spirv-val does not know: the workgroup's
invocations and its workgroup memory (the mesh fallback keeps a mesh program's outputs there, on top of the game's own
shared memory). A stage that fails is written to --out as <key>_<n>_<stage>.spv, its disassembly (.spvasm, spirv-dis) and
the reasons (.txt); the summary groups the validator's errors by message.
python -I scripts/validate_spirv.py "D:\\Games\\Hookshot\\hookshot_programs.bin" (the mesh-as-compute programs)
python -I scripts/validate_spirv.py <file> --all (every program)
python -I scripts/validate_spirv.py <file> --key a5da9f84b72d1471 (the programs whose key starts so)
The exit status is 1 when a stage failed, 0 when none did.
"""
import argparse
import collections
import concurrent.futures
import os
import re
import shutil
import struct
import subprocess
import sys
CACHE_MAGIC = 0x31505348 # (gpu/vk/vk_shader.cpp kDiskMagic)
FALLBACK = 4 # (a record's kind: a task or mesh program run as compute, issue 640)
SPV_MAGIC = b"\x03\x02\x23\x07"
STAGES = {0x1: "vertex", 0x2: "tesscontrol", 0x4: "tesseval", 0x8: "geometry", 0x10: "fragment", 0x20: "compute",
0x40: "task", 0x80: "mesh"}
def records(path, want):
"""(key, kind, bytes) of each record want(key, kind) takes, in file order. The layout is disk_init's: a 16-byte head
(magic, 0, stamp), then records of a u64 key, a u64 size and that many bytes, which start with the u32 kind. A record
cut short ends the walk, as it does there"""
with open(path, "rb") as f:
head = f.read(16)
if len(head) < 16 or struct.unpack_from("<I", head)[0] != CACHE_MAGIC:
raise SystemExit("%s: not a Hookshot program cache" % path)
while True:
rec = f.read(16)
if len(rec) < 16:
return
key, size = struct.unpack("<QQ", rec)
at = f.tell()
kind = f.read(4)
if len(kind) < 4:
return
kind = struct.unpack("<I", kind)[0]
if want(key, kind):
f.seek(at)
data = f.read(size)
if len(data) < size:
return
yield key, kind, data
f.seek(at + size)
def stages(data):
"""(nvn stage, vk stage, SPIR-V bytes) of each stage of a record. program_cache_save writes a stage as int nvn_stage,
VkShaderStageFlagBits vk_stage, then the SPIR-V as a list (a u64 word count, the words): each is found by its magic
word and kept when the three fields before it agree, so the record's other fields (the answers, Shader::Info) need no
layout of their own here"""
out = []
i = data.find(SPV_MAGIC)
while i != -1:
if i >= 16:
nvn, vk = struct.unpack_from("<iI", data, i - 16)
n = struct.unpack_from("<Q", data, i - 8)[0]
if 0 <= nvn <= 7 and vk in STAGES and 5 <= n and i + 4 * n <= len(data):
out.append((nvn, vk, data[i:i + 4 * n]))
i = data.find(SPV_MAGIC, i + 4 * n)
continue
i = data.find(SPV_MAGIC, i + 1)
return out
def reflect(spirv):
"""(local size or None, workgroup memory bytes) of a module. The bytes are its Workgroup variables' types summed with
no padding: a lower bound of what the driver lays out, so a module over the limit by it is over by any layout"""
w = struct.unpack("<%dI" % (len(spirv) // 4), spirv[:len(spirv) // 4 * 4])
size, const, pointee = {}, {}, {}
local, local_ids, shared = None, None, 0
i = 5
while i < len(w):
n, op = w[i] >> 16, w[i] & 0xFFFF
if n == 0 or i + n > len(w):
break
a = w[i + 1:i + n]
if op == 16 and len(a) >= 5 and a[1] == 17: # OpExecutionMode LocalSize x y z
local = tuple(a[2:5])
elif op == 331 and len(a) >= 5 and a[1] == 38: # OpExecutionModeId LocalSizeId (constants defined later)
local_ids = a[2:5]
elif op in (43, 50) and len(a) >= 3: # OpConstant, OpSpecConstant (its default): the low word
const[a[1]] = a[2]
elif op == 20: # OpTypeBool
size[a[0]] = 4
elif op in (21, 22): # OpTypeInt, OpTypeFloat: width in bits
size[a[0]] = a[1] // 8
elif op in (23, 24): # OpTypeVector, OpTypeMatrix: component type, count
size[a[0]] = size.get(a[1], 0) * a[2]
elif op == 28: # OpTypeArray: element type, length constant
size[a[0]] = size.get(a[1], 0) * const.get(a[2], 0)
elif op == 30: # OpTypeStruct: members
size[a[0]] = sum(size.get(m, 0) for m in a[1:])
elif op == 32: # OpTypePointer: id, storage class, pointee
pointee[a[0]] = a[2]
elif op == 59 and len(a) >= 3 and a[2] == 4: # OpVariable of storage class Workgroup
shared += size.get(pointee.get(a[0]), 0)
i += n
if local_ids is not None:
local = tuple(const.get(x, 0) for x in local_ids)
return local, shared
def find_tool(name):
"""A Vulkan SDK tool: on PATH, else in %VULKAN_SDK%\\Bin"""
found = shutil.which(name)
if not found and os.environ.get("VULKAN_SDK"):
candidate = os.path.join(os.environ["VULKAN_SDK"], "Bin", name + (".exe" if os.name == "nt" else ""))
found = candidate if os.path.isfile(candidate) else None
return found
def check(job, a, val):
"""The reasons a stage fails ([] when it passes): spirv-val's errors (on stdin), then the device limits"""
key, nvn, vk, spirv = job
reasons = []
r = subprocess.run([val, "--target-env", a.target_env, *a.val_arg, "-"], input=spirv, capture_output=True)
if r.returncode:
text = (r.stdout + r.stderr).decode("utf-8", "replace").strip()
reasons.append(("spirv-val", text or "spirv-val exit status %d" % r.returncode))
local, shared = reflect(spirv)
if local and local[0] * local[1] * local[2] > a.max_invocations:
reasons.append(("limit", "workgroup %dx%dx%d is %d invocations, over maxComputeWorkGroupInvocations %d"
% (*local, local[0] * local[1] * local[2], a.max_invocations)))
if shared > a.max_shared:
reasons.append(("limit", "workgroup memory at least %d bytes, over maxComputeSharedMemorySize %d"
% (shared, a.max_shared)))
return reasons, local, shared
def first_error(text):
"""A validator message with its numbers taken out, so the same fault in many stages counts as one"""
line = next((x for x in text.splitlines() if x.startswith("error")), text.splitlines()[0] if text else "")
return re.sub(r"\d+", "N", line).strip()
def dump(out, job, reasons, local, shared, dis):
key, nvn, vk, spirv = job
base = os.path.join(out, "%016x_%d_%s" % (key, nvn, STAGES[vk]))
with open(base + ".spv", "wb") as f:
f.write(spirv)
with open(base + ".txt", "w", encoding="utf-8") as f:
f.write("program %016x, NVN stage %d, %s stage, %d words\n" % (key, nvn, STAGES[vk], len(spirv) // 4))
f.write("local size %s, workgroup memory at least %d bytes\n\n" % (local, shared))
for kind, text in reasons:
f.write("[%s]\n%s\n\n" % (kind, text))
if dis:
subprocess.run([dis, base + ".spv", "-o", base + ".spvasm"], capture_output=True)
return base
def main(argv=None):
p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
p.add_argument("cache", help="hookshot_programs.bin")
p.add_argument("--all", action="store_true", help="every program, not only the mesh-as-compute ones")
p.add_argument("--key", action="append", default=[], help="a program key (hex) or its start; may be given again")
p.add_argument("--out", help="where failing stages are written (default: spirv_failures beside the cache)")
p.add_argument("--target-env", default="vulkan1.3", help="spirv-val's target (Hookshot asks for Vulkan 1.3)")
p.add_argument("--val-arg", action="append", default=[], help="an argument passed on to spirv-val")
p.add_argument("--max-shared", type=int, default=65536, help="maxComputeSharedMemorySize (RDNA1: 65536)")
p.add_argument("--max-invocations", type=int, default=1024, help="maxComputeWorkGroupInvocations (RDNA1: 1024)")
p.add_argument("--jobs", type=int, default=os.cpu_count() or 4)
a = p.parse_args(argv)
val, dis = find_tool("spirv-val"), find_tool("spirv-dis")
if not val:
raise SystemExit("spirv-val not found: install the Vulkan SDK or put its Bin folder on PATH")
keys = [k.lower().removeprefix("0x") for k in a.key]
out = a.out or os.path.join(os.path.dirname(os.path.abspath(a.cache)), "spirv_failures")
def want(key, kind):
if keys:
return any(("%016x" % key).startswith(k) for k in keys)
return a.all or bool(kind & FALLBACK)
programs, failed, groups = 0, [], collections.defaultdict(list)
with concurrent.futures.ThreadPoolExecutor(a.jobs) as pool:
pending = collections.deque()
def settle(future):
job, (reasons, local, shared) = future.job, future.result()
if not reasons:
return
os.makedirs(out, exist_ok=True)
base = dump(out, job, reasons, local, shared, dis)
failed.append(base)
for kind, text in reasons:
groups[first_error(text) if kind == "spirv-val" else re.sub(r"\d+", "N", text)].append(
"%016x %s" % (job[0], STAGES[job[2]]))
for key, kind, data in records(a.cache, want):
programs += 1
for nvn, vk, spirv in stages(data):
job = (key, nvn, vk, spirv)
future = pool.submit(check, job, a, val)
future.job = job
pending.append(future)
while len(pending) > 4 * a.jobs: # (a few batches in flight: --all reads gigabytes)
settle(pending.popleft())
while pending:
settle(pending.popleft())
checked = "programs" if a.all or keys else "mesh-as-compute programs"
print("%s: %d %s, %d stages failed" % (a.cache, programs, checked, len(failed)))
for message, where in sorted(groups.items(), key=lambda g: -len(g[1])):
print(" %4dx %s" % (len(where), message))
print(" e.g. %s" % ", ".join(where[:3]))
if failed:
print("written to %s (.spv, .spvasm, .txt)" % out)
if not programs:
print(" (no records matched%s)" % (": is the key right?" if keys else ""))
return 1 if failed else 0
if __name__ == "__main__":
sys.exit(main())

62
scripts/vkprof.py Normal file
View File

@@ -0,0 +1,62 @@
"""The renderer's threads from an all-thread sample (host/mkw_glue.c HOS_SAMPLE_ALL): for the threads whose name starts
with a prefix (default mkw_vk), the share of samples running, and the functions by inclusive and self share of the
running samples (leaf in hookshot_vk.dll, hookshot.exe or the driver; on Linux hookshot_vk.so, hookshot or the driver).
python -I scripts/vkprof.py work/runs/<run>_all.txt [top 40] [prefix mkw_vk]
(RX_MAPS=<folder>: another build's linker maps, or on Linux the game folder)
"""
import bisect, collections, os, sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import prof # noqa: E402
top = int(sys.argv[2]) if len(sys.argv) > 2 else 40
prefix = sys.argv[3] if len(sys.argv) > 3 else "mkw_vk"
maps = prof.load_symbols()
cache = {}
def name(f):
if f in cache:
return cache[f]
mod, _, off = f.rpartition("+")
r = mod
if mod in maps:
a, n = maps[mod]
i = bisect.bisect_right(a, int(off, 16)) - 1
if i >= 0:
r = prof.module_name(mod) + ":" + prof.demangle(n[i])[:70]
cache[f] = r
return r
RUN = prof.RUN_MODS + ("nvoglv64.dll",)
DRIVERS = ("libnvidia-glcore.so", "libvulkan_radeon.so", "libvulkan_intel.so") # (Linux: the driver's file has its version after .so)
seen, running = collections.Counter(), collections.Counter()
incl, self_ = collections.Counter(), collections.Counter()
nrun = 0
for line in open(sys.argv[1], encoding="utf-8", errors="replace"):
head, _, st = line.rstrip("\n").partition("|")
th = head.split(" ", 1)[-1]
if not st or not th.startswith(prefix):
continue
frames = st.split(";")
seen[th] += 1
leafmod = frames[0].rpartition("+")[0]
if leafmod not in RUN and not leafmod.startswith(DRIVERS):
continue
running[th] += 1
nrun += 1
names = [name(f) for f in frames]
self_[names[0]] += 1
for n in set(names):
incl[n] += 1
print("-- running share by thread")
for th in sorted(seen, key=lambda t: -running[t]):
print(" %5.1f%% %s (%d samples)" % (100.0 * running[th] / seen[th], th, seen[th]))
print("-- inclusive (share of %d running samples)" % nrun)
for k, c in incl.most_common(top):
print(" %5.1f%% %s" % (100.0 * c / max(1, nrun), k))
print("-- self")
for k, c in self_.most_common(top):
print(" %5.1f%% %s" % (100.0 * c / max(1, nrun), k))