mirror of
https://git.triforceproject.com/triforce/source
synced 2026-11-19 16:20:01 -04:00
Triforce v0.2.0
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
140
scripts/changelog.py
Normal file
140
scripts/changelog.py
Normal file
@@ -0,0 +1,140 @@
|
||||
"""Write the next entry of changelog.txt in the Factorio mod changelog format.
|
||||
|
||||
python3 scripts/changelog.py <version> [<last-tag>]
|
||||
|
||||
semantic-release calls this from its prepare step with the version being released and the last release's tag. The
|
||||
commits after that tag are read from git and the entry is put at the top of changelog.txt. Only Conventional Commits
|
||||
that make a release are listed, so the changelog says what the version rules say. HOOKSHOT_RELEASE_DATE=YYYY-MM-DD
|
||||
fixes the date, for a release job that runs again.
|
||||
|
||||
A prerelease (`0.2.0-rc.1`) changes nothing: Factorio's parser accepts only `major.minor.patch` versions, and two
|
||||
release candidates share one base version. The final release gets the entry, with every commit since the last stable
|
||||
tag.
|
||||
|
||||
The file stays in the format Factorio's changelog parser expects: a 99-dash separator before each version, Version:
|
||||
and Date: lines, section names indented two spaces, bullets four, no tabs, no trailing spaces, newest entry first. See
|
||||
docs/versioning.md.
|
||||
"""
|
||||
import argparse
|
||||
import datetime
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
CHANGELOG = os.path.join(ROOT, "changelog.txt")
|
||||
SEPARATOR = "-" * 99
|
||||
# (the order sections appear in; only the ones with commits are written)
|
||||
SECTIONS = ("Major Features", "Minor Features", "Features", "Changes", "Bugfixes", "Optimizations",
|
||||
"Graphics", "Sounds", "Balancing", "Scripting", "Modding", "Locale", "Info")
|
||||
# (what a released commit becomes; docs, style, test, build, ci and chore are not listed)
|
||||
SECTIONS_BY_TYPE = {"feat": "Features", "fix": "Bugfixes", "perf": "Optimizations", "revert": "Bugfixes",
|
||||
"refactor": "Changes"}
|
||||
HEADER = re.compile(r"^(?P<type>[A-Za-z][A-Za-z0-9-]*)(?:\((?P<scope>[^)\r\n]*)\))?(?P<breaking>!)?: (?P<text>.+)$")
|
||||
BREAKING = re.compile(r"^BREAKING[ -]CHANGES?:", re.MULTILINE)
|
||||
VERSION = re.compile(r"^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$")
|
||||
|
||||
|
||||
def commits(since=None):
|
||||
"""(subject, body) for every commit after `since`; the order does not matter, the entry is grouped by section."""
|
||||
rev = "%s..HEAD" % since if since else "HEAD"
|
||||
r = subprocess.run(["git", "-C", ROOT, "log", "--no-merges", "--format=%H%x1f%s%x1f%b%x1e", rev],
|
||||
capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=120)
|
||||
if r.returncode:
|
||||
raise SystemExit("git log %s failed: %s" % (rev, r.stderr.strip()))
|
||||
out = []
|
||||
for record in r.stdout.split("\x1e"):
|
||||
fields = record.strip("\n").split("\x1f", 2)
|
||||
if len(fields) == 3:
|
||||
out.append((fields[1].strip(), fields[2]))
|
||||
return out
|
||||
|
||||
|
||||
def bullet(scope, text):
|
||||
"""One Factorio bullet: a capital first letter and a full stop, as the format uses."""
|
||||
text = " ".join(text.split())
|
||||
if not text:
|
||||
return ""
|
||||
text = text[:1].upper() + text[1:]
|
||||
if text[-1] not in ".!?":
|
||||
text += "."
|
||||
if scope:
|
||||
scope = scope[:1].upper() + scope[1:]
|
||||
return "%s: %s" % (scope, text)
|
||||
return text
|
||||
|
||||
|
||||
def date_line(value=None):
|
||||
value = value or os.environ.get("HOOKSHOT_RELEASE_DATE", "").strip()
|
||||
day = datetime.date.fromisoformat(value) if value else datetime.date.today()
|
||||
return "%02d. %02d. %04d" % (day.day, day.month, day.year)
|
||||
|
||||
|
||||
def entry(version, since=None, date=None):
|
||||
found = {section: [] for section in SECTIONS}
|
||||
for subject, body in commits(since):
|
||||
m = HEADER.match(subject)
|
||||
if not m:
|
||||
continue
|
||||
text = bullet((m.group("scope") or "").strip(), m.group("text"))
|
||||
if not text:
|
||||
continue
|
||||
if m.group("breaking") or BREAKING.search(body):
|
||||
found["Changes"].append("Breaking: " + text)
|
||||
elif m.group("type").lower() in SECTIONS_BY_TYPE:
|
||||
found[SECTIONS_BY_TYPE[m.group("type").lower()]].append(text)
|
||||
lines = [SEPARATOR, "Version: %s" % version, "Date: %s" % date_line(date)]
|
||||
for section in SECTIONS:
|
||||
# (Factorio rejects an exact duplicate in one section, so a repeated subject is listed once)
|
||||
entries = list(dict.fromkeys(found[section]))
|
||||
if entries:
|
||||
lines.append(" %s:" % section)
|
||||
lines.extend(" - %s" % one for one in entries)
|
||||
if len(lines) == 3: # (no commit followed the convention: say the least that is true)
|
||||
lines.extend((" Changes:", " - Release %s." % version))
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def write(version, since=None, date=None):
|
||||
if "-" in version:
|
||||
print("changelog: %s is a prerelease; changelog.txt is not changed" % version)
|
||||
return
|
||||
try:
|
||||
with open(CHANGELOG, encoding="utf-8") as f:
|
||||
old = f.read()
|
||||
except FileNotFoundError:
|
||||
old = ""
|
||||
if "Version: %s" % version in old.splitlines(): # (a retried job must not add the version twice)
|
||||
print("changelog: %s is already in changelog.txt" % version)
|
||||
return
|
||||
content = entry(version, since, date) + old.lstrip("\n")
|
||||
temporary = None
|
||||
try:
|
||||
with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", newline="\n", delete=False,
|
||||
dir=os.path.dirname(CHANGELOG), prefix=".changelog-", suffix=".tmp") as f:
|
||||
temporary = f.name
|
||||
f.write(content)
|
||||
if os.path.exists(CHANGELOG):
|
||||
shutil.copymode(CHANGELOG, temporary)
|
||||
os.replace(temporary, CHANGELOG)
|
||||
finally:
|
||||
if temporary and os.path.exists(temporary):
|
||||
os.unlink(temporary)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description="Add the next version's entry to changelog.txt.")
|
||||
ap.add_argument("version", help="the version being released, without the v (0.2.0)")
|
||||
ap.add_argument("since", nargs="?", default="", help="the last release's tag (v0.1.0); every commit if omitted")
|
||||
ap.add_argument("--date", help="YYYY-MM-DD; default today, or HOOKSHOT_RELEASE_DATE")
|
||||
a = ap.parse_args()
|
||||
if not VERSION.match(a.version):
|
||||
ap.error("not a semantic version: %s" % a.version)
|
||||
write(a.version, a.since or None, a.date)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
115
scripts/crit.py
Normal file
115
scripts/crit.py
Normal file
@@ -0,0 +1,115 @@
|
||||
"""The frame's critical path from an all-thread sample (host/mkw_glue.c HOS_SAMPLE_ALL): every 2 ms each guest thread
|
||||
and renderer thread, "<ms> <name>|<frames>" (module+offset, leaf first).
|
||||
|
||||
python -I scripts/crit.py work/runs/<run>_all.txt [top]
|
||||
|
||||
A thread is running at an instant when its leaf is in hookshot.exe or hookshot_vk.dll (on Linux hookshot or
|
||||
hookshot_vk.so: game code, the runtime, the renderer), blocked otherwise (the leaf in ntdll, win32u, KERNELBASE, libc,
|
||||
the driver). RX_MAPS=<folder>: another build's linker maps, or on Linux the game folder (its binaries' symbols). Prints each thread's running share;
|
||||
then for the instants when no game thread runs (the frame waits on something outside the game's CPU work): what each
|
||||
thread waits in and what the renderer's threads do; then for the instants when the main thread is blocked: how many
|
||||
game threads run and the functions they are in.
|
||||
"""
|
||||
import collections, os, sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import prof # noqa: E402
|
||||
|
||||
RUN_MODS = prof.RUN_MODS
|
||||
|
||||
|
||||
def main():
|
||||
path = sys.argv[1]
|
||||
top = int(sys.argv[2]) if len(sys.argv) > 2 else 25
|
||||
maps = prof.load_symbols()
|
||||
cache = {}
|
||||
|
||||
def name(frame):
|
||||
if frame in cache:
|
||||
return cache[frame]
|
||||
mod, _, off = frame.rpartition("+")
|
||||
r = mod
|
||||
if mod in maps:
|
||||
addrs, names = maps[mod]
|
||||
import bisect
|
||||
i = bisect.bisect_right(addrs, int(off, 16)) - 1
|
||||
if i >= 0:
|
||||
r = prof.module_name(mod) + ":" + prof.demangle(names[i])
|
||||
cache[frame] = r
|
||||
return r
|
||||
|
||||
ticks = collections.defaultdict(dict) # ms -> thread -> (running, frames named)
|
||||
for line in open(path, encoding="utf-8", errors="replace"):
|
||||
head, _, stack = line.rstrip("\n").partition("|")
|
||||
ms, _, thread = head.partition(" ")
|
||||
if not stack or not ms.isdigit():
|
||||
continue
|
||||
frames = stack.split(";")
|
||||
leafmod = frames[0].rpartition("+")[0]
|
||||
ticks[int(ms)][thread] = (leafmod in RUN_MODS, [name(f) for f in frames])
|
||||
n = len(ticks)
|
||||
print("%d instants" % n)
|
||||
running = collections.Counter()
|
||||
seen = collections.Counter()
|
||||
for t in ticks.values():
|
||||
for th, (r, _) in t.items():
|
||||
seen[th] += 1
|
||||
running[th] += r
|
||||
print("-- running share by thread")
|
||||
for th, c in sorted(running.items(), key=lambda kv: -kv[1])[:30]:
|
||||
print(" %5.1f%% %s" % (100.0 * c / seen[th], th))
|
||||
|
||||
def is_game(th):
|
||||
return not th.startswith("mkw_vk")
|
||||
|
||||
def guest_frame(frames):
|
||||
"""the innermost game function (f_...) and the import it called, if any"""
|
||||
imp = next((f for f in frames if ":imp_" in f), None)
|
||||
fn = next((f for f in frames if ":f_" in f), None)
|
||||
return (imp.split(":imp_")[1][:48] + " <- " if imp else "") + (fn or frames[-1])
|
||||
|
||||
idle = [t for t in ticks.values() if not any(r for th, (r, _) in t.items() if is_game(th))]
|
||||
print("-- no game thread running: %.1f%% of instants" % (100.0 * len(idle) / max(1, n)))
|
||||
waits = collections.Counter()
|
||||
vk = collections.Counter()
|
||||
for t in idle:
|
||||
for th, (r, frames) in t.items():
|
||||
if th in ("sead::MainThread", "Presentation_Thread") or th.startswith("BaseProcExecutor0"):
|
||||
waits["%s: %s" % (th, guest_frame(frames))] += 1
|
||||
if not is_game(th):
|
||||
vk["%s %s: %s" % (th, "runs" if r else "waits", frames[0] if not r else
|
||||
next((f for f in frames if f.startswith("hookshot_vk:")), frames[0]))] += 1
|
||||
for k, c in waits.most_common(top):
|
||||
print(" %5.1f%% %s" % (100.0 * c / max(1, len(idle)), k))
|
||||
print(" (renderer threads then)")
|
||||
for k, c in vk.most_common(top):
|
||||
print(" %5.1f%% %s" % (100.0 * c / max(1, len(idle)), k))
|
||||
|
||||
mw = [t for t in ticks.values() if "sead::MainThread" in t and not t["sead::MainThread"][0]]
|
||||
print("-- main thread blocked: %.1f%% of instants" % (100.0 * len(mw) / max(1, n)))
|
||||
hist = collections.Counter(sum(1 for th, (r, _) in t.items() if r and is_game(th)) for t in mw)
|
||||
print(" game threads running then: " + ", ".join("%d: %.0f%%" % (k, 100.0 * v / len(mw)) for k, v in sorted(hist.items())))
|
||||
mwhere = collections.Counter(guest_frame(t["sead::MainThread"][1]) for t in mw)
|
||||
print(" main waits in:")
|
||||
for k, c in mwhere.most_common(8):
|
||||
print(" %5.1f%% %s" % (100.0 * c / len(mw), k))
|
||||
fns = collections.Counter()
|
||||
for t in mw:
|
||||
for th, (r, frames) in t.items():
|
||||
if r and is_game(th):
|
||||
fns[next((f for f in frames if ":f_" in f), frames[0])] += 1
|
||||
print(" game functions running then (innermost f_, share of those instants):")
|
||||
for k, c in fns.most_common(top):
|
||||
print(" %5.1f%% %s" % (100.0 * c / len(mw), k))
|
||||
leaves = collections.Counter()
|
||||
for t in mw:
|
||||
for th, (r, frames) in t.items():
|
||||
if r and is_game(th):
|
||||
leaves[frames[0]] += 1
|
||||
print(" leaves running then:")
|
||||
for k, c in leaves.most_common(top):
|
||||
print(" %5.1f%% %s" % (100.0 * c / len(mw), k))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
136
scripts/framelog.py
Normal file
136
scripts/framelog.py
Normal file
@@ -0,0 +1,136 @@
|
||||
"""The flashes in a frame log (RX_VK_FRAMELOG, gpu/vk/vk_replay.cpp): capture samples, 320x180.
|
||||
|
||||
Timestamps sample command recording before GPU submission.
|
||||
|
||||
python -I scripts/framelog.py <file> [out.png]
|
||||
python -I scripts/framelog.py <file> --inspect
|
||||
|
||||
A frame is a flash where a part of the picture goes brighter (or darker) than BOTH its neighbours by more than they differ
|
||||
from each other, with the brightness of that part back within reach the next frame. Blocks of 20x20, so that an edge moving
|
||||
past counts little; a block must jump in brightness, not only in detail. Prints the events by size, with whether the frame was
|
||||
the game's or a made one, and writes the worst as rows of (before, frame, after).
|
||||
Only consecutive sequence numbers form a triplet: dropped records cannot establish a one-frame flash. Inspect a
|
||||
completed, stable capture; --inspect validates its headers without NumPy. Sequence numbers may wrap at uint32;
|
||||
nonadvancing headers, partial records and captures shorter than three frames are rejected.
|
||||
"""
|
||||
import argparse
|
||||
import mmap
|
||||
from pathlib import Path
|
||||
import struct
|
||||
from typing import NamedTuple
|
||||
|
||||
W, H = 320, 180
|
||||
SIZE = 16 + W * H * 4
|
||||
MASK = (1 << 32) - 1
|
||||
|
||||
|
||||
class CaptureInfo(NamedTuple):
|
||||
timestamps: list[int]
|
||||
sequences: list[int]
|
||||
kinds: list[int]
|
||||
missing: int
|
||||
centres: list[int]
|
||||
|
||||
|
||||
def triplet_centres(sequences):
|
||||
"""Only stored consecutive frames can establish a one-frame brightness jump."""
|
||||
return [i for i in range(1, len(sequences) - 1)
|
||||
if ((sequences[i] - sequences[i - 1]) & MASK) == 1
|
||||
and ((sequences[i + 1] - sequences[i]) & MASK) == 1]
|
||||
|
||||
|
||||
def capture_info(path):
|
||||
size = Path(path).stat().st_size
|
||||
count, tail = divmod(size, SIZE)
|
||||
if tail:
|
||||
raise ValueError("truncated record %d: %d bytes, expected %d" % (count + 1, tail, SIZE))
|
||||
if count < 3:
|
||||
raise ValueError("need at least 3 frames for flash analysis; got %d" % count)
|
||||
timestamps, sequences, kinds = [], [], []
|
||||
missing = 0
|
||||
with open(path, "rb") as file, mmap.mmap(file.fileno(), 0, access=mmap.ACCESS_READ) as raw:
|
||||
for i in range(count):
|
||||
t, s, k = struct.unpack_from("<QII", raw, i * SIZE)
|
||||
if k not in (0, 1):
|
||||
raise ValueError("invalid frame kind %d at record %d (expected 0 or 1)" % (k, i + 1))
|
||||
if timestamps and t <= timestamps[-1]:
|
||||
raise ValueError("timestamp does not advance at record %d" % (i + 1))
|
||||
if sequences:
|
||||
delta = (s - sequences[-1]) & MASK
|
||||
if not delta or delta >= (1 << 31):
|
||||
raise ValueError("sequence does not advance at record %d" % (i + 1))
|
||||
missing += delta - 1
|
||||
timestamps.append(t), sequences.append(s), kinds.append(k)
|
||||
return CaptureInfo(timestamps, sequences, kinds, missing, triplet_centres(sequences))
|
||||
|
||||
|
||||
def blocks(a):
|
||||
"""luma means of 20x20 blocks"""
|
||||
l = a[..., 0] * 0.3 + a[..., 1] * 0.6 + a[..., 2] * 0.1
|
||||
return l[:H // 20 * 20, :].reshape(H // 20, 20, W // 20, 20).mean(axis=(1, 3))
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("path", type=Path)
|
||||
parser.add_argument("out", nargs="?", help="write the worst events to this PNG")
|
||||
parser.add_argument("--inspect", action="store_true", help="validate metadata and report eligible triplets without NumPy")
|
||||
args = parser.parse_args()
|
||||
path, out = args.path, args.out
|
||||
try:
|
||||
info = capture_info(path)
|
||||
except (OSError, ValueError, struct.error) as error:
|
||||
parser.error(str(error))
|
||||
n = len(info.sequences)
|
||||
print(n, "frames")
|
||||
eligible = set(info.centres)
|
||||
print("flash triplets: %d eligible, %d skipped across sequence gaps; missing %d frames" % (
|
||||
len(eligible), n - 2 - len(eligible), info.missing))
|
||||
if args.inspect:
|
||||
return 0
|
||||
try:
|
||||
import numpy as np
|
||||
except ImportError:
|
||||
parser.error("image analysis requires NumPy; --inspect validates capture metadata without it")
|
||||
raw = np.memmap(path, dtype=np.uint8, mode="r")
|
||||
ts = np.array(info.timestamps, dtype=np.uint64)
|
||||
kinds = np.array(info.kinds)
|
||||
d = np.diff(ts) / 1e6
|
||||
print("capture samples apart: median %.1f ms, 99%% %.1f, worst %.1f; made %d of %d; missing %d" % (
|
||||
np.median(d), np.percentile(d, 99), d.max(), kinds.sum(), n, info.missing))
|
||||
frame = lambda i: np.asarray(raw[i * SIZE + 16:(i + 1) * SIZE]).reshape(H, W, 4)[..., :3].astype(np.float32)
|
||||
prev = blocks(frame(0))
|
||||
cur = blocks(frame(1))
|
||||
events = []
|
||||
for i in range(1, n - 1):
|
||||
nxt = blocks(frame(i + 1))
|
||||
if i in eligible:
|
||||
up = np.minimum(cur - prev, cur - nxt)
|
||||
down = np.minimum(prev - cur, nxt - cur)
|
||||
span = np.abs(prev - nxt)
|
||||
for sign, jump in ((1, up), (-1, down)):
|
||||
hit = (jump > 12) & (jump > span * 1.5)
|
||||
c = int(hit.sum())
|
||||
if c:
|
||||
events.append((float(jump[hit].sum()), i, sign, c, float(jump.max())))
|
||||
prev, cur = cur, nxt
|
||||
events.sort(reverse=True)
|
||||
tot = {0: 0, 1: 0}
|
||||
for e in events:
|
||||
tot[int(kinds[e[1]])] += 1
|
||||
print("events: %d on the game's frames, %d on made frames" % (tot[0], tot[1]))
|
||||
for size, i, sign, c, peak in events[:24]:
|
||||
print("frame %d (%s) %s %d blocks, peak %.0f, at %.2f s" % (i, "made" if kinds[i] else "game", "brighter" if sign > 0 else "darker",
|
||||
c, peak, (ts[i] - ts[0]) / 1e9))
|
||||
if out and events:
|
||||
from PIL import Image
|
||||
rows = events[:8]
|
||||
im = Image.new("RGB", (W * 3, H * len(rows)))
|
||||
for r, e in enumerate(rows):
|
||||
for k in range(3):
|
||||
im.paste(Image.fromarray(frame(e[1] - 1 + k).astype(np.uint8)), (k * W, r * H))
|
||||
im.save(out)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
202
scripts/fsr3_shaders.py
Normal file
202
scripts/fsr3_shaders.py
Normal file
@@ -0,0 +1,202 @@
|
||||
"""Compile AMD's FSR 3.1 upscaler passes into the SPIR-V headers gpu/vk uses.
|
||||
|
||||
The FidelityFX SDK ships each pass as GLSL that includes the SDK's own GPU headers, so the eleven
|
||||
variants below are the same eight passes compiled with different options. They are checked in like
|
||||
every other shader in gpu/vk/shaders, so a clone builds without glslang; this script regenerates
|
||||
them, and --check proves the checked-in files still match their sources (tests/test_fsr3_shaders.py).
|
||||
|
||||
python -I -B scripts/fsr3_shaders.py # write gpu/vk/shaders/fsr3_*_comp_spv.h
|
||||
python -I -B scripts/fsr3_shaders.py --check # compare instead of writing
|
||||
"""
|
||||
import argparse
|
||||
import difflib
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SDK = ROOT / "third_party/fidelityfx-sdk"
|
||||
GPU = SDK / "include/FidelityFX/gpu"
|
||||
PASSES = SDK / "shaders/fsr3upscaler"
|
||||
FI_PASSES = SDK / "shaders/frameinterpolation"
|
||||
OF_PASSES = SDK / "shaders/opticalflow"
|
||||
OUT = ROOT / "gpu/vk/shaders"
|
||||
|
||||
# The option set the renderer records these passes under. Each one is a promise about the inputs:
|
||||
# the colour is HDR, the motion is low-resolution and not jittered, and the depth is reversed. They
|
||||
# have to match what gpu/vk/vk_fsr3.cpp hands the passes, which is why they live in one place.
|
||||
OPTIONS = [
|
||||
"-DFFX_GPU=1",
|
||||
"-DFFX_GLSL=1",
|
||||
"-DFFX_FSR3UPSCALER_OPTION_UPSAMPLE_USE_LANCZOS_TYPE=2",
|
||||
"-DFFX_FSR3UPSCALER_OPTION_REPROJECT_USE_LANCZOS_TYPE=0",
|
||||
"-DFFX_FSR3UPSCALER_OPTION_HDR_COLOR_INPUT=1",
|
||||
"-DFFX_FSR3UPSCALER_OPTION_LOW_RESOLUTION_MOTION_VECTORS=1",
|
||||
"-DFFX_FSR3UPSCALER_OPTION_JITTERED_MOTION_VECTORS=0",
|
||||
"-DFFX_FSR3UPSCALER_OPTION_INVERTED_DEPTH=1",
|
||||
]
|
||||
|
||||
# name, the pass GLSL it compiles, extra defines. The _nowave variants are for devices without the
|
||||
# subgroup operations the single-pass downsampler prefers.
|
||||
VARIANTS = [
|
||||
("prepare_inputs", "prepare_inputs", []),
|
||||
("luma_pyramid", "luma_pyramid", []),
|
||||
("luma_pyramid_nowave", "luma_pyramid", ["-DFFX_SPD_NO_WAVE_OPERATIONS=1"]),
|
||||
("shading_change_pyramid", "shading_change_pyramid", []),
|
||||
("shading_change_pyramid_nowave", "shading_change_pyramid", ["-DFFX_SPD_NO_WAVE_OPERATIONS=1"]),
|
||||
("shading_change", "shading_change", []),
|
||||
("prepare_reactivity", "prepare_reactivity", []),
|
||||
("luma_instability", "luma_instability", []),
|
||||
("accumulate", "accumulate", []),
|
||||
("accumulate_sharpen", "accumulate", ["-DFFX_FSR3UPSCALER_OPTION_APPLY_SHARPENING=1"]),
|
||||
("rcas", "rcas", []),
|
||||
]
|
||||
|
||||
|
||||
# (hookshot) Frame interpolation and the optical flow it runs on, the same arrangement: AMD ships
|
||||
# each pass as GLSL over the SDK's own GPU headers, and the renderer records them as SPIR-V headers
|
||||
# beside the upscaler's. The options are the same promises about the same inputs -- the depth is
|
||||
# reversed and the motion is at the render extent -- because the inputs are the ones the upscaler is
|
||||
# already given (gpu/vk/vk_fsr3fi.cpp).
|
||||
FI_OPTIONS = [
|
||||
"-DFFX_GPU=1",
|
||||
"-DFFX_GLSL=1",
|
||||
"-DFFX_FRAMEINTERPOLATION_OPTION_INVERTED_DEPTH=1",
|
||||
"-DFFX_FRAMEINTERPOLATION_OPTION_LOW_RESOLUTION_MOTION_VECTORS=1",
|
||||
"-DFFX_FRAMEINTERPOLATION_OPTION_JITTERED_MOTION_VECTORS=0",
|
||||
]
|
||||
OF_OPTIONS = ["-DFFX_GPU=1", "-DFFX_GLSL=1"]
|
||||
|
||||
# (name, the pass file, extra defines). The file is named outright rather than composed: AMD's
|
||||
# optical-flow v5 passes end "_pass_v5.glsl" and the interpolation pass itself is just
|
||||
# "ffx_frameinterpolation_pass.glsl", so a composed name is wrong for four of the eighteen.
|
||||
FI_VARIANTS = [
|
||||
("fi_setup", "ffx_frameinterpolation_setup_pass.glsl", []),
|
||||
("fi_reconstruct_previous_depth", "ffx_frameinterpolation_reconstruct_previous_depth_pass.glsl", []),
|
||||
("fi_reconstruct_and_dilate", "ffx_frameinterpolation_reconstruct_and_dilate_pass.glsl", []),
|
||||
("fi_game_motion_vector_field", "ffx_frameinterpolation_game_motion_vector_field_pass.glsl", []),
|
||||
("fi_optical_flow_vector_field", "ffx_frameinterpolation_optical_flow_vector_field_pass.glsl", []),
|
||||
("fi_disocclusion_mask", "ffx_frameinterpolation_disocclusion_mask_pass.glsl", []),
|
||||
("fi_game_vector_field_inpainting_pyramid",
|
||||
"ffx_frameinterpolation_compute_game_vector_field_inpainting_pyramid_pass.glsl", []),
|
||||
("fi_inpainting_pyramid", "ffx_frameinterpolation_compute_inpainting_pyramid_pass.glsl", []),
|
||||
("fi_interpolate", "ffx_frameinterpolation_pass.glsl", []),
|
||||
("fi_inpainting", "ffx_frameinterpolation_inpainting_pass.glsl", []),
|
||||
]
|
||||
OF_VARIANTS = [
|
||||
("of_prepare_luma", "ffx_opticalflow_prepare_luma_pass.glsl", []),
|
||||
("of_luminance_pyramid", "ffx_opticalflow_compute_luminance_pyramid_pass.glsl", []),
|
||||
("of_generate_scd_histogram", "ffx_opticalflow_generate_scd_histogram_pass.glsl", []),
|
||||
("of_compute_scd_divergence", "ffx_opticalflow_compute_scd_divergence_pass.glsl", []),
|
||||
("of_compute_optical_flow_v5", "ffx_opticalflow_compute_optical_flow_advanced_pass_v5.glsl", []),
|
||||
("of_filter_optical_flow_v5", "ffx_opticalflow_filter_optical_flow_pass_v5.glsl", []),
|
||||
("of_scale_optical_flow_v5", "ffx_opticalflow_scale_optical_flow_advanced_pass_v5.glsl", []),
|
||||
]
|
||||
|
||||
|
||||
def compile_family(glslang, name, source, extra, out_path, target_env, options, includes,
|
||||
variable=True):
|
||||
"""One frame-interpolation or optical-flow pass. The upscaler's compile_one is kept as it is:
|
||||
its passes are composed by name and its option set is the upscaler's own."""
|
||||
cmd = [glslang, "-V", "--quiet", "-S", "comp"]
|
||||
for d in includes:
|
||||
cmd.append("-I" + str(d))
|
||||
cmd += [*options, *extra]
|
||||
if variable:
|
||||
cmd += ["--variable-name", f"FSR3_{name.upper()}_COMP_SPV"]
|
||||
cmd += ["-o", str(out_path), str(source), "--target-env", target_env]
|
||||
subprocess.run(cmd, check=True)
|
||||
|
||||
|
||||
def compile_one(glslang, name, pass_name, extra, out_path, target_env, variable=True):
|
||||
"""Compile one variant. With `variable`, glslang writes a C header; without it, binary SPIR-V,
|
||||
which is what spirv-val reads."""
|
||||
defines = list(OPTIONS)
|
||||
if not any("APPLY_SHARPENING" in d for d in extra):
|
||||
defines.append("-DFFX_FSR3UPSCALER_OPTION_APPLY_SHARPENING=0")
|
||||
cmd = [glslang, "-V", "--quiet", "-S", "comp",
|
||||
"-I" + str(GPU), "-I" + str(GPU / "fsr3upscaler"),
|
||||
*defines, *extra]
|
||||
if variable:
|
||||
cmd += ["--variable-name", f"FSR3_{name.upper()}_COMP_SPV"]
|
||||
cmd += ["-o", str(out_path),
|
||||
str(PASSES / f"ffx_fsr3upscaler_{pass_name}_pass.glsl"),
|
||||
"--target-env", target_env]
|
||||
subprocess.run(cmd, check=True)
|
||||
|
||||
|
||||
def strip_generator(text):
|
||||
"""The header without glslang's own version comment.
|
||||
|
||||
glslang writes its version into the first line as `// 1116.2.0`, so two glslang builds produce
|
||||
byte-different headers from identical SPIR-V. Comparing the whole text made this check fail on
|
||||
a CI runner whose glslang was one release behind the one that generated the checked-in files --
|
||||
a version bump reported as a source mismatch, which is the kind of red that teaches people to
|
||||
ignore red. The words after it are the module, and those are what this is about.
|
||||
"""
|
||||
return [line for line in text.splitlines()
|
||||
if not line.strip().startswith("// ") or not line.strip()[3:].replace(".", "").isdigit()]
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
p.add_argument("--glslang", default=None, help="glslangValidator; found on PATH by default")
|
||||
p.add_argument("--spirv-val", default=None, help="spirv-val, to validate each module")
|
||||
p.add_argument("--target-env", default="spirv1.3")
|
||||
p.add_argument("--check", action="store_true", help="compare with the checked-in headers")
|
||||
a = p.parse_args()
|
||||
glslang = a.glslang or shutil.which("glslangValidator") or shutil.which("glslang")
|
||||
if not glslang:
|
||||
print("glslangValidator not found; pass --glslang", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
bad = []
|
||||
with tempfile.TemporaryDirectory(prefix="fsr3-shaders-") as tmp:
|
||||
# the upscaler's eleven, then frame interpolation's ten and optical flow's seven
|
||||
families = [(name, None, pass_name, extra, None, None) for name, pass_name, extra in VARIANTS]
|
||||
families += [(name, FI_PASSES / f, None, extra, FI_OPTIONS,
|
||||
[GPU, GPU / "frameinterpolation", GPU / "opticalflow"])
|
||||
for name, f, extra in FI_VARIANTS]
|
||||
families += [(name, OF_PASSES / f, None, extra, OF_OPTIONS, [GPU, GPU / "opticalflow"])
|
||||
for name, f, extra in OF_VARIANTS]
|
||||
for name, source, pass_name, extra, options, includes in families:
|
||||
header = OUT / f"fsr3_{name}_comp_spv.h"
|
||||
made = Path(tmp) / header.name
|
||||
if source is None:
|
||||
compile_one(glslang, name, pass_name, extra, made, a.target_env)
|
||||
else:
|
||||
compile_family(glslang, name, source, extra, made, a.target_env, options, includes)
|
||||
if a.spirv_val:
|
||||
spv = Path(tmp) / f"{name}.spv"
|
||||
if source is None:
|
||||
compile_one(glslang, name, pass_name, extra, spv, a.target_env, variable=False)
|
||||
else:
|
||||
compile_family(glslang, name, source, extra, spv, a.target_env, options,
|
||||
includes, variable=False)
|
||||
subprocess.run([a.spirv_val, str(spv)], check=True)
|
||||
if a.check:
|
||||
have = header.read_text() if header.exists() else ""
|
||||
want = made.read_text()
|
||||
if strip_generator(have) != strip_generator(want):
|
||||
bad.append(name)
|
||||
if not have:
|
||||
print(f"{header.name}: missing", file=sys.stderr)
|
||||
else:
|
||||
diff = difflib.unified_diff(have.splitlines(), want.splitlines(),
|
||||
"checked in", "regenerated", n=0, lineterm="")
|
||||
print("\n".join(list(diff)[:12]), file=sys.stderr)
|
||||
else:
|
||||
shutil.copyfile(made, header)
|
||||
print(f"wrote {header.relative_to(ROOT)} ({header.stat().st_size} bytes)")
|
||||
if bad:
|
||||
print(f"{len(bad)} header(s) do not match their source: {', '.join(bad)}", file=sys.stderr)
|
||||
return 1
|
||||
print(f"fsr3 shaders: {len(VARIANTS) + len(FI_VARIANTS) + len(OF_VARIANTS)} {'match' if a.check else 'written'}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
82
scripts/hardware_report.py
Normal file
82
scripts/hardware_report.py
Normal file
@@ -0,0 +1,82 @@
|
||||
"""Who reports problems: the GPU vendor, card class and OS named in the issue forms (docs/workflow.md, "Reach").
|
||||
|
||||
python -I scripts/hardware_report.py [--limit 1000] [--state all|open|closed]
|
||||
|
||||
Reads the issues with the gh CLI (logged in) and counts the "Graphics card and driver version" and "CPU, RAM and operating
|
||||
system" answers. It counts reports, not players: hardware where things break shows up more. Standard library only.
|
||||
"""
|
||||
import argparse
|
||||
import collections
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
REPO = "NotRexed/Hookshot"
|
||||
|
||||
|
||||
def section(body: str, name: str) -> str:
|
||||
m = re.search(r"### " + re.escape(name) + r"[^\n]*\n+(.*?)(\n### |\Z)", body or "", re.S)
|
||||
return m.group(1).strip() if m else ""
|
||||
|
||||
|
||||
def classify(gpu: str):
|
||||
g = gpu.lower()
|
||||
if re.search(r"nvidia|rtx|gtx|geforce", g):
|
||||
cls = ("RTX 50" if re.search(r"rtx ?50\d0", g) else "RTX 40" if re.search(r"rtx ?40\d0", g) else
|
||||
"RTX 30" if re.search(r"rtx ?30\d0", g) else "RTX 20" if re.search(r"rtx ?20\d0", g) else
|
||||
"GTX 10/16 (no mesh shaders)" if re.search(r"gtx ?1[06]\d0", g) else "NVIDIA other")
|
||||
return "NVIDIA", cls
|
||||
if re.search(r"amd|radeon|\brx ?\d|rdna|780m|680m|890m|860m|8060s|z1|steam deck|legion|ally", g):
|
||||
cls = ("AMD APU / handheld" if re.search(r"780m|680m|890m|860m|8060s|z1|steam deck|legion|ally|apu|vega", g) else
|
||||
"RX 9000" if re.search(r"9070|9060", g) else "RX 7000" if re.search(r"7900|7800|7700|7600", g) else
|
||||
"RX 6000" if re.search(r"6[5-9]\d0|6600|6650|6700|6800|6900", g) else
|
||||
"RX 5000 or older (no mesh shaders)" if re.search(r"5[5-7]00|rx ?5[78]0|rx ?4[78]0", g) else "AMD other")
|
||||
return "AMD", cls
|
||||
if re.search(r"intel|arc\b|iris|uhd", g):
|
||||
return "Intel", "Intel"
|
||||
return "unknown", "unknown"
|
||||
|
||||
|
||||
def os_of(text: str) -> str:
|
||||
t = text.lower()
|
||||
if re.search(r"linux|ubuntu|arch|fedora|steamos|bazzite|cachy|nixos|mint|pop!?_os|steam deck", t):
|
||||
return "Linux / SteamOS"
|
||||
if re.search(r"windows|win ?1[01]", t):
|
||||
return "Windows"
|
||||
return "unknown"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
|
||||
ap.add_argument("--limit", type=int, default=1000)
|
||||
ap.add_argument("--state", default="all", choices=("all", "open", "closed"))
|
||||
a = ap.parse_args()
|
||||
out = subprocess.run(["gh", "issue", "list", "-R", REPO, "--state", a.state, "--limit", str(a.limit),
|
||||
"--json", "number,title,body"], capture_output=True, text=True, encoding="utf-8")
|
||||
if out.returncode:
|
||||
print(out.stderr.strip() or "gh failed", file=sys.stderr)
|
||||
return 1
|
||||
issues = json.loads(out.stdout)
|
||||
vendor, cls, oses = collections.Counter(), collections.Counter(), collections.Counter()
|
||||
n = 0
|
||||
for i in issues:
|
||||
gpu = section(i["body"], "Graphics card")
|
||||
sysinfo = section(i["body"], "CPU, RAM and operating system")
|
||||
if not gpu and not sysinfo:
|
||||
continue
|
||||
n += 1
|
||||
v, c = classify(gpu + " " + i["title"])
|
||||
vendor[v] += 1
|
||||
cls[c] += 1
|
||||
oses[os_of(sysinfo + " " + gpu + " " + i["title"])] += 1
|
||||
print(f"{len(issues)} issues read, {n} with hardware fields")
|
||||
for title, counter in (("vendor", vendor), ("card class", cls), ("OS", oses)):
|
||||
print(f"\n{title}:")
|
||||
for k, v in counter.most_common():
|
||||
print(f" {k:36s} {v:5d} {100 * v / max(n, 1):5.1f}%")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
53
scripts/hotlist.py
Normal file
53
scripts/hotlist.py
Normal file
@@ -0,0 +1,53 @@
|
||||
"""Hot lifted functions from an all-thread sample (HOS_SAMPLE_ALL, scripts/crit.py): for each sample of a game thread
|
||||
running in hookshot.exe (on Linux hookshot), the innermost lifted function (f_...) is charged. Writes the functions that make up the
|
||||
first <share> of those samples, hottest first, one a line, for the lift's A64_HOT_FUNCS (their FP ops call the
|
||||
helpers' inline fast paths; with A64_HOT_CHUNK=1 they are compiled together in code_hot.c).
|
||||
|
||||
python -I scripts/hotlist.py work/runs/<run>_all.txt <out> [share 0.85] [max 1500]
|
||||
"""
|
||||
import bisect, collections, os, sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import prof # noqa: E402
|
||||
|
||||
maps = prof.load_symbols()
|
||||
game = next((m for m in prof.GAME_MODS if m in maps), None)
|
||||
if not game:
|
||||
sys.exit("no hookshot.map (Windows) or hookshot binary (Linux) in %s: set RX_MAPS" % os.environ.get("RX_MAPS", prof.BUILD))
|
||||
addrs, names = maps[game]
|
||||
share = float(sys.argv[3]) if len(sys.argv) > 3 else 0.85
|
||||
cap = int(sys.argv[4]) if len(sys.argv) > 4 else 1500
|
||||
c = collections.Counter()
|
||||
total = 0
|
||||
for line in open(sys.argv[1], encoding="utf-8", errors="replace"):
|
||||
head, _, st = line.rstrip("\n").partition("|")
|
||||
if not st or head.split(" ", 1)[-1].startswith("mkw_vk"):
|
||||
continue
|
||||
frames = st.split(";")
|
||||
if not frames[0].startswith(game + "+"):
|
||||
continue
|
||||
for f in frames:
|
||||
mod, _, off = f.rpartition("+")
|
||||
if mod != game:
|
||||
continue
|
||||
i = bisect.bisect_right(addrs, int(off, 16)) - 1
|
||||
n = names[i] if i >= 0 else ""
|
||||
if n.startswith("f_"):
|
||||
c[n] += 1
|
||||
total += 1
|
||||
break
|
||||
out, acc = [], 0
|
||||
for n, k in c.most_common():
|
||||
if acc >= share * total or len(out) >= cap:
|
||||
break
|
||||
out.append(n)
|
||||
acc += k
|
||||
# (the game's Zstd decoder: the job loop's f_80115bf0 -> f_80115c80 tree. It runs while areas load and stream, which a play
|
||||
# profile seldom samples, so it fell to the cold -O1 chunks. At the hot flags a whole decode takes 16% less time and the
|
||||
# code is no bigger; the functions match Unicorn on every register and written page either way. Listed whatever the profile says)
|
||||
ZSTD_DECODER = ("f_80115c80 f_80109790 f_80113880 f_80116490 f_801099a0 f_8010b160 f_8010ffc0 f_801102e0 f_80110860 "
|
||||
"f_801117a0 f_80112880 f_80113080 f_801156e0 f_80115880 f_8010ad60 f_834ab610 f_834abda0 f_834ab4a0 "
|
||||
"f_834ac0b0 f_834ac110 f_834ac2a0 f_8572f040").split()
|
||||
out += [n for n in ZSTD_DECODER if n not in out]
|
||||
open(sys.argv[2], "w").write("".join(n + "\n" for n in out))
|
||||
print("%d samples in %d functions; %d functions cover %.0f%%" % (total, len(c), len(out), 100.0 * acc / max(1, total)))
|
||||
155
scripts/linux/appimage.sh
Executable file
155
scripts/linux/appimage.sh
Executable file
@@ -0,0 +1,155 @@
|
||||
#!/bin/bash
|
||||
# The Linux AppImage: Hookshot Builder with everything it needs to build the game on the player's PC. No game data and
|
||||
# nothing made from it is in it: the game is built from the player's own copy, as on Windows.
|
||||
#
|
||||
# scripts/linux/deps.sh (first: the compiler, Python, FFmpeg, SDL2, headers into build-linux/deps)
|
||||
# scripts/linux/appimage.sh -> build-linux/Hookshot-<version>-x86_64.AppImage
|
||||
# The same image builds the game and, with --play, starts the one it saved.
|
||||
#
|
||||
# Built on the oldest system it is for (CI: Ubuntu 22.04): the sysroot it carries is that system's C library, so the
|
||||
# game it builds runs on glibc 2.35 and newer. work/ (hot_funcs.txt, hot_order.txt, hookshot_pieces.prof,
|
||||
# image/chunks.txt: addresses and counts from playing, no game code) is carried along when it is there.
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/../.."
|
||||
ROOT=$PWD
|
||||
B=$ROOT/build-linux
|
||||
DEPS=${DEPS:-$B/deps}
|
||||
APPDIR=$B/AppDir
|
||||
VERSION=${HOOKSHOT_VERSION:-$(git describe --tags --always --match "v[0-9]*" 2>/dev/null || echo dev)}
|
||||
export EMSDK=$DEPS/emsdk RX_CPM=$DEPS/cpm
|
||||
PY=$DEPS/python/bin/python3
|
||||
|
||||
# ---- the Vulkan back end, built here once (it needs no game) ----
|
||||
VKW=$B/vk-work
|
||||
rm -rf "$VKW/sa/build/hookshot_vk.so"
|
||||
OOT_WORK=$VKW RX_MARCH=x86-64-v3 RX_VK_ONLY=1 RX_RELEASE=1 "$PY" -I nx2/build.py --standalone --jobs "${JOBS:-$(nproc)}"
|
||||
VK=$VKW/sa/build/hookshot_vk.so
|
||||
[ -f "$VK" ] || { echo "hookshot_vk.so was not built"; exit 1; }
|
||||
|
||||
# ---- the tools' native helpers, built now: the AppImage is read-only. a64emit with its C++ runtime inside (the one this system
|
||||
# builds with is newer than older systems have), and the stamp build_emit.py checks, so it is not built again ----
|
||||
E=third_party/a64-kit/lift
|
||||
"$EMSDK/upstream/bin/clang++" -std=c++20 -O2 -shared -fPIC -static-libstdc++ -static-libgcc -Wno-everything \
|
||||
-o "$E/liba64emit.so" "$E/a64emit.cpp"
|
||||
"$PY" -I -c "import hashlib, sys; f = ['$E/a64emit.cpp', '$E/suyu/arm64_to_c.h']; open('$E/liba64emit.so.sha1', 'w').write(hashlib.sha1(b''.join(open(x, 'rb').read() for x in f)).hexdigest())"
|
||||
"$PY" -I third_party/a64-kit/lift/build_emit.py
|
||||
"$PY" -I -c "import sys; sys.path.insert(0, 'nx2'); import zbic; zbic.lib()"
|
||||
|
||||
rm -rf "$APPDIR"
|
||||
mkdir -p "$APPDIR/usr/share/hookshot" "$APPDIR/usr/lib/hookshot"
|
||||
T=$APPDIR/usr/share/hookshot
|
||||
L=$APPDIR/usr/lib/hookshot
|
||||
|
||||
# ---- the tools (the sources the player's build compiles, and the scripts) ----
|
||||
cp -a nx2 host scripts LICENSE readme.md assets "$T/"
|
||||
printf '%s
|
||||
' "$VERSION" > "$T/VERSION" # (no git history in here: nx2/version.py reads the release name from it)
|
||||
mkdir -p "$T/builder" "$T/gpu/render" "$T/gpu/vk" "$T/third_party"
|
||||
cp -a builder/linux "$T/builder/"
|
||||
cp -a gpu/render/*.h "$T/gpu/render/"
|
||||
# (every gpu/vk header the host's sources include: v2.0.6 shipped without oot_cheat_items.h and no Linux build finished, issue 990.
|
||||
# tests/test_appimage_headers.py checks that this covers what host/ includes)
|
||||
for h in $(grep -rhoE 'gpu/vk/[A-Za-z0-9_]+[.]h' host | sort -u); do if [ -f "$h" ]; then cp -a "$h" "$T/gpu/vk/"; fi; done
|
||||
cp -a third_party/opus "$T/third_party/"
|
||||
cp -a third_party/kenney_input "$T/third_party/"
|
||||
# (NVIDIA DLSS, when ticked in the builder: scripts/ngx_sdk.py and the Vulkan back end's sources, compiled with HOOKSHOT_NGX_SDK)
|
||||
mkdir -p "$T/scripts"
|
||||
cp scripts/ngx_sdk.py "$T/scripts/"
|
||||
cp -a gpu/vk gpu/compat gpu/third_party "$T/gpu/"
|
||||
cp -a third_party/imgui third_party/zstd third_party/fonts "$T/third_party/"
|
||||
mkdir -p "$T/third_party/zbic" "$T/third_party/a64-kit"
|
||||
cp -a third_party/zbic/zbic.so third_party/zbic/LICENSE third_party/zbic/PROVENANCE.md "$T/third_party/zbic/"
|
||||
cp -a third_party/a64-kit/* "$T/third_party/a64-kit/"
|
||||
rm -f "$T/third_party/a64-kit/tools/lld-link.exe"
|
||||
if [ -d work ]; then
|
||||
mkdir -p "$T/work/image"
|
||||
for f in hot_funcs.txt hot_order.txt hookshot_pieces.prof pipeline_states.bin; do [ -f "work/$f" ] && cp "work/$f" "$T/work/"; done
|
||||
[ -f work/image/chunks.txt ] && cp work/image/chunks.txt "$T/work/image/"
|
||||
fi
|
||||
find "$T" -name __pycache__ -prune -exec rm -rf {} +
|
||||
|
||||
# ---- what it builds with ----
|
||||
cp -a "$DEPS/python" "$APPDIR/usr/python"
|
||||
cp "$DEPS/tk/libtcl9tk9.0.so" "$DEPS/tk/libXft.so.2" "$DEPS/tk/libXrender.so.1" "$APPDIR/usr/python/lib/"
|
||||
mkdir -p "$L/emsdk"
|
||||
cp -a "$DEPS/emsdk/upstream" "$L/emsdk/"
|
||||
cp -a "$DEPS/ffmpeg" "$L/ffmpeg"
|
||||
cp "$VK" "$L/hookshot_vk.so"
|
||||
cp "$DEPS/sdl2/libSDL2-2.0.so.0" "$L/libSDL2-2.0.so.0"
|
||||
for d in "$DEPS"/cpm/*/*; do
|
||||
mkdir -p "$T/cpm/${d#"$DEPS/cpm/"}"
|
||||
cp -a "$d/include" "$T/cpm/${d#"$DEPS/cpm/"}/"
|
||||
done
|
||||
cp -a "$DEPS/licenses" "$T/licenses" # (LLVM, FFmpeg, dav1d, SDL2: what this AppImage carries besides Hookshot)
|
||||
# ---- the licences of the third-party parts compiled into hookshot_vk.so (tests/test_appimage_licenses.py) ----
|
||||
mkdir -p "$T/licenses/imgui" "$T/licenses/zstd" "$T/licenses/libertinus"
|
||||
cp third_party/imgui/LICENSE.txt third_party/imgui/PROVENANCE.md "$T/licenses/imgui/"
|
||||
cp third_party/zstd/LICENSE third_party/zstd/PROVENANCE.md "$T/licenses/zstd/"
|
||||
cp third_party/fonts/libertinus/OFL.txt third_party/fonts/PROVENANCE.md "$T/licenses/libertinus/"
|
||||
# ---- end of the licences ----
|
||||
|
||||
# the sysroot: this system's C library (headers, start files, the libraries to link against) and GCC's runtime pieces
|
||||
S=$L/sysroot
|
||||
mkdir -p "$S/usr/lib/x86_64-linux-gnu" "$S/usr/lib/gcc/x86_64-linux-gnu" "$S/lib64"
|
||||
ln -s usr/lib "$S/lib"
|
||||
cp -a /usr/include "$S/usr/"
|
||||
rm -rf "$S/usr/include/c++" "$S/usr/include/x86_64-linux-gnu/c++" "$S/usr/include/llvm"* "$S/usr/include/clang"*
|
||||
M=/usr/lib/x86_64-linux-gnu
|
||||
for f in crt1.o crti.o crtn.o Scrt1.o rcrt1.o libc.so libc.so.6 libc_nonshared.a libm.so libm.so.6 libm.a libmvec.so libmvec.so.1 \
|
||||
libmvec.a libpthread.so.0 libpthread.a libdl.so.2 libdl.a librt.so.1 librt.a libutil.a libgcc_s.so.1 ld-linux-x86-64.so.2; do
|
||||
# System aliases can have absolute targets (e.g. libmvec.so). Carry their bytes;
|
||||
# update parts refuse absolute-target symlinks, and must not use the host's library.
|
||||
[ -e "$M/$f" ] && cp -aL "$M/$f" "$S/usr/lib/x86_64-linux-gnu/" || true
|
||||
done
|
||||
cp -aL /lib64/ld-linux-x86-64.so.2 "$S/lib64/"
|
||||
GCC=$(ls -d /usr/lib/gcc/x86_64-linux-gnu/*/ | sort -V | tail -1)
|
||||
GV=$(basename "$GCC")
|
||||
G=$S/usr/lib/gcc/x86_64-linux-gnu/$GV
|
||||
mkdir -p "$G" "$S/usr/include/c++" "$S/usr/include/x86_64-linux-gnu/c++"
|
||||
cp -aL "/usr/include/c++/$GV" "$S/usr/include/c++/"
|
||||
cp -aL "/usr/include/x86_64-linux-gnu/c++/$GV" "$S/usr/include/x86_64-linux-gnu/c++/"
|
||||
for f in crtbegin.o crtbeginS.o crtbeginT.o crtend.o crtendS.o libgcc.a libgcc_eh.a libgcc_s.so libstdc++.a libstdc++fs.a libstdc++exp.a; do
|
||||
[ -e "$GCC/$f" ] && cp -aL "$GCC/$f" "$G/" || true
|
||||
done
|
||||
# (libgcc_s.so is a linker script naming /lib/x86_64-linux-gnu/libgcc_s.so.1: found in the sysroot through lib -> usr/lib)
|
||||
|
||||
# ---- check: the bundled compiler builds and links a program against the sysroot, and it runs ----
|
||||
cat > "$B/t.c" <<'EOF'
|
||||
#include <pthread.h>
|
||||
#include <stdio.h>
|
||||
#include <math.h>
|
||||
static void *f(void *p) { return p; }
|
||||
int main(void) { pthread_t t; pthread_create(&t, 0, f, 0); pthread_join(t, 0); printf("ok %g\n", sqrt(2.0)); return 0; }
|
||||
EOF
|
||||
"$L/emsdk/upstream/bin/clang" --sysroot="$S" -fuse-ld=lld -O2 -march=x86-64-v3 "$B/t.c" -o "$B/t" -lpthread -lm
|
||||
"$B/t"
|
||||
cat > "$B/t.cpp" <<'EOF'
|
||||
#include <algorithm>
|
||||
#include <filesystem>
|
||||
#include <format>
|
||||
#include <string>
|
||||
extern "C" int t_cpp(int v) { return (int)std::format("{}", v).size(); }
|
||||
EOF
|
||||
"$L/emsdk/upstream/bin/clang++" --sysroot="$S" -fuse-ld=lld -std=c++20 -O2 -march=x86-64-v3 -shared -fPIC -static-libstdc++ -static-libgcc \
|
||||
"$B/t.cpp" -o "$B/t.so" -ldl -lpthread -lm
|
||||
|
||||
# ---- the AppDir's own files ----
|
||||
cp builder/linux/AppRun "$APPDIR/AppRun"
|
||||
cp builder/linux/hookshot-builder.desktop "$APPDIR/"
|
||||
# (RecompLauncher reads this from inside the AppImage, at /recomplauncher.json)
|
||||
cp recomplauncher.json "$APPDIR/recomplauncher.json"
|
||||
cp assets/hookshot.png "$APPDIR/hookshot.png"
|
||||
mkdir -p "$APPDIR/usr/share/icons/hicolor/256x256/apps"
|
||||
cp assets/hookshot.png "$APPDIR/usr/share/icons/hicolor/256x256/apps/hookshot.png"
|
||||
|
||||
# (the update list, beside AppRun and not inside a hashed part: hookshot-linux.txt names each part's checksum)
|
||||
# (the parts are served from the linux-parts release of the repository this is built in, where
|
||||
# scripts/linux/publish_parts.sh uploads them: without --base the list named NotRexed/Hookshot's release, which a fork's
|
||||
# CI does not upload to, so a fork's update list named parts that were not there)
|
||||
python3 scripts/linux/make_parts.py "$APPDIR" "$VERSION" "$B/parts" \
|
||||
--base "https://github.com/${GITHUB_REPOSITORY:-NotRexed/Hookshot}/releases/download/linux-parts"
|
||||
|
||||
OUT=$B/Hookshot-$VERSION-x86_64.AppImage
|
||||
rm -f "$OUT"
|
||||
ARCH=x86_64 APPIMAGE_EXTRACT_AND_RUN=1 "$DEPS/appimagetool" --runtime-file "$DEPS/runtime-x86_64" --comp zstd "$APPDIR" "$OUT"
|
||||
echo "$OUT: $(du -h "$OUT" | cut -f1)"
|
||||
51
scripts/linux/check_player_build.sh
Executable file
51
scripts/linux/check_player_build.sh
Executable file
@@ -0,0 +1,51 @@
|
||||
#!/bin/bash
|
||||
# Game-free: a player's build started from what the AppImage carries, not from the repository. CI's AppImage job packages
|
||||
# the builder but never builds a game with it, so a file the AppImage leaves out was found by every player instead: #1459
|
||||
# made nx2/build.py write the menu's fonts from third_party/fonts at every build start, the AppImage does not carry that
|
||||
# folder, and every Linux build stopped at LibertinusSans-Regular.ttf while CI stayed green (2026-11-10, #1567).
|
||||
#
|
||||
# scripts/linux/check_player_build.sh [<Hookshot-...-x86_64.AppImage>] (default: the newest in build-linux/)
|
||||
#
|
||||
# The image is extracted, as its runtime does, and run with AppRun's own environment (the AppImage's Python, compiler,
|
||||
# sysroot, FFmpeg and the prebuilt Vulkan back end, HOOKSHOT_VK_PREBUILT) in an otherwise empty one, so nothing of the
|
||||
# checkout or of CI's caches stands in for what the image lacks:
|
||||
# - AppRun --cli --help: nx2/make_game.py, as the builder window starts it;
|
||||
# - nx2/build.py --standalone --dry-run, as make_game.py runs it: its whole start-up up to the compile, which is where
|
||||
# the game's lifted code is first needed. Every other file the compile would read must be in the image.
|
||||
# Neither may write into the image's tree: on a player's PC it is read-only.
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/../.."
|
||||
IMAGE=${1:-$(ls -t build-linux/Hookshot-*-x86_64.AppImage 2>/dev/null | head -1 || true)}
|
||||
[ -n "$IMAGE" ] && [ -f "$IMAGE" ] || { echo "no AppImage to check (scripts/linux/appimage.sh makes one)"; exit 1; }
|
||||
IMAGE=$(readlink -f "$IMAGE")
|
||||
TMP=$(mktemp -d)
|
||||
trap 'rm -rf "$TMP"' EXIT
|
||||
(cd "$TMP" && "$IMAGE" --appimage-extract >/dev/null)
|
||||
R=$TMP/squashfs-root
|
||||
T=$R/usr/share/hookshot
|
||||
mkdir -p "$TMP/home" "$TMP/work"
|
||||
# (AppRun's environment, from AppRun itself: its lines before it starts Python, sourced with $0 naming it, so HERE is the image)
|
||||
sed '/^PY=/,$d' "$R/AppRun" > "$TMP/apprun-env.sh"
|
||||
in_image() {
|
||||
env -i PATH=/usr/bin:/bin HOME="$TMP/home" LANG=C.UTF-8 bash -c '. "$1"; shift; exec "$@"' "$R/AppRun" "$TMP/apprun-env.sh" "$@"
|
||||
}
|
||||
touch "$TMP/before"
|
||||
sleep 1 # (so a file written below is newer than the stamp)
|
||||
|
||||
echo "== AppRun --cli --help"
|
||||
in_image "$R/AppRun" --cli --help > "$TMP/help.txt"
|
||||
grep -q -- "--nup" "$TMP/help.txt" || { cat "$TMP/help.txt"; echo "nx2/make_game.py did not start"; exit 1; }
|
||||
|
||||
echo "== nx2/build.py --standalone --dry-run (as nx2/make_game.py runs it, with no game)"
|
||||
# (-B: no __pycache__ written into the image's tree. make_game.py's other settings: nx2/make_game.py, the compile's run())
|
||||
(cd "$T" && in_image env OOT_WORK="$TMP/work" RX_RELEASE=1 A64_HOT_CHUNK=1 PYTHONIOENCODING=utf-8 \
|
||||
"$R/usr/python/bin/python3" -I -B -u -X utf8 nx2/build.py --standalone --jobs 1 --dry-run)
|
||||
|
||||
# (Python's own __pycache__ aside: AppRun runs Python with -I, which writes it where it can, and does without where it cannot)
|
||||
written=$(find "$R" -type f -newer "$TMP/before" ! -path "*/__pycache__/*" -print)
|
||||
if [ -n "$written" ]; then
|
||||
echo "a player's build wrote into the AppImage's tree, which is read-only on a player's PC:"
|
||||
echo "$written" | head -20
|
||||
exit 1
|
||||
fi
|
||||
echo "a player's build starts from the AppImage's tree"
|
||||
208
scripts/linux/deps.sh
Executable file
208
scripts/linux/deps.sh
Executable file
@@ -0,0 +1,208 @@
|
||||
#!/bin/bash
|
||||
# What the Linux AppImage carries, fetched and built into build-linux/deps (or $DEPS): each part once, kept while its
|
||||
# version stays the same. Run on the oldest system the AppImage is for (CI: Ubuntu 22.04, glibc 2.35).
|
||||
#
|
||||
# emsdk/upstream/ LLVM's clang and lld (the layout third_party/a64-kit/tools/toolchain.py finds through EMSDK):
|
||||
# the player's PC compiles the game with them. The play profile (work/hookshot_pieces.prof) wants
|
||||
# LLVM 23 or newer.
|
||||
# python/ CPython (python-build-standalone) with lz4, numpy, capstone, unicorn and zstandard: the tools
|
||||
# (and nx2/prompt_requirements.txt's, which draw the button prompts)
|
||||
# tk/ libtcl9tk9.0.so built with Xft, with libXft and libXrender: put over python/'s, which draws only
|
||||
# X core fonts (under XWayland one 13 px bitmap font, whatever the size asked for or the screen)
|
||||
# ffmpeg/ FFmpeg's libavcodec, libswscale, libswresample, libavutil, static and LGPL, with dav1d for OoT's
|
||||
# AV1 movies (VP8 / VP9 too) - linked into the game
|
||||
# sdl2/ libSDL2-2.0.so.0: the window, sound and controllers, placed beside the game
|
||||
# cpm/ Vulkan-Headers, SPIRV-Headers and VulkanMemoryAllocator, in the CPM cache layout nx2/build.py reads
|
||||
# appimagetool the AppImage packer
|
||||
# runtime-x86_64 the AppImage runtime it puts in front of the image (pinned: appimagetool's own download of the
|
||||
# 'continuous' one, at packing time, fails whenever GitHub answers 504)
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/../.."
|
||||
DEPS=${DEPS:-$PWD/build-linux/deps}
|
||||
DL=$DEPS/downloads
|
||||
mkdir -p "$DEPS" "$DL"
|
||||
JOBS=$(nproc)
|
||||
|
||||
LLVM_VER=23.1.5
|
||||
PY_TAG=20261026
|
||||
PY_VER=3.12.16
|
||||
TCLTK_VER=9.0.4 # (python-build-standalone's Tcl and Tk: its tk9.0 scripts stay, beside this libtcl9tk9.0.so)
|
||||
TCL_SHA256=d0aed49230bc02a65c1e0229e65f34590a4b037ec40d546f32573b467f7551ea
|
||||
TK_SHA256=d7a146d2917eb8b5cc95276dbf0e3d03c7464d2b19c1675357857c989301dbb4
|
||||
SDL_VER=2.32.10
|
||||
FFMPEG_VER=9.0.2
|
||||
DAV1D_VER=1.5.4
|
||||
VK_HEADERS=v1.4.345
|
||||
SPIRV_HEADERS=vulkan-sdk-1.4.341.0
|
||||
VMA=v3.3.0
|
||||
APPIMAGETOOL=1.9.1
|
||||
APPIMAGE_RUNTIME=20251108
|
||||
APPIMAGE_RUNTIME_SHA256=2fca8b443c92510f1483a883f60061ad09b46b978b2631c807cd873a47ec260d
|
||||
# (the SHA-256 of each release download: a part whose download differs stops the script. Change them with the versions.
|
||||
# GitHub's and GitLab's tag archives (dav1d, the headers) are made on demand and not pinned here)
|
||||
LLVM_SHA256=f72edb0b36580ece7f52e23858f00c18fad4e87395ea7ad05d7e55c39f8fc78a
|
||||
PY_SHA256=ce4f6d0a6cfdb3bed6e44c2f9affd85943c76a42e25ad3a259002f31d6fb8a20
|
||||
FFMPEG_SHA256=8c3850283eb25fa026482078a04051e0be17347b09ef81a0849bec15a96e002e
|
||||
SDL_SHA256=5f5993c530f084535c65a6879e9b26ad441169b3e25d789d83287040a9ca5165
|
||||
APPIMAGETOOL_SHA256=ed4ce84f0d9caff66f50bcca6ff6f35aae54ce8135408b3fa33abfc3cb384eb0
|
||||
|
||||
CURL=(curl -fL --retry 5 --retry-delay 10 --proto '=https')
|
||||
# A GitHub release link (github.com/O/R/releases/download/TAG/NAME) can answer 504 for a long while: the same file is
|
||||
# then fetched through the GitHub API (with $GH_TOKEN when it is set).
|
||||
fetch_github_api() { # url out
|
||||
[[ $1 =~ ^https://github\.com/([^/]+/[^/]+)/releases/download/([^/]+)/([^/]+)$ ]] || return 1
|
||||
local repo=${BASH_REMATCH[1]} tag=${BASH_REMATCH[2]} name=${BASH_REMATCH[3]} auth=() id
|
||||
[ -n "${GH_TOKEN:-}" ] && auth=(-H "Authorization: Bearer $GH_TOKEN")
|
||||
id=$("${CURL[@]}" -sS "${auth[@]}" "https://api.github.com/repos/$repo/releases/tags/$tag" |
|
||||
python3 -c 'import json,sys; print(next(a["id"] for a in json.load(sys.stdin)["assets"] if a["name"] == sys.argv[1]))' "$name") ||
|
||||
return 1
|
||||
echo "release link failed: fetching $name through the GitHub API"
|
||||
"${CURL[@]}" "${auth[@]}" -H "Accept: application/octet-stream" -o "$2" "https://api.github.com/repos/$repo/releases/assets/$id"
|
||||
}
|
||||
fetch() { # url file [sha256: checked on every run, the download removed when it differs]
|
||||
if [ ! -f "$DL/$2" ]; then
|
||||
"${CURL[@]}" -o "$DL/$2.part" "$1" || fetch_github_api "$1" "$DL/$2.part"
|
||||
mv -f "$DL/$2.part" "$DL/$2"
|
||||
fi
|
||||
if [ -n "${3:-}" ] && ! echo "$3 $DL/$2" | sha256sum -c --quiet -; then
|
||||
rm -f "$DL/$2"
|
||||
echo "$2: not the expected download (SHA-256 differs)"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
done_for() { [ -f "$DEPS/.$1" ] && [ "$(cat "$DEPS/.$1")" = "$2" ]; }
|
||||
mark() { echo "$2" > "$DEPS/.$1"; }
|
||||
|
||||
# ---- LLVM: clang, lld and clang's own headers, nothing else ----
|
||||
if ! done_for llvm "$LLVM_VER"; then
|
||||
fetch "https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VER/LLVM-$LLVM_VER-Linux-X64.tar.xz" llvm.tar.xz "$LLVM_SHA256"
|
||||
rm -rf "$DEPS/emsdk" "$DEPS/llvm-x"
|
||||
mkdir -p "$DEPS/llvm-x"
|
||||
tar -xJf "$DL/llvm.tar.xz" -C "$DEPS/llvm-x" --strip-components=1 --wildcards \
|
||||
'*/bin/clang' '*/bin/clang++' '*/bin/clang-[0-9]*' '*/bin/lld' '*/bin/ld.lld' '*/bin/llvm-profdata' '*/lib/clang/*'
|
||||
U=$DEPS/emsdk/upstream
|
||||
mkdir -p "$U/bin" "$U/lib"
|
||||
mv "$DEPS/llvm-x/bin/"* "$U/bin/"
|
||||
mv "$DEPS/llvm-x/lib/clang" "$U/lib/clang"
|
||||
rm -rf "$DEPS/llvm-x"
|
||||
# (the runtimes: only the profile and builtins libraries; no sanitizers)
|
||||
find "$U"/lib/clang/*/lib -type f ! -name 'libclang_rt.profile*.a' ! -name 'libclang_rt.builtins*.a' -delete
|
||||
strip "$U/bin/clang-"[0-9]* "$U/bin/lld" "$U/bin/llvm-profdata" 2>/dev/null || true
|
||||
"$U/bin/clang" --version | head -1
|
||||
[ -f "$U/lib/clang/${LLVM_VER%%.*}/include/stddef.h" ] || { echo "clang's own headers are missing"; exit 1; }
|
||||
mark llvm "$LLVM_VER"
|
||||
fi
|
||||
|
||||
# ---- Python and the tools' modules ----
|
||||
PY_MARK="$PY_VER-$PY_TAG-$(sha256sum requirements.txt nx2/prompt_requirements.txt | sha256sum | cut -c1-12)" # (a new pin installs again)
|
||||
if ! done_for python "$PY_MARK"; then
|
||||
fetch "https://github.com/astral-sh/python-build-standalone/releases/download/$PY_TAG/cpython-$PY_VER+$PY_TAG-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz" python.tar.gz "$PY_SHA256"
|
||||
rm -rf "$DEPS/python"
|
||||
tar -xzf "$DL/python.tar.gz" -C "$DEPS"
|
||||
"$DEPS/python/bin/python3" -m pip install --no-cache-dir --only-binary :all: --disable-pip-version-check -q \
|
||||
-r requirements.txt
|
||||
rm -rf "$DEPS/python/lib/python3.12/test" "$DEPS/python/lib/python3.12/idlelib" "$DEPS/python/lib/python3.12/ensurepip"
|
||||
find "$DEPS/python" -name __pycache__ -prune -exec rm -rf {} +
|
||||
mark python "$PY_MARK"
|
||||
fi
|
||||
|
||||
# ---- Tk with Xft (the builder window's fonts; Tcl is built only for Tk's configure and stub library) ----
|
||||
if ! done_for tk "$TCLTK_VER"; then
|
||||
fetch "https://downloads.sourceforge.net/project/tcl/Tcl/$TCLTK_VER/tcl$TCLTK_VER-src.tar.gz" "tcl$TCLTK_VER.tar.gz"
|
||||
fetch "https://downloads.sourceforge.net/project/tcl/Tcl/$TCLTK_VER/tk$TCLTK_VER-src.tar.gz" "tk$TCLTK_VER.tar.gz"
|
||||
printf '%s %s\n' "$TCL_SHA256" "$DL/tcl$TCLTK_VER.tar.gz" "$TK_SHA256" "$DL/tk$TCLTK_VER.tar.gz" | sha256sum -c -
|
||||
rm -rf "$DEPS/src-tcl" "$DEPS/src-tk" "$DEPS/tk"
|
||||
mkdir -p "$DEPS/src-tcl" "$DEPS/src-tk" "$DEPS/tk"
|
||||
tar -xzf "$DL/tcl$TCLTK_VER.tar.gz" -C "$DEPS/src-tcl" --strip-components=1
|
||||
tar -xzf "$DL/tk$TCLTK_VER.tar.gz" -C "$DEPS/src-tk" --strip-components=1
|
||||
(cd "$DEPS/src-tcl/unix" && ./configure --enable-shared --with-system-libtommath=no && make -j"$JOBS" binaries)
|
||||
# (no rpath: libXft and libXrender are found beside it through _tkinter's $ORIGIN/../..)
|
||||
(cd "$DEPS/src-tk/unix" && ./configure --enable-shared --with-tcl="$DEPS/src-tcl/unix" --enable-xft --disable-libcups \
|
||||
--disable-xss --disable-zipfs --disable-rpath && make -j"$JOBS" binaries)
|
||||
cp "$DEPS/src-tk/unix/libtcl9tk9.0.so" "$DEPS/tk/"
|
||||
strip --strip-unneeded "$DEPS/tk/libtcl9tk9.0.so"
|
||||
readelf -d "$DEPS/tk/libtcl9tk9.0.so" | grep -q 'libXft\.so\.2' || { echo "Tk was built without Xft"; exit 1; }
|
||||
cp -L /usr/lib/x86_64-linux-gnu/libXft.so.2 /usr/lib/x86_64-linux-gnu/libXrender.so.1 "$DEPS/tk/"
|
||||
cp "$DEPS/src-tk/license.terms" "$DEPS/tk/Tk-license.terms"
|
||||
rm -rf "$DEPS/src-tcl" "$DEPS/src-tk"
|
||||
mark tk "$TCLTK_VER"
|
||||
fi
|
||||
|
||||
# ---- dav1d and FFmpeg (static, position independent: linked into the game's executable) ----
|
||||
P=$DEPS/ffmpeg
|
||||
if ! done_for ffmpeg "$FFMPEG_VER-$DAV1D_VER"; then
|
||||
rm -rf "$P" "$DEPS/src-dav1d" "$DEPS/src-ffmpeg"
|
||||
fetch "https://code.videolan.org/videolan/dav1d/-/archive/$DAV1D_VER/dav1d-$DAV1D_VER.tar.gz" dav1d.tar.gz
|
||||
mkdir -p "$DEPS/src-dav1d" && tar -xzf "$DL/dav1d.tar.gz" -C "$DEPS/src-dav1d" --strip-components=1
|
||||
(cd "$DEPS/src-dav1d" && meson setup build --prefix="$P" --libdir=lib --buildtype=release --default-library=static \
|
||||
-Db_staticpic=true -Denable_tools=false -Denable_tests=false -Denable_examples=false && ninja -C build install)
|
||||
fetch "https://ffmpeg.org/releases/ffmpeg-$FFMPEG_VER.tar.xz" ffmpeg.tar.xz "$FFMPEG_SHA256"
|
||||
mkdir -p "$DEPS/src-ffmpeg" && tar -xJf "$DL/ffmpeg.tar.xz" -C "$DEPS/src-ffmpeg" --strip-components=1
|
||||
(cd "$DEPS/src-ffmpeg" && PKG_CONFIG_PATH="$P/lib/pkgconfig" ./configure --prefix="$P" --pkg-config-flags=--static \
|
||||
--enable-static --disable-shared --enable-pic --disable-programs --disable-doc --disable-network --disable-autodetect \
|
||||
--disable-everything --disable-avformat --disable-avfilter --disable-avdevice \
|
||||
--enable-avcodec --enable-swscale --enable-swresample --enable-avutil \
|
||||
--enable-libdav1d --enable-decoder=vp8,vp9,libdav1d --enable-parser=vp8,vp9,av1 \
|
||||
&& make -j"$JOBS" && make install)
|
||||
rm -rf "$DEPS/src-dav1d" "$DEPS/src-ffmpeg" "$P/share" "$P/lib/pkgconfig"
|
||||
mark ffmpeg "$FFMPEG_VER-$DAV1D_VER"
|
||||
fi
|
||||
|
||||
# ---- SDL2 (its X11, Wayland and sound back ends load their libraries at run time) ----
|
||||
if ! done_for sdl2 "$SDL_VER"; then
|
||||
fetch "https://github.com/libsdl-org/SDL/releases/download/release-$SDL_VER/SDL2-$SDL_VER.tar.gz" sdl2.tar.gz "$SDL_SHA256"
|
||||
rm -rf "$DEPS/src-sdl2" "$DEPS/sdl2"
|
||||
mkdir -p "$DEPS/src-sdl2" && tar -xzf "$DL/sdl2.tar.gz" -C "$DEPS/src-sdl2" --strip-components=1
|
||||
cmake -S "$DEPS/src-sdl2" -B "$DEPS/src-sdl2/build" -G Ninja -DCMAKE_BUILD_TYPE=Release -DSDL_SHARED=ON -DSDL_STATIC=OFF \
|
||||
-DSDL_TEST=OFF -DSDL_X11=ON -DSDL_WAYLAND=ON -DSDL_PULSEAUDIO=ON -DSDL_PIPEWIRE=ON -DSDL_ALSA=ON
|
||||
ninja -C "$DEPS/src-sdl2/build"
|
||||
mkdir -p "$DEPS/sdl2"
|
||||
cp -L "$DEPS/src-sdl2/build/libSDL2-2.0.so.0" "$DEPS/sdl2/libSDL2-2.0.so.0"
|
||||
strip "$DEPS/sdl2/libSDL2-2.0.so.0"
|
||||
cp "$DEPS/src-sdl2/LICENSE.txt" "$DEPS/sdl2/LICENSE.txt"
|
||||
rm -rf "$DEPS/src-sdl2"
|
||||
mark sdl2 "$SDL_VER"
|
||||
fi
|
||||
|
||||
# ---- the Vulkan back end's headers ----
|
||||
if ! done_for cpm "$VK_HEADERS-$SPIRV_HEADERS-$VMA"; then
|
||||
rm -rf "$DEPS/cpm"
|
||||
get_headers() { # repo tag name
|
||||
fetch "https://github.com/$1/archive/refs/tags/$2.tar.gz" "$3-$2.tar.gz"
|
||||
mkdir -p "$DEPS/cpm/$3/$2"
|
||||
tar -xzf "$DL/$3-$2.tar.gz" -C "$DEPS/cpm/$3/$2" --strip-components=1
|
||||
}
|
||||
get_headers KhronosGroup/Vulkan-Headers "$VK_HEADERS" vulkanheaders
|
||||
get_headers KhronosGroup/SPIRV-Headers "$SPIRV_HEADERS" spirv-headers
|
||||
get_headers GPUOpen-LibrariesAndSDKs/VulkanMemoryAllocator "$VMA" vulkanmemoryallocator
|
||||
mark cpm "$VK_HEADERS-$SPIRV_HEADERS-$VMA"
|
||||
fi
|
||||
|
||||
# ---- the licenses of what the AppImage carries (Python's and its modules' are in python/) ----
|
||||
if ! done_for licenses "$LLVM_VER-$FFMPEG_VER-$DAV1D_VER-$SDL_VER-$TCLTK_VER"; then
|
||||
L=$DEPS/licenses
|
||||
rm -rf "$L" && mkdir -p "$L"
|
||||
curl -fsSL -o "$L/LLVM-LICENSE.TXT" "https://raw.githubusercontent.com/llvm/llvm-project/llvmorg-$LLVM_VER/llvm/LICENSE.TXT"
|
||||
curl -fsSL -o "$L/FFmpeg-COPYING.LGPLv2.1" "https://raw.githubusercontent.com/FFmpeg/FFmpeg/n$FFMPEG_VER/COPYING.LGPLv2.1"
|
||||
curl -fsSL -o "$L/FFmpeg-LICENSE.md" "https://raw.githubusercontent.com/FFmpeg/FFmpeg/n$FFMPEG_VER/LICENSE.md"
|
||||
curl -fsSL -o "$L/dav1d-COPYING" "https://code.videolan.org/videolan/dav1d/-/raw/$DAV1D_VER/COPYING"
|
||||
cp "$DEPS/sdl2/LICENSE.txt" "$L/SDL2-LICENSE.txt"
|
||||
cp "$DEPS/tk/Tk-license.terms" "$L/Tk-license.terms"
|
||||
cp /usr/share/doc/libxft2/copyright "$L/libXft-copyright"
|
||||
cp /usr/share/doc/libxrender1/copyright "$L/libXrender-copyright"
|
||||
mark licenses "$LLVM_VER-$FFMPEG_VER-$DAV1D_VER-$SDL_VER-$TCLTK_VER"
|
||||
fi
|
||||
|
||||
# ---- appimagetool ----
|
||||
if ! done_for appimagetool "$APPIMAGETOOL"; then
|
||||
fetch "https://github.com/AppImage/appimagetool/releases/download/$APPIMAGETOOL/appimagetool-x86_64.AppImage" appimagetool "$APPIMAGETOOL_SHA256"
|
||||
cp "$DL/appimagetool" "$DEPS/appimagetool" && chmod +x "$DEPS/appimagetool"
|
||||
mark appimagetool "$APPIMAGETOOL"
|
||||
fi
|
||||
if ! done_for runtime "$APPIMAGE_RUNTIME"; then
|
||||
fetch "https://github.com/AppImage/type2-runtime/releases/download/$APPIMAGE_RUNTIME/runtime-x86_64" "runtime-x86_64-$APPIMAGE_RUNTIME"
|
||||
echo "$APPIMAGE_RUNTIME_SHA256 $DL/runtime-x86_64-$APPIMAGE_RUNTIME" | sha256sum -c -
|
||||
cp "$DL/runtime-x86_64-$APPIMAGE_RUNTIME" "$DEPS/runtime-x86_64"
|
||||
mark runtime "$APPIMAGE_RUNTIME"
|
||||
fi
|
||||
echo "deps in $DEPS: $(du -sh --exclude=downloads "$DEPS" | cut -f1)"
|
||||
117
scripts/linux/make_parts.py
Normal file
117
scripts/linux/make_parts.py
Normal file
@@ -0,0 +1,117 @@
|
||||
"""The parts a Linux update downloads (builder/linux/linux_update.py), from an AppDir.
|
||||
|
||||
python3 scripts/linux/make_parts.py <AppDir> <tag> <out folder> [--base <url>]
|
||||
|
||||
Each directory that changes on its own is one zip: the tools (every release), Python, clang, FFmpeg, the sysroot, and
|
||||
the two libraries beside them. A zip is deterministic, so a directory whose files did not change has the same SHA-256
|
||||
next time and the builder copies the one it already has. The zips are uploaded to the linux-parts release; the version's
|
||||
own release carries hookshot-linux.txt, which names them.
|
||||
|
||||
The list is not inside a zip (it names the zips' checksums).
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import os
|
||||
import sys
|
||||
import zipfile
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
|
||||
"builder", "linux"))
|
||||
from part_links import SYSROOT_PART, is_absolute_link, relative_sysroot_link # noqa: E402
|
||||
|
||||
# (dest under the AppDir, and None for the whole directory or the file names when the directory holds other parts too)
|
||||
PARTS = (
|
||||
("usr/share/hookshot", None),
|
||||
("usr/python", None),
|
||||
("usr/lib/hookshot/emsdk", None),
|
||||
("usr/lib/hookshot/ffmpeg", None),
|
||||
("usr/lib/hookshot/sysroot", None),
|
||||
("usr/lib/hookshot", ("hookshot_vk.so", "libSDL2-2.0.so.0")),
|
||||
)
|
||||
|
||||
|
||||
def _entries(root, dest, files):
|
||||
base = os.path.join(root, dest)
|
||||
if files is not None:
|
||||
return [(name, os.path.join(base, name)) for name in files]
|
||||
out = []
|
||||
for dirpath, dirnames, names in os.walk(base, followlinks=False):
|
||||
rel = os.path.relpath(dirpath, base)
|
||||
real = []
|
||||
for name in sorted(dirnames):
|
||||
link = os.path.join(dirpath, name)
|
||||
arc = name if rel == os.curdir else rel.replace(os.sep, "/") + "/" + name
|
||||
if os.path.islink(link):
|
||||
out.append((arc, link))
|
||||
else:
|
||||
real.append(name)
|
||||
dirnames[:] = real
|
||||
for name in sorted(names):
|
||||
full = os.path.join(dirpath, name)
|
||||
arc = name if rel == os.curdir else rel.replace(os.sep, "/") + "/" + name
|
||||
out.append((arc, full))
|
||||
return out
|
||||
|
||||
|
||||
def write_zip(path, root, dest, files):
|
||||
entries = _entries(root, dest, files)
|
||||
with zipfile.ZipFile(path + ".new", "w", zipfile.ZIP_DEFLATED, compresslevel=6) as zipped:
|
||||
for arc, full in entries:
|
||||
info = zipfile.ZipInfo(arc, date_time=(2026, 1, 1, 0, 0, 0))
|
||||
info.compress_type = zipfile.ZIP_DEFLATED
|
||||
info.create_system = 3
|
||||
if os.path.islink(full):
|
||||
info.external_attr = 0o120777 << 16
|
||||
link_to = os.readlink(full)
|
||||
if dest == SYSROOT_PART and is_absolute_link(link_to):
|
||||
link_to = relative_sysroot_link(os.path.join(root, dest), full, link_to)
|
||||
zipped.writestr(info, link_to, compresslevel=6)
|
||||
continue
|
||||
info.external_attr = (0o755 if os.access(full, os.X_OK) else 0o644) << 16
|
||||
with open(full, "rb") as handle:
|
||||
zipped.writestr(info, handle.read(), compresslevel=6)
|
||||
os.replace(path + ".new", path)
|
||||
digest = hashlib.sha256()
|
||||
with open(path, "rb") as handle:
|
||||
while True:
|
||||
block = handle.read(1 << 20)
|
||||
if not block:
|
||||
break
|
||||
digest.update(block)
|
||||
return digest.hexdigest(), os.path.getsize(path)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("appdir")
|
||||
parser.add_argument("tag")
|
||||
parser.add_argument("out")
|
||||
parser.add_argument("--base", default="https://github.com/NotRexed/Hookshot/releases/download/linux-parts")
|
||||
args = parser.parse_args()
|
||||
root = os.path.abspath(args.appdir)
|
||||
if not os.path.isfile(os.path.join(root, "AppRun")):
|
||||
sys.exit("not an AppDir: %s" % root)
|
||||
os.makedirs(args.out, exist_ok=True)
|
||||
base = args.base.rstrip("/")
|
||||
lines = ["tag %s" % args.tag]
|
||||
for dest, files in PARTS:
|
||||
if not os.path.isdir(os.path.join(root, dest)):
|
||||
sys.exit("the AppDir has no %s" % dest)
|
||||
leaf = dest.replace("/", "-")
|
||||
path = os.path.join(args.out, "HookshotLinux-%s.zip" % leaf)
|
||||
digest, size = write_zip(path, root, dest, files)
|
||||
named = os.path.join(args.out, "HookshotLinux-%s-%s.zip" % (leaf, digest[:12]))
|
||||
os.replace(path, named)
|
||||
lines.append("part %s %d %s %s/%s" % (digest, size, dest, base, os.path.basename(named)))
|
||||
lines.append("tree" if files is None else "\n".join("file %s" % name for name in files))
|
||||
print("%-32s %5d MB %s" % (dest, size >> 20, digest[:16]), flush=True)
|
||||
text = "\n".join(lines) + "\n"
|
||||
with open(os.path.join(args.out, "hookshot-linux.txt"), "w", encoding="utf-8", newline="\n") as handle:
|
||||
handle.write(text)
|
||||
# (beside AppRun, not inside a hashed part: the list names those checksums)
|
||||
with open(os.path.join(root, "hookshot-linux.txt"), "w", encoding="utf-8", newline="\n") as handle:
|
||||
handle.write(text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
15
scripts/linux/ngx-sdk.sh
Executable file
15
scripts/linux/ngx-sdk.sh
Executable file
@@ -0,0 +1,15 @@
|
||||
#!/bin/bash
|
||||
# ngx-sdk.sh: NVIDIA's DLSS SDK files for a local Linux build with NVIDIA DLSS (HOOKSHOT_NGX_SDK): scripts/ngx_sdk.py, which
|
||||
# says what it fetches and asks for NVIDIA's license (--accept-nvidia-license: accepted without the question).
|
||||
#
|
||||
# Usage: scripts/linux/ngx-sdk.sh [--accept-nvidia-license] [<sdk folder>] [<runtime folder>]
|
||||
# <sdk folder>: default build-linux/ngx-sdk; <runtime folder>: libnvidia-ngx-dlss.so copied there too (a game folder's
|
||||
# ngx/, or $XDG_DATA_HOME/hookshot/ngx; RX_VK_DLSS_DIR names another)
|
||||
# Then build with HOOKSHOT_NGX_SDK=<sdk folder>.
|
||||
set -euo pipefail
|
||||
ROOT=$(cd "$(dirname "$0")/../.." && pwd)
|
||||
accept=()
|
||||
if [ "${1:-}" = --accept-nvidia-license ]; then accept=(--accept-nvidia-license); shift; fi
|
||||
args=(--platform linux "${accept[@]}")
|
||||
[ -n "${2:-}" ] && args+=(--runtime-dir "$2")
|
||||
exec "${PYTHON:-python3}" -I "$ROOT/scripts/ngx_sdk.py" "${args[@]}" ${1:+"$1"}
|
||||
47
scripts/linux/publish_parts.sh
Executable file
47
scripts/linux/publish_parts.sh
Executable file
@@ -0,0 +1,47 @@
|
||||
#!/bin/bash
|
||||
# Upload the Linux builder's parts (scripts/linux/make_parts.py) to the linux-parts release, which is not "latest".
|
||||
# A zip whose name is already there is left (its checksum is in the name). hookshot-linux.txt itself goes on the
|
||||
# version release, with the AppImage. Run from CI on a tag, with GH_TOKEN and GITHUB_REPOSITORY.
|
||||
set -euo pipefail
|
||||
ROOT=$(cd "$(dirname "$0")/../.." && pwd)
|
||||
PARTS=${1:-$ROOT/build-linux/parts}
|
||||
REPO=${GITHUB_REPOSITORY:?}
|
||||
TOKEN=${GH_TOKEN:?}
|
||||
API="https://api.github.com/repos/$REPO"
|
||||
AUTH=(-H "Authorization: Bearer $TOKEN" -H "Accept: application/vnd.github+json" -H "User-Agent: Hookshot")
|
||||
J=$(mktemp) # (the release's JSON: a private file, not a fixed name in /tmp)
|
||||
trap 'rm -f "$J"' EXIT
|
||||
|
||||
code=$(curl -s -o "$J" -w '%{http_code}' "${AUTH[@]}" "$API/releases/tags/linux-parts")
|
||||
if [ "$code" = "404" ]; then
|
||||
curl -fsSL "${AUTH[@]}" -X POST "$API/releases" -d "$(python3 - <<'PY'
|
||||
import json, os
|
||||
print(json.dumps({
|
||||
"tag_name": "linux-parts",
|
||||
"target_commitish": os.environ["GITHUB_SHA"],
|
||||
"name": "Linux builder parts",
|
||||
"body": "Parts Hookshot's Linux builder downloads. A file's name is its checksum; a part that did not change is not uploaded again. This release is not a version of the game.",
|
||||
"prerelease": True,
|
||||
"make_latest": "false",
|
||||
}))
|
||||
PY
|
||||
)" > "$J"
|
||||
fi
|
||||
python3 - "$J" <<'PY'
|
||||
import json, sys
|
||||
json.load(open(sys.argv[1]))["upload_url"]
|
||||
PY
|
||||
upload=$(python3 -c 'import json, sys; print(json.load(open(sys.argv[1]))["upload_url"].split("{")[0])' "$J")
|
||||
have=$(python3 -c 'import json, sys; print("\n".join(a["name"] for a in json.load(open(sys.argv[1])).get("assets", [])))' "$J")
|
||||
for zip in "$PARTS"/HookshotLinux-*.zip; do
|
||||
[ -f "$zip" ] || continue
|
||||
name=$(basename "$zip")
|
||||
if printf '%s\n' "$have" | grep -qx "$name"; then
|
||||
echo "already on linux-parts: $name"
|
||||
continue
|
||||
fi
|
||||
echo "uploading $name"
|
||||
curl -fsSL --retry 5 --retry-delay 10 -X POST "${AUTH[@]}" -H "Content-Type: application/zip" \
|
||||
--data-binary @"$zip" "$upload?name=$name" >/dev/null
|
||||
done
|
||||
echo "parts published"
|
||||
73
scripts/make_py_builder.py
Normal file
73
scripts/make_py_builder.py
Normal file
@@ -0,0 +1,73 @@
|
||||
"""The Python Hookshot Builder's download: one zip with python.org's own Python (python.exe and pythonw.exe, signed by the Python
|
||||
Software Foundation), Qt for Python's DLLs (signed by The Qt Company), the builder's code (builder/py) and the .bat that starts it.
|
||||
Nothing in it is an executable of Hookshot's own.
|
||||
|
||||
python -I scripts/make_py_builder.py --python <python folder> --qt <PySide6 site folder> --tag <tag> <out zip>
|
||||
|
||||
--python: a python.org Windows embeddable Python of the version the build tools use (its pythonw.exe starts the window without a
|
||||
console). --qt: a folder holding PySide6 and shiboken6 (the two wheels unpacked). Only the parts of Qt the window uses are taken:
|
||||
QtCore, QtGui, QtWidgets, the Windows platform plugin and the icon and image plugins (~40 MB of the wheel's ~200)."""
|
||||
import argparse
|
||||
import os
|
||||
import zipfile
|
||||
|
||||
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
TOP = "Hookshot Builder"
|
||||
QT_KEEP_DLLS = ("Qt6Core.dll", "Qt6Gui.dll", "Qt6Widgets.dll", "Qt6Svg.dll", "pyside6.abi3.dll", "msvcp140.dll", "msvcp140_1.dll",
|
||||
"msvcp140_2.dll", "vcruntime140.dll", "vcruntime140_1.dll", "concrt140.dll")
|
||||
QT_KEEP_PY = ("QtCore.pyd", "QtGui.pyd", "QtWidgets.pyd", "__init__.py", "_config.py", "_git_pyside_version.py")
|
||||
QT_PLUGINS = ("platforms/qwindows.dll", "imageformats/qico.dll", "imageformats/qjpeg.dll", "imageformats/qsvg.dll",
|
||||
"styles/qmodernwindowsstyle.dll")
|
||||
BAT = ('@echo off\r\n'
|
||||
'rem Hookshot Builder: python.org\'s pythonw.exe runs the builder (app\\hookshot_builder_start.py) with no console window.\r\n'
|
||||
'start "" "%~dp0python\\pythonw.exe" -I "%~dp0app\\hookshot_builder_start.py" %*\r\n')
|
||||
|
||||
|
||||
def add_tree(z, src, dst, keep=lambda rel: True):
|
||||
n = 0
|
||||
for d, ds, fs in os.walk(src):
|
||||
ds[:] = sorted(x for x in ds if x != "__pycache__")
|
||||
for f in sorted(fs):
|
||||
full = os.path.join(d, f)
|
||||
rel = os.path.relpath(full, src).replace(os.sep, "/")
|
||||
if keep(rel):
|
||||
z.write(full, "%s/%s/%s" % (TOP, dst, rel))
|
||||
n += 1
|
||||
return n
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--python", required=True)
|
||||
p.add_argument("--qt", required=True)
|
||||
p.add_argument("--tag", required=True)
|
||||
p.add_argument("out")
|
||||
a = p.parse_args()
|
||||
for need in ("python.exe", "pythonw.exe"):
|
||||
if not os.path.isfile(os.path.join(a.python, need)):
|
||||
raise SystemExit("%s has no %s (use python.org's Windows embeddable package)" % (a.python, need))
|
||||
pyside, shiboken = os.path.join(a.qt, "PySide6"), os.path.join(a.qt, "shiboken6")
|
||||
if not os.path.isdir(pyside) or not os.path.isdir(shiboken):
|
||||
raise SystemExit("%s has no PySide6 and shiboken6" % a.qt)
|
||||
|
||||
def qt_keep(rel):
|
||||
if "/" not in rel:
|
||||
return rel in QT_KEEP_DLLS or rel in QT_KEEP_PY
|
||||
return rel.startswith("plugins/") and rel[len("plugins/"):] in QT_PLUGINS
|
||||
|
||||
with zipfile.ZipFile(a.out + ".new", "w", zipfile.ZIP_DEFLATED, compresslevel=9) as z:
|
||||
z.writestr("%s/Hookshot Builder.bat" % TOP, BAT)
|
||||
n = add_tree(z, a.python, "python")
|
||||
n += add_tree(z, pyside, "qt/PySide6", qt_keep)
|
||||
n += add_tree(z, shiboken, "qt/shiboken6")
|
||||
py = os.path.join(ROOT, "builder", "py")
|
||||
n += add_tree(z, py, "app", lambda rel: rel not in ("gen_strings.py", "shots.py"))
|
||||
z.writestr("%s/app/VERSION" % TOP, a.tag + "\n")
|
||||
for lic in ("LICENSE",):
|
||||
z.write(os.path.join(ROOT, lic), "%s/%s" % (TOP, lic))
|
||||
os.replace(a.out + ".new", a.out)
|
||||
print("%s: %d files, %d MB" % (a.out, n, os.path.getsize(a.out) >> 20))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
122
scripts/make_setup_parts.py
Normal file
122
scripts/make_setup_parts.py
Normal file
@@ -0,0 +1,122 @@
|
||||
"""The parts HookshotBuilder-Setup.exe downloads (builder/setup/setup.c), from a Hookshot Builder package folder.
|
||||
|
||||
python -I scripts/make_setup_parts.py <package folder> <tag> <out folder> [--base <url the parts are served from>]
|
||||
[--reuse <an earlier release's hookshot-setup.txt>]
|
||||
|
||||
A release is split by how often each part changes, so an update downloads only what changed: the builder's own code (a few MB)
|
||||
in every release, the .NET runtime, Python, emsdk, the CPM packages and third_party only when they change. llvm-mingw comes from
|
||||
its official release. Each zip is deterministic (sorted names, fixed dates and attributes): a part whose files did not change
|
||||
has the same SHA-256 in the next release, and the setup takes it from its download cache instead of downloading it again.
|
||||
|
||||
Writes <out>/hookshot-setup.txt and one HookshotBuilder-<part>-<sha256 prefix>.zip per part (upload the ones the builder-parts
|
||||
release lacks). Scripts the build does not use (.ps1, .bat, .js,
|
||||
.html: antivirus machine-learning guesses fire on them) are left out.
|
||||
|
||||
--reuse: a part whose tools/<name> folder the package lacks is taken from that list as it is (its line, with its URL), so a
|
||||
package made in CI (.github/workflows/windows-builder.yml) carries the Python, emsdk, CPM and wgpu-native parts the maintainer
|
||||
made. Without the folder and without the part in that list it is an error.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import os
|
||||
import sys
|
||||
import zipfile
|
||||
|
||||
SKIP_EXT = {".ps1", ".psm1", ".bat", ".cmd", ".js", ".vbs", ".html", ".htm", ".hta", ".wsf", ".pyc"}
|
||||
SKIP_DIRS = {"__pycache__"}
|
||||
TOOLS_PARTS = ("python", "emsdk", "cpm", "third_party", "wgpu-native") # (tools/<name>: each its own part)
|
||||
LLVM = ("552191b5754c49a5a20723276a6fc56541ea1922c4c50b7b669022333fc1b40f", 191037046,
|
||||
"llvm-mingw-20261104-ucrt-x86_64",
|
||||
"https://github.com/mstorsjo/llvm-mingw/releases/download/20261104/llvm-mingw-20261104-ucrt-x86_64.zip")
|
||||
|
||||
|
||||
def files_under(root, top):
|
||||
out = []
|
||||
for d, ds, fs in os.walk(os.path.join(root, top)):
|
||||
ds[:] = sorted(x for x in ds if x not in SKIP_DIRS)
|
||||
for f in fs:
|
||||
if os.path.splitext(f)[1].lower() not in SKIP_EXT:
|
||||
out.append(os.path.relpath(os.path.join(d, f), root).replace(os.sep, "/"))
|
||||
return out
|
||||
|
||||
|
||||
def write_zip(path, root, names):
|
||||
"""Deterministic: the same files give the same bytes (sorted, a fixed date, fixed attributes)"""
|
||||
with zipfile.ZipFile(path + ".new", "w", zipfile.ZIP_DEFLATED, compresslevel=6) as z:
|
||||
for n in sorted(names):
|
||||
info = zipfile.ZipInfo("HookshotBuilder/" + n, date_time=(2026, 1, 1, 0, 0, 0))
|
||||
info.compress_type = zipfile.ZIP_DEFLATED
|
||||
info.external_attr = 0o644 << 16
|
||||
with open(os.path.join(root, n.replace("/", os.sep)), "rb") as f:
|
||||
z.writestr(info, f.read(), compresslevel=6)
|
||||
os.replace(path + ".new", path)
|
||||
h = hashlib.sha256()
|
||||
with open(path, "rb") as f:
|
||||
while b := f.read(1 << 20):
|
||||
h.update(b)
|
||||
return h.hexdigest(), os.path.getsize(path)
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("package")
|
||||
p.add_argument("tag")
|
||||
p.add_argument("out")
|
||||
p.add_argument("--base", help="where the parts are served from (default: this repository's builder-parts release)")
|
||||
p.add_argument("--reuse", help="an earlier hookshot-setup.txt: the parts the package has no folder for, from it")
|
||||
a = p.parse_args()
|
||||
root = os.path.abspath(a.package)
|
||||
if not os.path.isfile(os.path.join(root, "HookshotBuilder.exe")):
|
||||
sys.exit("not a Hookshot Builder package folder: %s" % root)
|
||||
# (the parts live in one release of their own, named by their checksum: a part that did not change is not uploaded again, and a
|
||||
# release itself holds only HookshotBuilder.exe (the setup) and the list it reads)
|
||||
base = (a.base or "https://github.com/NotRexed/Hookshot/releases/download/builder-parts").rstrip("/")
|
||||
os.makedirs(a.out, exist_ok=True)
|
||||
with open(os.path.join(root, "tools", "version.txt"), "w", newline="\n") as f:
|
||||
f.write(a.tag + "\n")
|
||||
|
||||
top = sorted(os.listdir(root))
|
||||
builder_files = [n for n in top if os.path.isfile(os.path.join(root, n)) and
|
||||
(n.startswith("HookshotBuilder") or n in ("LICENSE", "README.txt"))] # (HookshotBuilder-Setup.exe too: Repair runs it)
|
||||
runtime_files = [n for n in top if os.path.isfile(os.path.join(root, n)) and n not in builder_files]
|
||||
runtime_dirs = [n for n in top if os.path.isdir(os.path.join(root, n)) and n != "tools"]
|
||||
tools = sorted(os.listdir(os.path.join(root, "tools")))
|
||||
parts = {
|
||||
"app": builder_files + [n for t in tools if t not in TOOLS_PARTS and t != "llvm-mingw"
|
||||
for n in (["tools/" + t] if os.path.isfile(os.path.join(root, "tools", t)) else files_under(root, "tools/" + t))],
|
||||
"dotnet": runtime_files + [n for d in runtime_dirs for n in files_under(root, d)],
|
||||
}
|
||||
for t in TOOLS_PARTS:
|
||||
if os.path.isdir(os.path.join(root, "tools", t)):
|
||||
parts[t] = files_under(root, "tools/" + t)
|
||||
reuse = {}
|
||||
if a.reuse:
|
||||
with open(a.reuse, encoding="utf-8") as f:
|
||||
for line in f.read().splitlines():
|
||||
w = line.split()
|
||||
if len(w) == 7 and w[0] == "part" and w[6].rsplit("/", 1)[-1].startswith("HookshotBuilder-"):
|
||||
reuse[w[6].rsplit("/", 1)[-1][len("HookshotBuilder-"):].rsplit("-", 1)[0]] = line
|
||||
lines = ["tag %s" % a.tag]
|
||||
for name in ("app", "dotnet") + TOOLS_PARTS:
|
||||
if not parts.get(name):
|
||||
if a.reuse and name in TOOLS_PARTS:
|
||||
if name not in reuse:
|
||||
sys.exit("no tools/%s in the package and no %s part in %s" % (name, name, a.reuse))
|
||||
lines.append(reuse[name])
|
||||
print("%-12s (as in %s)" % (name, os.path.basename(a.reuse)), flush=True)
|
||||
continue
|
||||
zp = os.path.join(a.out, "HookshotBuilder-%s.zip" % name)
|
||||
sha, size = write_zip(zp, root, parts[name])
|
||||
named = os.path.join(a.out, "HookshotBuilder-%s-%s.zip" % (name, sha[:12]))
|
||||
os.replace(zp, named)
|
||||
zp = named
|
||||
lines.append("part %s %d . - - %s/%s" % (sha, size, base, os.path.basename(zp)))
|
||||
print("%-12s %5d files %5d MB %s" % (name, len(parts[name]), size >> 20, sha[:16]), flush=True)
|
||||
lines.append("part %s %d HookshotBuilder/tools %s llvm-mingw %s" % LLVM)
|
||||
lines.append("run HookshotBuilder/HookshotBuilder.exe")
|
||||
with open(os.path.join(a.out, "hookshot-setup.txt"), "w", newline="\n") as f:
|
||||
f.write("\n".join(lines) + "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
207
scripts/ngx_sdk.py
Normal file
207
scripts/ngx_sdk.py
Normal file
@@ -0,0 +1,207 @@
|
||||
"""NVIDIA's DLSS SDK files for a local build with NVIDIA DLSS Super Resolution (HOOKSHOT_NGX_SDK, gpu/vk/vk_dlss.cpp).
|
||||
|
||||
NVIDIA's files are not part of Hookshot and are never committed or bundled: they are under NVIDIA's own license (the DLSS
|
||||
SDK's LICENSE.txt, fetched beside them; read it before using them). This fetches them from NVIDIA's DLSS repository at one
|
||||
pinned release, each file checked against its SHA-256, into <sdk folder>:
|
||||
- LICENSE.txt and the SDK's headers (include/), the same for every platform;
|
||||
- the library the build links (lib/): libnvsdk_ngx.a on Linux, nvsdk_ngx_s.lib (the /MT one) on Windows;
|
||||
- the DLSS runtime the driver loads (runtime/): libnvidia-ngx-dlss.so.<version> on Linux, nvngx_dlss.dll on Windows.
|
||||
The NGX core itself comes with NVIDIA's driver (libnvidia-ngx.so on Linux; _nvngx.dll on Windows, which is loaded from the
|
||||
driver's own folder and is never to be put beside the game).
|
||||
|
||||
Usage: python -I scripts/ngx_sdk.py [--accept-nvidia-license] [--platform linux|windows] [--game <game folder>]
|
||||
[--runtime-dir <folder>] [--build-only] [<sdk folder>]
|
||||
<sdk folder>: default build-linux/ngx-sdk on Linux, work/ngx-sdk on Windows
|
||||
--game: the DLSS runtime copied into <game folder>/ngx too, where the game looks first (nx2/make_game.py does the same for
|
||||
the folder it builds when HOOKSHOT_NGX_SDK is set)
|
||||
--runtime-dir: the runtime copied into that folder instead (Linux: $XDG_DATA_HOME/hookshot/ngx; Windows:
|
||||
%LOCALAPPDATA%\\Hookshot\\ngx. RX_VK_DLSS_DIR names another when the game runs)
|
||||
--build-only: only what the build reads, not DLSS's runtime (CI's link check)
|
||||
--accept-nvidia-license: NVIDIA's license accepted without the question (scripts, the Hookshot Builder's DLSS tick, CI)
|
||||
Then build with HOOKSHOT_NGX_SDK=<sdk folder>.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import urllib.request
|
||||
|
||||
COMMIT = "9981c6350dc58c0ce21f024fd65e31e851c31689" # (v310.9.16)
|
||||
BASE = "https://raw.githubusercontent.com/NVIDIA/DLSS/" + COMMIT
|
||||
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
LICENSE_URL = "https://github.com/NVIDIA/DLSS/blob/%s/LICENSE.txt" % COMMIT
|
||||
|
||||
# (path in NVIDIA's repository, its SHA-256): what every platform's build reads
|
||||
COMMON = [
|
||||
("LICENSE.txt", "d4216e39ebef5f9b50a6712ebb37beeb5379862a67733a9999c651f21592aaf0"),
|
||||
("include/nvsdk_ngx_defs_dlssd.h", "2e97fe1595b1334f71b51724f2cb67496e8c296a7a2eec5606da53055882e716"),
|
||||
("include/nvsdk_ngx_defs_dlssg.h", "c29765ab9e0125f9d724ae9811cc2abe03738d17486ec8eee80d78a1504549e1"),
|
||||
("include/nvsdk_ngx_defs.h", "ea23f33497cd274860d1c25a97644fce807dcb0037c594547203343103fad03e"),
|
||||
("include/nvsdk_ngx_defs_vk.h", "0a24d0861ace7d6b9362a67b7f08bea1b33ea123c5ce730b19133de8a891d031"),
|
||||
("include/nvsdk_ngx.h", "dc38e7467cf415379c9d12ae1b6e4a494c453ed92720fb53e92aecb523e7b848"),
|
||||
("include/nvsdk_ngx_helpers_cuda.h", "f2851a76107bdf4fbcb7b261f3c35573c312d8ecb4bc18fb02d9e2cec7705042"),
|
||||
("include/nvsdk_ngx_helpers_d3d.h", "ec75224f36ed6580baaf250fa83620405ad81a1d98c2c436654a0c9cf6a6b8ba"),
|
||||
("include/nvsdk_ngx_helpers_dlssd_cuda.h", "7e35cde214d905346a487b9a28c05d35e1c55b480e237fe3de69580a7e332730"),
|
||||
("include/nvsdk_ngx_helpers_dlssd_d3d.h", "6df9d02c6f47feee3afc8d12340a9f554a7ddce4ddacb1b6a13a42b26c2c6293"),
|
||||
("include/nvsdk_ngx_helpers_dlssd.h", "a25fdda925b479dfe869593aeba311ef08793da54e0391f2fe2fe756cbef5532"),
|
||||
("include/nvsdk_ngx_helpers_dlssd_vk.h", "11df8fbda532a97fe867c8ef2b40058b49b7898183710f3468e55b1eedb8f40b"),
|
||||
("include/nvsdk_ngx_helpers_dlssg_d3d.h", "6335ebcb95de39a361e87fe0257d6ce0723e5d8deaa15db3b35afacf2b3f4220"),
|
||||
("include/nvsdk_ngx_helpers_dlssg.h", "6881d21419c40c78ec948b326902ece09fc112b0223144d51c4905f3fced7a1f"),
|
||||
("include/nvsdk_ngx_helpers_dlssg_vk.h", "b371d90b9f4372b9a415e68a0afbdc2f31b7b552e219a9637d848749bc965316"),
|
||||
("include/nvsdk_ngx_helpers.h", "5bcbadfe7478b802cf6d3aca4dc5ddd7d0889b99726e69c63f9e9bd555f44471"),
|
||||
("include/nvsdk_ngx_helpers_vk.h", "c192bff72138f12f770db48e12c1d8f712dfc81ea0e8e055ebb27d8dd1b31623"),
|
||||
("include/nvsdk_ngx_loader.h", "abdf5f66a692620538357d29fdf78e3e2a677ee8140b7a638962ecb766585e75"),
|
||||
("include/nvsdk_ngx_params_dlssd.h", "d41e3ba8f1d72583b55273092c99bf426dc4a0d46de827229e25d5b1325a370e"),
|
||||
("include/nvsdk_ngx_params_dlssg.h", "c24fa5aa68ffa2808c828b75cfa32843f1606f80a145f84055ac450d7e6c74ad"),
|
||||
("include/nvsdk_ngx_params.h", "943bc8cc5cdae03b6303016fbad3183636f2335ae27a2d18776798c3b4efabbc"),
|
||||
("include/nvsdk_ngx_standalone_common.h", "38413c3338b0aea0202c07da95d426bfee7080a42913823fa2d9dd585b087b53"),
|
||||
("include/nvsdk_ngx_standalone_cuda.h", "b23436ed5040051ac8e81554c9af080f2463c941435bbc3b82222e5431a09499"),
|
||||
("include/nvsdk_ngx_vk.h", "2d364ce7132881eb669e9498fd570d74cba563b1fbebc82c235f8e1ad2dd8b6d"),
|
||||
]
|
||||
# each platform's library (into lib/) and DLSS runtime (into runtime/)
|
||||
PLATFORMS = {
|
||||
"linux": {
|
||||
"library": ("lib/Linux_x86_64/libnvsdk_ngx.a", "d3c50bb0d9624fc35e502a8e929b8fc1e2b505017ebdb84c0d5b627e11a235df"),
|
||||
"runtime": ("lib/Linux_x86_64/rel/libnvidia-ngx-dlss.so.310.9.16",
|
||||
"30982c937afed85c96b8d313323de9a650a2eec6de1bb0f714b21750f4ce98f6"),
|
||||
},
|
||||
"windows": {
|
||||
# (the static NGX loader built against the static C runtime (/MT); llvm-mingw links it with gpu/vk/vk_ngx_msvc_rt.cpp)
|
||||
"library": ("lib/Windows_x86_64/x64/nvsdk_ngx_s.lib", "7b9a5239c1145af5f645ab472aa066f9ea79db36ac819562312aae69d65f3765"),
|
||||
"runtime": ("lib/Windows_x86_64/rel/nvngx_dlss.dll", "d00b5b0ab29c3ca101b70c3a5ecb3a01b7ef33766a3383a28a18623083e6123e"),
|
||||
},
|
||||
}
|
||||
|
||||
NOTICE = """\
|
||||
NVIDIA DLSS: this downloads NVIDIA's DLSS SDK (release v310.9.16) from github.com/NVIDIA/DLSS.
|
||||
NVIDIA's files are NVIDIA's, not Hookshot's, and are under NVIDIA's own license, the NVIDIA RTX SDKs license:
|
||||
%s
|
||||
(saved as LICENSE.txt beside the files). Using them means accepting it. Hookshot never includes or redistributes them:
|
||||
they stay on this PC, in the SDK folder and the game folder's ngx folder. Never share a game folder built with them.""" % LICENSE_URL
|
||||
|
||||
|
||||
def host_platform():
|
||||
"""The platform a build here is for: Windows on Windows or with RX_TARGET=windows (a cross build), else Linux"""
|
||||
return "windows" if os.name == "nt" or os.environ.get("RX_TARGET", "").lower() == "windows" else "linux"
|
||||
|
||||
|
||||
def destination(path):
|
||||
"""Where a file of NVIDIA's repository goes in the SDK folder (a relative path)"""
|
||||
if path.startswith("lib/"):
|
||||
return ("runtime/" if "/rel/" in path else "lib/") + path.rsplit("/", 1)[1]
|
||||
return path
|
||||
|
||||
|
||||
def files(platform, runtime=True):
|
||||
"""[(path in NVIDIA's repository, SHA-256, path in the SDK folder)] for one platform; runtime=False: what a build reads only"""
|
||||
spec = PLATFORMS[platform]
|
||||
return [(p, s, destination(p)) for p, s in COMMON + [spec["library"]] + ([spec["runtime"]] if runtime else [])]
|
||||
|
||||
|
||||
def library(sdk, platform):
|
||||
"""The library a build with this SDK folder links, for that platform"""
|
||||
return os.path.join(sdk, *destination(PLATFORMS[platform]["library"][0]).split("/"))
|
||||
|
||||
|
||||
def runtime(sdk, platform):
|
||||
"""The DLSS runtime in this SDK folder, for that platform"""
|
||||
return os.path.join(sdk, *destination(PLATFORMS[platform]["runtime"][0]).split("/"))
|
||||
|
||||
|
||||
def runtime_name(platform):
|
||||
return PLATFORMS[platform]["runtime"][0].rsplit("/", 1)[1]
|
||||
|
||||
|
||||
def sha256(path):
|
||||
h = hashlib.sha256()
|
||||
with open(path, "rb") as f:
|
||||
for block in iter(lambda: f.read(1 << 20), b""):
|
||||
h.update(block)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
TIMEOUT = 60 # (seconds without an answer, or between two reads, before a download gives up: a stalled one ends the build)
|
||||
|
||||
|
||||
def fetch_one(path, digest, dst, opener=urllib.request.urlopen):
|
||||
"""One file into dst, unless it is there already with that SHA-256. The download goes to dst.part and is moved in only
|
||||
when its SHA-256 is the expected one; otherwise it is deleted and ValueError says which"""
|
||||
if os.path.isfile(dst) and sha256(dst) == digest:
|
||||
return False
|
||||
os.makedirs(os.path.dirname(dst) or ".", exist_ok=True)
|
||||
tmp = dst + ".part"
|
||||
try:
|
||||
with opener(BASE + "/" + path, timeout=TIMEOUT) as r, open(tmp, "wb") as f:
|
||||
shutil.copyfileobj(r, f, 1 << 20)
|
||||
if sha256(tmp) != digest:
|
||||
raise ValueError("ngx-sdk: %s is not the file expected (SHA-256): not used" % path)
|
||||
os.replace(tmp, dst)
|
||||
finally:
|
||||
if os.path.exists(tmp):
|
||||
os.remove(tmp)
|
||||
return True
|
||||
|
||||
|
||||
def fetch(sdk, platform, opener=urllib.request.urlopen, say=print, runtime=True):
|
||||
"""Every file of that platform into the SDK folder (runtime=False: not DLSS's runtime)"""
|
||||
for path, digest, rel in files(platform, runtime):
|
||||
if fetch_one(path, digest, os.path.join(sdk, *rel.split("/")), opener):
|
||||
say("ngx-sdk: %s" % rel)
|
||||
|
||||
|
||||
def install_runtime(sdk, platform, folder):
|
||||
"""The SDK folder's DLSS runtime copied into folder (made when missing): where the game finds it"""
|
||||
src = runtime(sdk, platform)
|
||||
dst = os.path.join(folder, runtime_name(platform))
|
||||
os.makedirs(folder, exist_ok=True)
|
||||
if not (os.path.isfile(dst) and os.path.getsize(dst) == os.path.getsize(src) and sha256(dst) == sha256(src)):
|
||||
shutil.copyfile(src, dst + ".part")
|
||||
os.replace(dst + ".part", dst)
|
||||
return dst
|
||||
|
||||
|
||||
def accepted(flag, ask=input, interactive=None):
|
||||
"""NVIDIA's license, accepted: the flag, or a typed yes when there is someone to ask"""
|
||||
if flag:
|
||||
return True
|
||||
if not (sys.stdin.isatty() if interactive is None else interactive):
|
||||
return False
|
||||
try:
|
||||
return ask("Do you accept NVIDIA's license? Type yes to go on: ").strip().lower() in ("y", "yes")
|
||||
except EOFError:
|
||||
return False
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
p = argparse.ArgumentParser(description=__doc__.split("\n")[0])
|
||||
p.add_argument("sdk", nargs="?")
|
||||
p.add_argument("--platform", choices=sorted(PLATFORMS), default=host_platform())
|
||||
p.add_argument("--game", help="a game folder: the DLSS runtime into its ngx folder")
|
||||
p.add_argument("--runtime-dir", help="the DLSS runtime into this folder")
|
||||
p.add_argument("--accept-nvidia-license", action="store_true")
|
||||
p.add_argument("--build-only", action="store_true", help="not DLSS's runtime: only what the build reads")
|
||||
a = p.parse_args(argv)
|
||||
if a.build_only and (a.game or a.runtime_dir):
|
||||
p.error("--build-only fetches no runtime for --game or --runtime-dir")
|
||||
sdk = os.path.abspath(a.sdk or os.path.join(ROOT, "build-linux" if a.platform == "linux" else "work", "ngx-sdk"))
|
||||
print(NOTICE, flush=True)
|
||||
if not accepted(a.accept_nvidia_license):
|
||||
print("ngx-sdk: NVIDIA's license was not accepted (--accept-nvidia-license, or yes when asked): nothing downloaded",
|
||||
file=sys.stderr, flush=True)
|
||||
return 2
|
||||
try:
|
||||
fetch(sdk, a.platform, runtime=not a.build_only)
|
||||
except (OSError, ValueError) as e:
|
||||
print("ngx-sdk: %s" % e, file=sys.stderr, flush=True)
|
||||
return 1
|
||||
print("ngx-sdk: the SDK's headers and library in %s (NVIDIA's license: %s)" % (sdk, os.path.join(sdk, "LICENSE.txt")),
|
||||
flush=True)
|
||||
for folder in ([os.path.join(a.game, "ngx")] if a.game else []) + ([a.runtime_dir] if a.runtime_dir else []):
|
||||
print("ngx-sdk: the DLSS runtime in %s" % install_runtime(sdk, a.platform, folder), flush=True)
|
||||
print("ngx-sdk: build with HOOKSHOT_NGX_SDK=%s" % sdk, flush=True)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
409
scripts/oot_saves.py
Normal file
409
scripts/oot_saves.py
Normal file
@@ -0,0 +1,409 @@
|
||||
"""Copy Ocarina of Time saves between Hookshot and the emulators and ports that run the game: they all keep the same
|
||||
files, so a copy is all it takes.
|
||||
|
||||
python -I scripts/oot_saves.py list
|
||||
python -I scripts/oot_saves.py import <name> [--profile ID] [--keep-settings] [--dry-run] [--force]
|
||||
python -I scripts/oot_saves.py export <name> [--profile ID] [--keep-settings] [--dry-run] [--force]
|
||||
python -I scripts/oot_saves.py import --from DIR (a save from any folder into Hookshot)
|
||||
python -I scripts/oot_saves.py export --to DIR (Hookshot's save into any folder)
|
||||
|
||||
import copies into Hookshot, export copies out of it. <name> is one of EMULATORS below: lichii, frizti, orchid,
|
||||
pokedinner, eden, eglantine or nx2navi (to-lichii and to-hookshot still work: they are export lichii and import
|
||||
lichii). --from/--to name the other side's folder instead of where it is looked for: the save folder itself (the one
|
||||
with account.sav and slot_*), or a folder above it (an emulator's data folder, a portable user folder, NX2navi's game
|
||||
folder), which is searched for the game's saves. --hookshot DIR does the same for Hookshot's side: its data folder
|
||||
(Linux: $XDG_DATA_HOME/hookshot, default ~/.local/share/hookshot) or, on Windows, its game folder (the one with
|
||||
image.bin and user\\), which has to be given there. The side being overwritten is copied first to
|
||||
<backup>/<import or export>-<name>-<YYYYmmdd-HHMMSS> (Linux: $XDG_DATA_HOME/oot-save-backups, Windows:
|
||||
%LOCALAPPDATA%\\Hookshot\\oot-save-backups; --backup-dir changes it). Close every game first: --force skips that check.
|
||||
Standard library only; the guide is docs/saves.md.
|
||||
"""
|
||||
import argparse
|
||||
import collections
|
||||
import datetime
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
TITLE = "0400486024DEC000"
|
||||
NO_ACCOUNT = "0000000000000000"
|
||||
WINDOWS = os.name == "nt"
|
||||
|
||||
Program = collections.namedtuple("Program", "name display folders processes aliases")
|
||||
|
||||
# Everything that runs the game and where it keeps its data, one entry each. folders: per OS, where to look, in order
|
||||
# ({data} = $XDG_DATA_HOME or ~/.local/share, {appdata} = %APPDATA%, {localappdata} = %LOCALAPPDATA%, {home} = the
|
||||
# home folder); the saves are then found under it by LAYOUTS. processes: lower case, without .exe, for the
|
||||
# running-game check. The sources are in docs/saves.md.
|
||||
EDEN = {"linux": ["{data}/eden"], "windows": ["{appdata}/eden"]}
|
||||
EMULATORS = [
|
||||
Program("lichii", "Lichii", EDEN, ["lichii"], []),
|
||||
Program("frizti", "Frizti (Fritchii)", EDEN, ["lichii", "fritchii"], ["fritchii"]),
|
||||
Program("orchid", "Orchid", EDEN, ["orchid"], []),
|
||||
Program("pokedinner", "PokeDinner", EDEN, ["eglantine"], []),
|
||||
Program("eden", "Eden", EDEN, ["eden"], []),
|
||||
Program("eglantine", "Eglantine", {"linux": ["{data}/eglantine"], "windows": ["{appdata}/eglantine"]},
|
||||
["eglantine"], []),
|
||||
Program("nx2navi", "NX2navi", {"linux": ["{home}/Games/Ocarina"],
|
||||
"windows": ["{localappdata}/Programs/NX2navi/game", "C:/Games/Ocarina"]},
|
||||
["nx2navi", "ocarina"], []),
|
||||
]
|
||||
HOOKSHOT = Program("hookshot", "Hookshot", {"linux": ["{data}/hookshot"], "windows": []}, ["hookshot"], [])
|
||||
FOLDER = Program("folder", "the folder", {"linux": [], "windows": []}, [], [])
|
||||
|
||||
# Where the save folders sit under a data folder. "<id>" is one folder per user profile (its name is the id shown).
|
||||
# A folder deeper down the same path is accepted too. The first layout that finds a profile wins.
|
||||
LAYOUTS = (
|
||||
("nand", "user", "save", "account", "<id>", TITLE, "0"), # Eden's newer layout: used first when it exists
|
||||
("nand", "user", "save", NO_ACCOUNT, "<id>", TITLE), # Eden and its forks (Lichii, Eglantine, ...)
|
||||
("profile", "save", "<id>"), # NX2navi's game folder
|
||||
("user", "save", "<id>"), # Hookshot's game folder on Windows
|
||||
("save", "<id>"), # Hookshot's data folder
|
||||
)
|
||||
|
||||
|
||||
class Fail(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def data_home():
|
||||
return os.environ.get("XDG_DATA_HOME") or os.path.join(os.path.expanduser("~"), ".local", "share")
|
||||
|
||||
|
||||
def default_backup_dir():
|
||||
if WINDOWS:
|
||||
base = os.environ.get("LOCALAPPDATA") or os.path.join(os.path.expanduser("~"), "AppData", "Local")
|
||||
return os.path.join(base, "Hookshot", "oot-save-backups")
|
||||
return os.path.join(data_home(), "oot-save-backups")
|
||||
|
||||
|
||||
def folders(program):
|
||||
"""program's data folders on this OS, in the order they are tried."""
|
||||
home = os.path.expanduser("~")
|
||||
values = {"data": data_home(), "home": home,
|
||||
"appdata": os.environ.get("APPDATA") or os.path.join(home, "AppData", "Roaming"),
|
||||
"localappdata": os.environ.get("LOCALAPPDATA") or os.path.join(home, "AppData", "Local")}
|
||||
return [os.path.normpath(t.format(**values)) for t in program.folders["windows" if WINDOWS else "linux"]]
|
||||
|
||||
|
||||
def subdirs(path):
|
||||
try:
|
||||
return sorted(n for n in os.listdir(path) if os.path.isdir(os.path.join(path, n)))
|
||||
except OSError:
|
||||
return []
|
||||
|
||||
|
||||
def slots(path):
|
||||
return [n for n in subdirs(path) if n.startswith("slot_")]
|
||||
|
||||
|
||||
def is_save_folder(path):
|
||||
return os.path.isfile(os.path.join(path, "account.sav")) or bool(slots(path))
|
||||
|
||||
|
||||
def walk(root, parts, key):
|
||||
if not parts:
|
||||
yield key, root
|
||||
return
|
||||
for n in subdirs(root):
|
||||
if parts[0] == "<id>":
|
||||
yield from walk(os.path.join(root, n), parts[1:], n)
|
||||
elif n.lower() == parts[0].lower():
|
||||
yield from walk(os.path.join(root, n), parts[1:], key)
|
||||
|
||||
|
||||
def find_saves(root):
|
||||
"""The game's save folders under root, as {profile id: path}; root may be the save folder itself."""
|
||||
norm = os.path.normpath(os.path.abspath(root))
|
||||
title_folder = os.path.basename(norm).upper() == TITLE
|
||||
if is_save_folder(norm) or title_folder and "0" not in subdirs(norm):
|
||||
key = norm
|
||||
while os.path.basename(key).upper() in (TITLE, "0") and os.path.dirname(key) != key:
|
||||
key = os.path.dirname(key)
|
||||
return {os.path.basename(key) or norm: norm}
|
||||
found = {}
|
||||
for layout in LAYOUTS:
|
||||
idpos = layout.index("<id>")
|
||||
for start in range(len(layout)):
|
||||
if TITLE in layout and TITLE not in layout[start:] and not title_folder:
|
||||
continue # (below the title id folder, only the title id folder itself is a place to start)
|
||||
key = None
|
||||
if start > idpos: # (root is at or below the <id> folder: the id is a folder above it)
|
||||
up = norm
|
||||
for _ in range(start - idpos - 1):
|
||||
up = os.path.dirname(up)
|
||||
key = os.path.basename(up)
|
||||
for k, path in walk(norm, layout[start:], key):
|
||||
# A folder named by the layout (the title id folder) counts as soon as it is there: the game may not
|
||||
# have saved yet. A profile folder counts only when it holds a save (Eden's profile folders do not).
|
||||
if (layout[-1] != "<id>" or is_save_folder(path)) and k not in found and path not in found.values():
|
||||
found[k] = path
|
||||
return found
|
||||
|
||||
|
||||
def locate(program, override, flag):
|
||||
"""(folder, {profile id: save folder}) for program: override when given, else the first of its folders with saves."""
|
||||
if override is not None:
|
||||
if not os.path.isdir(override):
|
||||
raise Fail("no folder at %s" % override)
|
||||
found = find_saves(override)
|
||||
if not found:
|
||||
raise Fail("no Ocarina of Time save under %s (a folder with account.sav and slot_*, or %s under an "
|
||||
"emulator's nand\\user\\save): start the game there once and save" % (override, TITLE))
|
||||
return override, found
|
||||
tried = folders(program)
|
||||
for d in tried:
|
||||
found = find_saves(d) if os.path.isdir(d) else {}
|
||||
if found:
|
||||
return d, found
|
||||
if program is HOOKSHOT and not tried:
|
||||
raise Fail("Hookshot's saves are in the user folder of your game folder on Windows: pass the game folder with "
|
||||
"--hookshot (for example --hookshot C:\\Games\\Hookshot)")
|
||||
raise Fail("no %s save found (looked in %s): start the game in %s once and save, or pass %s <folder>"
|
||||
% (program.display, ", ".join(tried) or "no default folder", program.display, flag))
|
||||
|
||||
|
||||
def pick(found, name, flag, wanted=None):
|
||||
if wanted is not None:
|
||||
for k, v in found.items():
|
||||
if k.lower() == wanted.lower():
|
||||
return k, v
|
||||
raise Fail("%s has no profile %s; it has: %s" % (name, wanted, ", ".join(sorted(found))))
|
||||
if len(found) > 1:
|
||||
lines = "".join("\n %s %s" % (k, found[k]) for k in sorted(found))
|
||||
raise Fail("%s has Ocarina of Time saves in %d profiles, so pick one with %s:%s" % (name, len(found), flag, lines))
|
||||
return next(iter(found.items()))
|
||||
|
||||
|
||||
def pending_commit(save):
|
||||
"""NX2navi writes a save to profile/save-tx first and moves it into profile/save/<account> after: while something is
|
||||
waiting there, that is the save the game loads next, not the account folder."""
|
||||
parent = os.path.dirname(os.path.normpath(save))
|
||||
tx = os.path.join(os.path.dirname(parent), "save-tx")
|
||||
if os.path.basename(parent).lower() == "save" and os.path.isdir(tx):
|
||||
for _, _, files in os.walk(tx):
|
||||
if files:
|
||||
return tx
|
||||
return None
|
||||
|
||||
|
||||
def process_names():
|
||||
"""{process name: what it belongs to} for every program in the tables."""
|
||||
names = collections.OrderedDict()
|
||||
for program in [HOOKSHOT] + EMULATORS:
|
||||
for p in program.processes:
|
||||
names[p] = names[p] + " or " + program.display if p in names else program.display
|
||||
return names
|
||||
|
||||
|
||||
def running_games():
|
||||
"""What is running among Hookshot and the emulators in EMULATORS."""
|
||||
known = process_names()
|
||||
names = set()
|
||||
if WINDOWS:
|
||||
out = subprocess.run(["tasklist", "/FO", "CSV", "/NH"], capture_output=True, text=True).stdout
|
||||
for line in out.splitlines():
|
||||
image = line.split('","')[0].strip('"').lower()
|
||||
if image.endswith(".exe") and image[:-4] in known:
|
||||
names.add(known[image[:-4]])
|
||||
return sorted(names)
|
||||
appimage = re.compile(r"^(%s)[-_.].*\.appimage$" % "|".join(map(re.escape, known)))
|
||||
for pid in filter(str.isdigit, os.listdir("/proc")):
|
||||
try:
|
||||
with open("/proc/%s/cmdline" % pid, "rb") as f:
|
||||
argv0 = f.read().split(b"\0")[0].decode("utf-8", "replace")
|
||||
with open("/proc/%s/comm" % pid, encoding="utf-8", errors="replace") as f:
|
||||
comm = f.read().strip()
|
||||
except OSError:
|
||||
continue
|
||||
for candidate in (comm, os.path.basename(argv0)):
|
||||
low = candidate.lower()
|
||||
m = appimage.match(low)
|
||||
if low in known:
|
||||
names.add(known[low])
|
||||
elif m:
|
||||
names.add(known[m.group(1)])
|
||||
return sorted(names)
|
||||
|
||||
|
||||
def describe(path):
|
||||
out = []
|
||||
for s in slots(path):
|
||||
p = os.path.join(path, s, "progress.sav")
|
||||
if os.path.isfile(p):
|
||||
when = datetime.datetime.fromtimestamp(os.path.getmtime(p)).strftime("%Y-%m-%d %H:%M")
|
||||
out.append("%s (saved %s)" % (s, when))
|
||||
else:
|
||||
out.append(s)
|
||||
return out
|
||||
|
||||
|
||||
def show(title, folder, found):
|
||||
print("%s: %s" % (title, folder))
|
||||
for k in sorted(found):
|
||||
print(" %s %s" % (k, found[k]))
|
||||
for line in describe(found[k]) or ["(no slots)"]:
|
||||
print(" " + line)
|
||||
if pending_commit(found[k]):
|
||||
print(" (a save is still waiting in %s: start and quit NX2navi before copying)" % pending_commit(found[k]))
|
||||
|
||||
|
||||
def list_all(args):
|
||||
try:
|
||||
show("Hookshot", *locate(HOOKSHOT, args.hookshot, "--hookshot"))
|
||||
except Fail as e:
|
||||
print("Hookshot: %s" % e)
|
||||
groups = collections.OrderedDict()
|
||||
for program in EMULATORS:
|
||||
groups.setdefault(tuple(folders(program)), []).append(program.display)
|
||||
for tried, displays in groups.items():
|
||||
title = " / ".join(displays) + (" (one shared folder)" if len(displays) > 1 else "")
|
||||
shown = False
|
||||
for d in tried:
|
||||
found = find_saves(d) if os.path.isdir(d) else {}
|
||||
if found:
|
||||
show(title, d, found)
|
||||
shown = True
|
||||
if not shown:
|
||||
print("%s: nothing in %s" % (title, ", ".join(tried) or "(no default folder)"))
|
||||
if args.folder:
|
||||
show("Folder", *locate(FOLDER, args.folder, "--from"))
|
||||
|
||||
|
||||
def refuse_links(path):
|
||||
"""A save from someone else is only files and folders: a link in it would copy what it points to (any file or folder
|
||||
of this PC) into the save"""
|
||||
def link(p):
|
||||
# (any reparse point on Windows: a junction is not a symbolic link, and os.path.isjunction is only in 3.12 and
|
||||
# newer; copytree walks into a junction as into a folder)
|
||||
try:
|
||||
st = os.lstat(p)
|
||||
except OSError:
|
||||
return False
|
||||
return stat.S_ISLNK(st.st_mode) or bool(getattr(st, "st_file_attributes", 0) & 0x400)
|
||||
if link(path):
|
||||
raise Fail("%s is a link: a save holds only files and folders, so nothing was copied" % path)
|
||||
for root, dirs, files in os.walk(path):
|
||||
for n in dirs + files:
|
||||
if link(os.path.join(root, n)):
|
||||
raise Fail("%s is a link: a save holds only files and folders, so nothing was copied" % os.path.join(root, n))
|
||||
|
||||
|
||||
def copy_saves(src, dst, backup, keep_settings, dry_run):
|
||||
target_path = os.path.normcase(os.path.realpath(dst))
|
||||
backup_path = os.path.normcase(os.path.realpath(backup))
|
||||
try:
|
||||
inside_target = os.path.commonpath((target_path, backup_path)) == target_path
|
||||
except ValueError:
|
||||
inside_target = False # Windows paths can be on different drives.
|
||||
if inside_target:
|
||||
raise Fail("choose a --backup-dir outside the target save folder (%s)" % dst)
|
||||
names = slots(src)
|
||||
if not names:
|
||||
raise Fail("%s has no slot_* folders: nothing to copy" % src)
|
||||
for s in names:
|
||||
refuse_links(os.path.join(src, s))
|
||||
for f in ("account.sav", "option.sav"):
|
||||
if os.path.lexists(os.path.join(src, f)):
|
||||
refuse_links(os.path.join(src, f))
|
||||
files = [f for f in ("account.sav", "option.sav") if os.path.isfile(os.path.join(src, f))]
|
||||
if keep_settings and "option.sav" in files:
|
||||
files.remove("option.sav")
|
||||
print("%s %s -> %s" % ("Would back up" if dry_run else "Backing up", dst, backup))
|
||||
print("%s %d slot(s) (%s) and %s from %s to %s" % ("Would copy" if dry_run else "Copying", len(names), ", ".join(names),
|
||||
", ".join(files) or "no loose files", src, dst))
|
||||
if keep_settings:
|
||||
print("option.sav kept as it is on the target")
|
||||
if dry_run:
|
||||
print("Dry run: nothing was changed")
|
||||
return
|
||||
os.makedirs(os.path.dirname(backup), exist_ok=True)
|
||||
shutil.copytree(dst, backup)
|
||||
for s in slots(dst):
|
||||
shutil.rmtree(os.path.join(dst, s))
|
||||
for s in names: # (links were refused above; copied as links, never followed, all the same)
|
||||
shutil.copytree(os.path.join(src, s), os.path.join(dst, s), symlinks=True)
|
||||
for f in files:
|
||||
shutil.copy2(os.path.join(src, f), os.path.join(dst, f), follow_symlinks=False)
|
||||
print("Done. Backup of the old %s in %s" % (dst, backup))
|
||||
|
||||
|
||||
def program_named(name):
|
||||
for program in EMULATORS:
|
||||
if name.lower() in [program.name] + program.aliases:
|
||||
return program
|
||||
raise Fail("unknown emulator %s: use one of %s, or --from/--to <folder>"
|
||||
% (name, ", ".join(p.name for p in EMULATORS)))
|
||||
|
||||
|
||||
def run(args):
|
||||
if args.command == "list":
|
||||
list_all(args)
|
||||
return
|
||||
other = program_named(args.name) if args.name else FOLDER
|
||||
if other is FOLDER and not args.folder:
|
||||
raise Fail("%s what? Name the emulator (%s) or the folder (%s <folder>)"
|
||||
% (args.command, ", ".join(p.name for p in EMULATORS), "--from" if args.command == "import" else "--to"))
|
||||
flag = "--from" if args.command == "import" else "--to"
|
||||
_, hs = locate(HOOKSHOT, args.hookshot, "--hookshot")
|
||||
_, em = locate(other, args.folder, flag)
|
||||
hs_id, hs_path = pick(hs, "Hookshot", "--hookshot <the user id folder>")
|
||||
em_id, em_path = pick(em, other.display, "--profile <id>", args.profile)
|
||||
src, dst = (em_path, hs_path) if args.command == "import" else (hs_path, em_path)
|
||||
if os.path.realpath(src) == os.path.realpath(dst):
|
||||
raise Fail("%s is both the source and the target" % src)
|
||||
if not args.force:
|
||||
tx = pending_commit(em_path)
|
||||
if tx:
|
||||
raise Fail("NX2navi still has a save waiting in %s, which it loads instead of %s: start NX2navi and quit "
|
||||
"it from the game, then copy (or pass --force)" % (tx, em_path))
|
||||
if not args.dry_run and not args.force:
|
||||
busy = running_games()
|
||||
if busy:
|
||||
raise Fail("close %s first (or pass --force): a running game can write its save again and undo the copy"
|
||||
% " and ".join(busy))
|
||||
stamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
|
||||
backup = base = os.path.join(args.backup_dir or default_backup_dir(), "%s-%s-%s" % (args.command, other.name, stamp))
|
||||
n = 1
|
||||
while os.path.exists(backup): # (two copies in the same second)
|
||||
n += 1
|
||||
backup = "%s-%d" % (base, n)
|
||||
copy_saves(src, dst, backup, args.keep_settings, args.dry_run)
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0], prog="oot_saves.py")
|
||||
p.add_argument("command", choices=["list", "import", "export", "to-lichii", "to-hookshot"],
|
||||
help="import: into Hookshot; export: out of Hookshot; to-lichii, to-hookshot: export/import lichii")
|
||||
p.add_argument("name", nargs="?", help="the emulator: %s" % ", ".join(e.name for e in EMULATORS))
|
||||
p.add_argument("--from", dest="from_dir", metavar="DIR", help="import from this folder (a save, or one to search)")
|
||||
p.add_argument("--to", dest="to_dir", metavar="DIR", help="export to this folder (a save, or one to search)")
|
||||
p.add_argument("--lichii", metavar="DIR", help=argparse.SUPPRESS) # (the old name of --from/--to)
|
||||
p.add_argument("--hookshot", metavar="DIR", help="Hookshot's data folder (Windows: the game folder)")
|
||||
p.add_argument("--profile", metavar="ID", help="the emulator's profile, when several have saves")
|
||||
p.add_argument("--backup-dir", metavar="DIR", help="where the backup of the overwritten side goes")
|
||||
p.add_argument("--keep-settings", action="store_true", help="leave the target's option.sav (the game's settings) alone")
|
||||
p.add_argument("--dry-run", action="store_true", help="say what would happen and change nothing")
|
||||
p.add_argument("--force", action="store_true",
|
||||
help="copy even if a game seems to be running, or NX2navi has a save waiting")
|
||||
args = p.parse_args(argv)
|
||||
if args.command in ("to-lichii", "to-hookshot"):
|
||||
if args.name:
|
||||
p.error("%s takes no name" % args.command)
|
||||
args.command, args.name = ("export" if args.command == "to-lichii" else "import"), "lichii"
|
||||
if args.command == "import" and args.to_dir or args.command == "export" and args.from_dir:
|
||||
p.error("import reads --from, export writes --to")
|
||||
args.folder = args.from_dir or args.to_dir or args.lichii
|
||||
try:
|
||||
run(args)
|
||||
except Fail as e:
|
||||
print("error: %s" % e, file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
90
scripts/probe.py
Normal file
90
scripts/probe.py
Normal file
@@ -0,0 +1,90 @@
|
||||
"""Which pass made a flash: reads a probe_<frame>.txt (vk_draw.cpp RX_VK_PROBE) and aligns the frames before, at and
|
||||
after it by (kind, program, target, nth use), then prints the entries of the flash frame in order whose colour is far
|
||||
from both neighbours', decoded by format. The comparison skips decoded NaNs.
|
||||
The first reported entry is the pass that went wrong.
|
||||
|
||||
python -I scripts/probe.py work/runs/<run>/probe_<frame>.txt [ratio 4] [lines 25]
|
||||
"""
|
||||
import collections, math, struct, sys
|
||||
|
||||
|
||||
def half(h):
|
||||
return struct.unpack("<e", struct.pack("<H", h))[0]
|
||||
|
||||
|
||||
def uf(v, mbits):
|
||||
e, m = v >> mbits, v & ((1 << mbits) - 1)
|
||||
return (m / (1 << mbits)) * 2.0 ** -14 if e == 0 else 1e30 if e == 31 else (1 + m / (1 << mbits)) * 2.0 ** (e - 15)
|
||||
|
||||
|
||||
def decode(fmt, w):
|
||||
if fmt == 122: # B10G11R11
|
||||
return [uf(w[0] & 2047, 6), uf((w[0] >> 11) & 2047, 6), uf(w[0] >> 22, 5)]
|
||||
if fmt == 123: # E5B9G9R9
|
||||
s = 2.0 ** ((w[0] >> 27) - 24)
|
||||
return [(w[0] & 511) * s, ((w[0] >> 9) & 511) * s, ((w[0] >> 18) & 511) * s]
|
||||
if fmt == 97: # R16G16B16A16_SFLOAT
|
||||
return [half(w[0] & 0xffff), half(w[0] >> 16), half(w[1] & 0xffff)]
|
||||
if fmt == 83: # R16G16_SFLOAT
|
||||
return [half(w[0] & 0xffff), half(w[0] >> 16)]
|
||||
if fmt == 76: # R16_SFLOAT
|
||||
return [half(w[0] & 0xffff)]
|
||||
if fmt == 100: # R32_SFLOAT
|
||||
return list(struct.unpack("<f", struct.pack("<I", w[0])))
|
||||
if fmt == 109: # R32G32B32A32_SFLOAT
|
||||
return list(struct.unpack("<4f", struct.pack("<4I", *w)))
|
||||
return [(w[0] >> k) & 255 for k in (0, 8, 16, 24)]
|
||||
|
||||
|
||||
def key_rows(rows):
|
||||
seen, out = collections.Counter(), {}
|
||||
for r in rows:
|
||||
k = (r[2], r[3], r[4])
|
||||
out[k + (seen[k],)] = r
|
||||
seen[k] += 1
|
||||
return out
|
||||
|
||||
|
||||
frames = collections.OrderedDict()
|
||||
with open(sys.argv[1], encoding="utf-8") as capture:
|
||||
for line_number, line in enumerate(capture, 1):
|
||||
r = line.split()
|
||||
if not r:
|
||||
continue
|
||||
if len(r) < 11:
|
||||
raise SystemExit(f"Probe capture line {line_number} is incomplete. Capture a new probe_*.txt file.")
|
||||
frames.setdefault(r[0], []).append(r)
|
||||
if len(frames) != 3:
|
||||
raise SystemExit(f"Probe capture expected 3 frames, found {len(frames)}. "
|
||||
"Capture a new probe_*.txt file with the frame before, the flash, and the frame after.")
|
||||
a, b, c = [key_rows(v) for v in frames.values()]
|
||||
ratio = float(sys.argv[2]) if len(sys.argv) > 2 else 4.0
|
||||
lines = int(sys.argv[3]) if len(sys.argv) > 3 else 25
|
||||
|
||||
|
||||
def val(r):
|
||||
return decode(int(r[6]), [int(x, 16) for x in r[7:11]])
|
||||
|
||||
|
||||
def far(x, y):
|
||||
for p, q in zip(x, y):
|
||||
if math.isnan(p) or math.isnan(q):
|
||||
continue
|
||||
hi, lo = max(abs(p), abs(q)), min(abs(p), abs(q))
|
||||
if hi > 1e-3 and hi > lo * ratio:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
print("frames", list(frames), {k: len(v) for k, v in frames.items()}, "unmatched at the flash:", sum(1 for k in b if k not in a or k not in c))
|
||||
told = 0
|
||||
for k, r in b.items():
|
||||
if k not in a or k not in c:
|
||||
continue
|
||||
va, vb, vc = val(a[k]), val(r), val(c[k])
|
||||
if far(va, vb) and far(vc, vb):
|
||||
print("%4s %s prog %s %s %s f%s: %s | %s | %s" % (r[1], r[2], r[3][:8], r[4], r[5], r[6], ["%.4g" % x for x in va], ["%.4g" % x for x in vb], ["%.4g" % x for x in vc]))
|
||||
told += 1
|
||||
if told >= lines:
|
||||
break
|
||||
print("%d entries far from both neighbours" % told)
|
||||
151
scripts/prof.py
Normal file
151
scripts/prof.py
Normal file
@@ -0,0 +1,151 @@
|
||||
"""Offline profile of a thread sampled with HOS_SAMPLE + HOS_SAMPLE_FILE (host/mkw_glue.c, mkw-recomp work/prof.py): each line is a stack of
|
||||
module+offset frames, leaf first. Frames in hookshot_vk.dll / hookshot.exe are named from the linker maps beside them;
|
||||
on Linux, frames in hookshot_vk.so / hookshot from the binaries' own symbols (nm). RX_MAPS=<folder>: another build's
|
||||
maps or binaries (on Linux, the game folder).
|
||||
|
||||
python -I scripts/prof.py work/runs/<run>/stacks.txt [top] [skip] [only]
|
||||
(skip: the first fraction of the samples, e.g. 0.5; only: samples with that function in their stack, e.g.
|
||||
bench_replay)
|
||||
|
||||
Prints the functions by inclusive share (in the stack at all) and by self share (the leaf), and the leaf modules.
|
||||
"""
|
||||
import bisect, collections, itertools, os, re, stat, subprocess, sys
|
||||
from contextlib import ExitStack
|
||||
from tempfile import TemporaryFile
|
||||
|
||||
BUILD = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "work", "sa", "build")
|
||||
|
||||
|
||||
def load_map(path, base):
|
||||
syms = []
|
||||
pat = re.compile(r"^\s*0001:[0-9a-f]+\s+(\S+)\s+([0-9a-f]{16})\s", re.I)
|
||||
with open(path, encoding="utf-8", errors="replace") as f:
|
||||
for line in f:
|
||||
m = pat.match(line)
|
||||
if m:
|
||||
syms.append((int(m.group(2), 16) - base, m.group(1)))
|
||||
syms.sort()
|
||||
return [a for a, _ in syms], [n for _, n in syms]
|
||||
|
||||
|
||||
# The modules whose frames are Hookshot's own code (the game and the runtime, the renderer): Windows, then Linux
|
||||
GAME_MODS = ("hookshot.exe", "hookshot")
|
||||
VK_MODS = ("hookshot_vk.dll", "hookshot_vk.so")
|
||||
RUN_MODS = GAME_MODS + VK_MODS
|
||||
|
||||
|
||||
def load_elf(path):
|
||||
"""An ELF's function symbols by their offset in the file's address space, which is the offset from the load address
|
||||
that a sample of a position-independent executable or a shared object gives (binutils' nm, demangled)"""
|
||||
out = subprocess.run(["nm", "--defined-only", "-C", path], capture_output=True, text=True, encoding="utf-8",
|
||||
errors="replace").stdout
|
||||
syms = []
|
||||
for line in out.splitlines():
|
||||
parts = line.split(" ", 2)
|
||||
if len(parts) == 3 and parts[1] in ("t", "T", "w", "W") and len(parts[0]) >= 8:
|
||||
syms.append((int(parts[0], 16), parts[2].split("(")[0]))
|
||||
syms.sort()
|
||||
return [a for a, _ in syms], [n for _, n in syms]
|
||||
|
||||
|
||||
def load_symbols(folder=None):
|
||||
"""{module: (addresses, names)} for the modules found in folder (RX_MAPS, else the build): the linker maps of the
|
||||
Windows build, the binaries of the Linux one"""
|
||||
folder = folder or os.environ.get("RX_MAPS", BUILD)
|
||||
maps = {}
|
||||
for mod in ("hookshot_vk.dll", "hookshot.exe"):
|
||||
mp = os.path.join(folder, mod[:-4] + ".map")
|
||||
if os.path.exists(mp):
|
||||
maps[mod] = load_map(mp, 0x180000000 if mod.endswith(".dll") else 0x140000000)
|
||||
for mod in ("hookshot_vk.so", "hookshot"):
|
||||
b = os.path.join(folder, mod)
|
||||
if os.path.isfile(b):
|
||||
try:
|
||||
maps[mod] = load_elf(b)
|
||||
except OSError: # (no nm: frames stay module+offset)
|
||||
pass
|
||||
return maps
|
||||
|
||||
|
||||
def module_name(mod):
|
||||
"""hookshot_vk / hookshot, whichever system the sample came from"""
|
||||
return mod.split(".")[0]
|
||||
|
||||
|
||||
def demangle(n):
|
||||
# ?name@ns@@... -> ns::name (enough to read)
|
||||
m = re.match(r"\?([^@]+)@([^@]*)@", n)
|
||||
if m and not n.startswith("??"):
|
||||
return "%s::%s" % (m.group(2), m.group(1)) if m.group(2) else m.group(1)
|
||||
m = re.match(r"\?\?\$?([^@]+)@", n)
|
||||
return m.group(1) if m else n
|
||||
|
||||
|
||||
def profile_lines(path, skip):
|
||||
"""Keep the old line-based slice without holding the capture in memory."""
|
||||
with open(path, encoding="utf-8", errors="replace") as f, ExitStack() as files:
|
||||
start = 0
|
||||
if skip:
|
||||
# Windows can report a pipe as seekable, even though it cannot be replayed.
|
||||
if stat.S_ISREG(os.fstat(f.fileno()).st_mode) and f.seekable():
|
||||
count = sum(1 for _ in f)
|
||||
else:
|
||||
# A pipe can only be read once. Save its lines on disk to keep memory bounded.
|
||||
spool = files.enter_context(TemporaryFile(mode="w+t", encoding="utf-8", newline="\n"))
|
||||
count = 0
|
||||
for count, line in enumerate(f, 1):
|
||||
spool.write(line)
|
||||
f = spool
|
||||
start = int(count * skip)
|
||||
if start < 0:
|
||||
start = max(0, count + start)
|
||||
else:
|
||||
start = min(start, count)
|
||||
f.seek(0)
|
||||
yield from itertools.islice(f, start, None)
|
||||
|
||||
|
||||
def main():
|
||||
path = sys.argv[1]
|
||||
top = int(sys.argv[2]) if len(sys.argv) > 2 else 40
|
||||
skip = float(sys.argv[3]) if len(sys.argv) > 3 else 0.0
|
||||
only = sys.argv[4] if len(sys.argv) > 4 else None
|
||||
maps = load_symbols()
|
||||
|
||||
def name(frame):
|
||||
mod, _, off = frame.rpartition("+")
|
||||
if mod in maps:
|
||||
addrs, names = maps[mod]
|
||||
i = bisect.bisect_right(addrs, int(off, 16)) - 1
|
||||
if i >= 0:
|
||||
return module_name(mod) + ":" + demangle(names[i])
|
||||
return mod
|
||||
|
||||
incl, self_, leafmod = collections.Counter(), collections.Counter(), collections.Counter()
|
||||
n = 0
|
||||
for line in profile_lines(path, skip):
|
||||
frames = line.strip().split(";")
|
||||
if not frames or not frames[0]:
|
||||
continue
|
||||
named = [name(f) for f in frames]
|
||||
if only and not any(only in f for f in named):
|
||||
continue
|
||||
n += 1
|
||||
self_[named[0]] += 1
|
||||
leafmod[frames[0].rpartition("+")[0]] += 1
|
||||
for f in set(named):
|
||||
incl[f] += 1
|
||||
print("%d samples" % n)
|
||||
print("-- leaf modules")
|
||||
for k, v in leafmod.most_common(10):
|
||||
print("%6.1f%% %s" % (100.0 * v / n, k))
|
||||
print("-- inclusive")
|
||||
for k, v in incl.most_common(top):
|
||||
print("%6.1f%% %s" % (100.0 * v / n, k))
|
||||
print("-- self")
|
||||
for k, v in self_.most_common(top):
|
||||
print("%6.1f%% %s" % (100.0 * v / n, k))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
268
scripts/release_parts.py
Normal file
268
scripts/release_parts.py
Normal file
@@ -0,0 +1,268 @@
|
||||
"""What the release workflows need from earlier releases, and the Hookshot Builder package folder made in CI
|
||||
(.github/workflows/windows-builder.yml, linux-appimage.yml). No game data is involved: the parts hold the builder, its tools
|
||||
and the play profile (addresses and counts made on the development PC).
|
||||
|
||||
python -I scripts/release_parts.py previous <out list> --repo <owner/name> [--repo ...] [--skip-tag <tag>]
|
||||
the newest stable release's hookshot-setup.txt (of the first repository that has one; not --skip-tag's, the release
|
||||
being made); prints that release's tag. Exits 1 when no repository has one.
|
||||
python -I scripts/release_parts.py profile <setup list> <work folder>
|
||||
the play profile (tools/work of the list's app part) into <work folder>
|
||||
python -I scripts/release_parts.py llvm-mingw <folder> [--name]
|
||||
llvm-mingw as scripts/make_setup_parts.py names it (its official release, SHA-256 checked) unpacked into <folder>;
|
||||
prints its folder. --name: only prints its name (a cache key)
|
||||
python -I scripts/release_parts.py package <out folder> --publish <dotnet publish folder> --setup <setup exe>
|
||||
[--list <setup list>] [--python-packages <requirements>]
|
||||
the package folder make_setup_parts.py splits: the published builder, the setup, LICENSE, the tracked gpu, host, nx2
|
||||
and scripts under tools/, tools/work (the profile) and tools/third_party: the tracked files and, from the list's
|
||||
third_party part, the prebuilt files the repository does not carry (a64emit.dll, lld-link.exe, SDL2.dll, SDL3.dll,
|
||||
zbic.dll; rebuilt on the player's PC when their sources' stamps differ). Python, emsdk, CPM and wgpu-native are not
|
||||
in it: make_setup_parts.py --reuse takes their parts from the list. --python-packages: when the list's Python part
|
||||
lacks one of those pinned packages, the part goes into tools/python with them installed (by this Python's pip, for
|
||||
the part's version), and make_setup_parts.py makes a new Python part of it
|
||||
|
||||
GitHub's API is read with $GH_TOKEN when it is set. Downloads are retried (GitHub answers 5xx now and then). Progress goes to
|
||||
stderr: what a command prints on stdout is its answer.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import zipfile
|
||||
|
||||
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
CACHE = os.path.join(os.environ.get("RUNNER_TEMP") or tempfile.gettempdir(), "release-parts")
|
||||
# (the prebuilt files of the third_party part: what it holds and git does not)
|
||||
PREBUILT_EXT = (".dll", ".lib", ".exe", ".sha1")
|
||||
|
||||
|
||||
def get(url, api=False, tries=5):
|
||||
if not url.startswith("https://"): # (only https: the token is never sent over plain http, nor a file: link read)
|
||||
raise ValueError("not an https link: " + url)
|
||||
headers = {"User-Agent": "hookshot-ci"}
|
||||
token = None
|
||||
if api:
|
||||
headers["Accept"] = "application/vnd.github+json"
|
||||
token = os.environ.get("GH_TOKEN")
|
||||
for i in range(tries):
|
||||
try:
|
||||
req = urllib.request.Request(url, headers=headers)
|
||||
if token: # (not sent again on a redirect, which may lead to another host: urllib copies plain headers)
|
||||
req.add_unredirected_header("Authorization", "Bearer " + token)
|
||||
with urllib.request.urlopen(req, timeout=120) as r:
|
||||
return r.read()
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code < 500 or i == tries - 1:
|
||||
raise
|
||||
except (urllib.error.URLError, TimeoutError):
|
||||
if i == tries - 1:
|
||||
raise
|
||||
print(" %s: retrying in 10 s" % url, file=sys.stderr, flush=True)
|
||||
time.sleep(10)
|
||||
|
||||
|
||||
def previous(a):
|
||||
for repo in a.repo:
|
||||
try:
|
||||
releases = json.loads(get("https://api.github.com/repos/%s/releases?per_page=30" % repo, api=True))
|
||||
except urllib.error.HTTPError as e:
|
||||
print("%s: %s" % (repo, e), file=sys.stderr)
|
||||
continue
|
||||
for r in releases:
|
||||
if r["draft"] or r["prerelease"] or r["tag_name"] == a.skip_tag:
|
||||
continue
|
||||
url = next((x["browser_download_url"] for x in r["assets"] if x["name"] == "hookshot-setup.txt"), None)
|
||||
if url:
|
||||
with open(a.out, "wb") as f:
|
||||
f.write(get(url))
|
||||
print(r["tag_name"])
|
||||
return 0
|
||||
print("no release with a hookshot-setup.txt in %s" % ", ".join(a.repo), file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def part_lines(path):
|
||||
with open(path, encoding="utf-8") as f:
|
||||
return [l.split() for l in f.read().splitlines() if l.startswith("part ")]
|
||||
|
||||
|
||||
def fetch_part(sha, size, url):
|
||||
"""A part downloaded once into the cache, its size and SHA-256 checked"""
|
||||
os.makedirs(CACHE, exist_ok=True)
|
||||
path = os.path.join(CACHE, url.rsplit("/", 1)[-1])
|
||||
if not (os.path.isfile(path) and os.path.getsize(path) == size and sha256(path) == sha):
|
||||
print("downloading %s (%d MB)" % (url, size >> 20), file=sys.stderr, flush=True)
|
||||
data = get(url)
|
||||
if len(data) != size or hashlib.sha256(data).hexdigest() != sha:
|
||||
sys.exit("%s: not the size and SHA-256 the list gives" % url)
|
||||
with open(path + ".new", "wb") as f:
|
||||
f.write(data)
|
||||
os.replace(path + ".new", path)
|
||||
return path
|
||||
|
||||
|
||||
def sha256(path):
|
||||
h = hashlib.sha256()
|
||||
with open(path, "rb") as f:
|
||||
while b := f.read(1 << 20):
|
||||
h.update(b)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def named_part(list_path, name):
|
||||
for w in part_lines(list_path):
|
||||
if w[6].rsplit("/", 1)[-1].startswith("HookshotBuilder-%s-" % name):
|
||||
return fetch_part(w[1], int(w[2]), w[6])
|
||||
return None
|
||||
|
||||
|
||||
def extract(zip_path, prefix, dest, keep=lambda rel: True):
|
||||
"""The zip's members under prefix (HookshotBuilder/...), into dest without that prefix; how many"""
|
||||
n = 0
|
||||
with zipfile.ZipFile(zip_path) as z:
|
||||
for m in z.infolist():
|
||||
if m.is_dir() or not m.filename.startswith(prefix):
|
||||
continue
|
||||
rel = m.filename[len(prefix):]
|
||||
if ".." in rel.split("/") or not keep(rel):
|
||||
continue
|
||||
out = os.path.join(dest, *rel.split("/"))
|
||||
# (a name with a drive or a root, such as "C:/x" on Windows, would leave dest: such members are skipped)
|
||||
root = os.path.realpath(dest)
|
||||
if os.path.commonpath([os.path.realpath(out), root]) != root:
|
||||
continue
|
||||
os.makedirs(os.path.dirname(out), exist_ok=True)
|
||||
with z.open(m) as src, open(out, "wb") as f:
|
||||
shutil.copyfileobj(src, f)
|
||||
n += 1
|
||||
return n
|
||||
|
||||
|
||||
def profile(a):
|
||||
app = named_part(a.list, "app")
|
||||
if not app:
|
||||
sys.exit("no app part in %s" % a.list)
|
||||
n = extract(app, "HookshotBuilder/tools/work/", a.work)
|
||||
print("%d profile files into %s" % (n, a.work))
|
||||
return 0
|
||||
|
||||
|
||||
def llvm_spec():
|
||||
spec = importlib.util.spec_from_file_location("make_setup_parts", os.path.join(ROOT, "scripts", "make_setup_parts.py"))
|
||||
m = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(m)
|
||||
return m.LLVM # (sha256, size, top folder in the zip, url)
|
||||
|
||||
|
||||
def llvm_mingw(a):
|
||||
sha, size, name, url = llvm_spec()
|
||||
if a.name:
|
||||
print(name)
|
||||
return 0
|
||||
folder = os.path.join(a.folder, name)
|
||||
if not os.path.isfile(os.path.join(folder, "bin", "clang.exe")):
|
||||
z = fetch_part(sha, size, url)
|
||||
with zipfile.ZipFile(z) as zf:
|
||||
zf.extractall(a.folder)
|
||||
print(folder)
|
||||
return 0
|
||||
|
||||
|
||||
def tracked(*paths):
|
||||
r = subprocess.run(["git", "ls-files", "-z", "--"] + list(paths), cwd=ROOT, check=True, stdout=subprocess.PIPE)
|
||||
return [f for f in r.stdout.decode("utf-8").split("\0") if f]
|
||||
|
||||
|
||||
def copy(src, dst):
|
||||
os.makedirs(os.path.dirname(dst), exist_ok=True)
|
||||
shutil.copyfile(src, dst)
|
||||
|
||||
|
||||
def package(a):
|
||||
out = os.path.abspath(a.out)
|
||||
if os.path.exists(out):
|
||||
shutil.rmtree(out)
|
||||
shutil.copytree(a.publish, out)
|
||||
copy(a.setup, os.path.join(out, "HookshotBuilder-Setup.exe")) # (the builder's Repair runs it)
|
||||
copy(os.path.join(ROOT, "LICENSE"), os.path.join(out, "LICENSE"))
|
||||
tools = os.path.join(out, "tools")
|
||||
for f in tracked("gpu", "host", "nx2", "scripts", "third_party"):
|
||||
copy(os.path.join(ROOT, f), os.path.join(tools, *f.split("/")))
|
||||
if a.list:
|
||||
if not profile(argparse.Namespace(list=a.list, work=os.path.join(tools, "work"))) == 0:
|
||||
return 1
|
||||
tp = named_part(a.list, "third_party")
|
||||
if tp:
|
||||
known = set(tracked("third_party"))
|
||||
prebuilt = lambda rel: rel.startswith("third_party/") and rel not in known and rel.lower().endswith(PREBUILT_EXT)
|
||||
n = extract(tp, "HookshotBuilder/tools/", tools, keep=prebuilt)
|
||||
print("%d prebuilt third_party files from %s" % (n, os.path.basename(tp)))
|
||||
if a.python_packages and python_packages(a.list, a.python_packages, tools) != 0:
|
||||
return 1
|
||||
else:
|
||||
print("no setup list: the package has no play profile and no prebuilt third_party files")
|
||||
print("package folder: %s" % out)
|
||||
return 0
|
||||
|
||||
|
||||
def python_packages(list_path, requirements, tools):
|
||||
"""The list's Python part with requirements' pins (name==version) installed into tools/python, if it lacks one: by this
|
||||
Python's pip, for the part's version and win_amd64 (the part has no pip)"""
|
||||
with open(requirements, encoding="utf-8") as f:
|
||||
pins = [p for p in (l.split("#")[0].strip() for l in f.read().splitlines()) if p]
|
||||
part = named_part(list_path, "python")
|
||||
if not part:
|
||||
sys.exit("no python part in %s" % list_path)
|
||||
with zipfile.ZipFile(part) as z:
|
||||
names = [m.split("/") for m in z.namelist()]
|
||||
have = {m[-2].lower() for m in names if len(m) > 1 and m[-2].endswith(".dist-info")}
|
||||
missing = [p for p in pins if "%s-%s.dist-info" % tuple(x.strip().lower().replace("-", "_") for x in p.split("==")) not in have]
|
||||
if not missing:
|
||||
print("the Python part has %s" % ", ".join(pins))
|
||||
return 0
|
||||
dll = [m[-1] for m in names if m[:-1] == ["HookshotBuilder", "tools", "python"] and m[-1].lower().startswith("python3")
|
||||
and m[-1][7:-4].isdigit()]
|
||||
if len(dll) != 1:
|
||||
sys.exit("%s: no pythonXY.dll to give the version" % os.path.basename(part))
|
||||
version = "3." + dll[0][7:-4]
|
||||
py = os.path.join(tools, "python")
|
||||
n = extract(part, "HookshotBuilder/tools/python/", py)
|
||||
print("%d files of %s (Python %s) into %s, with %s" % (n, os.path.basename(part), version, py, ", ".join(missing)))
|
||||
subprocess.run([sys.executable, "-m", "pip", "install", "--no-deps", "--only-binary", ":all:", "--no-compile", "--no-cache-dir",
|
||||
"--disable-pip-version-check", "-q", "--platform", "win_amd64", "--python-version", version,
|
||||
"--implementation", "cp", "--target", os.path.join(py, "Lib", "site-packages")] + missing, check=True)
|
||||
return 0
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
sub = p.add_subparsers(dest="cmd", required=True)
|
||||
s = sub.add_parser("previous")
|
||||
s.add_argument("out")
|
||||
s.add_argument("--repo", action="append", required=True)
|
||||
s.add_argument("--skip-tag", default="")
|
||||
s = sub.add_parser("profile")
|
||||
s.add_argument("list")
|
||||
s.add_argument("work")
|
||||
s = sub.add_parser("llvm-mingw")
|
||||
s.add_argument("folder", nargs="?", default="")
|
||||
s.add_argument("--name", action="store_true")
|
||||
s = sub.add_parser("package")
|
||||
s.add_argument("out")
|
||||
s.add_argument("--publish", required=True)
|
||||
s.add_argument("--setup", required=True)
|
||||
s.add_argument("--list")
|
||||
s.add_argument("--python-packages")
|
||||
a = p.parse_args()
|
||||
return {"previous": previous, "profile": profile, "llvm-mingw": llvm_mingw, "package": package}[a.cmd](a)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
21
scripts/release_upload.sh
Executable file
21
scripts/release_upload.sh
Executable file
@@ -0,0 +1,21 @@
|
||||
#!/bin/bash
|
||||
# A tag's GitHub release with these files attached. The Linux AppImage and Windows builder workflows both call it: the
|
||||
# first to get there makes the release (from the tag, with GitHub's generated notes), the other adds its files to it, so
|
||||
# the two never race to make two releases. A release candidate (a tag with -rc: v0.2.0-rc.1) is a prerelease and never
|
||||
# "latest". Needs gh, with GH_TOKEN (contents: write) and GH_REPO set. No game data is involved.
|
||||
#
|
||||
# scripts/release_upload.sh <tag> <file> ...
|
||||
set -euo pipefail
|
||||
tag=$1
|
||||
shift
|
||||
if [[ $tag == *-rc* ]]; then
|
||||
kind=(--prerelease --latest=false)
|
||||
else
|
||||
kind=(--latest)
|
||||
fi
|
||||
if ! gh release view "$tag" >/dev/null 2>&1; then
|
||||
# (the other workflow can make it between the view and the create: its release is then the one used)
|
||||
gh release create "$tag" --verify-tag --title "$tag" --generate-notes "${kind[@]}" ||
|
||||
gh release view "$tag" >/dev/null
|
||||
fi
|
||||
gh release upload "$tag" --clobber "$@"
|
||||
54
scripts/run-vk.ps1
Normal file
54
scripts/run-vk.ps1
Normal file
@@ -0,0 +1,54 @@
|
||||
# A test run of the standalone build (mkw-recomp's work/run-vk.ps1, for hookshot.exe): the Vulkan back end
|
||||
# (RX_BACKEND=vulkan), headless unless -Window (then it never takes focus), High priority (-Priority), muted unless
|
||||
# -Sound, screenshots every -ShotEvery presents into work\runs\<name>\shots. Extra environment: -Env "A=1;B=2".
|
||||
#
|
||||
# Guards (the PC is the user's, and one game runs at a time): no run while Rainbow Six Siege is open, nor while
|
||||
# another session's mkw.exe or hookshot.exe runs (-Force overrides both); the game is stopped when its working set passes
|
||||
# -MaxGB. Delete work\runs\<name>\shots once the shots are looked at: they reach gigabytes.
|
||||
param([int]$Seconds = 120, [string]$Name = "vk", [int]$ShotEvery = 300, [string]$Env = "", [switch]$Window,
|
||||
[int]$MaxGB = 10, [switch]$Force, [switch]$Sound, [string]$Romfs = "", [string]$Priority = "High")
|
||||
$ErrorActionPreference = "Stop"
|
||||
$root = Split-Path -Parent $PSScriptRoot
|
||||
if (-not $Force) {
|
||||
if (Get-Process RainbowSix, RainbowSix_BE, RainbowSix_Vulkan -ErrorAction SilentlyContinue) {
|
||||
"a game is running (Rainbow Six): no test run now"; exit 2
|
||||
}
|
||||
$other = Get-Process mkw, hookshot -ErrorAction SilentlyContinue
|
||||
if ($other) { "another game run is going ($(($other | ForEach-Object { "$($_.Name).exe $($_.Id)" }) -join ', ')): one at a time"; exit 2 }
|
||||
}
|
||||
if (-not $Romfs) { # (the dump's romfs, from work\dump.txt: nx2/dump.py --set)
|
||||
$cfg = Join-Path $root "work\dump.txt"
|
||||
if (-not (Test-Path $cfg)) { "no -Romfs and no work\dump.txt (python -I nx2/dump.py --dump <folder> --set)"; exit 2 }
|
||||
$dump = (Get-Content $cfg | Where-Object { $_ -and -not $_.StartsWith("#") } | Select-Object -First 1).Trim()
|
||||
$Romfs = Join-Path $dump "romfs"
|
||||
}
|
||||
$exe = Join-Path $root "work\sa\build\hookshot.exe"
|
||||
$image = Join-Path $root "work\sa\image\image.bin"
|
||||
$user = Join-Path $root "work\sa\user"
|
||||
$out = Join-Path $root "work\runs\$Name"
|
||||
$shots = Join-Path $out "shots"
|
||||
New-Item -ItemType Directory -Force $out, $user, $shots | Out-Null
|
||||
$env:RX_BACKEND = "vulkan"
|
||||
if ($Window) { Remove-Item Env:HOS_HEADLESS -ErrorAction SilentlyContinue; $env:HOS_WINDOW_NOACTIVATE = "1" } else { $env:HOS_HEADLESS = "1" }
|
||||
$env:RX_VK_SHOTS = $shots
|
||||
$env:RX_VK_SHOT_EVERY = "$ShotEvery"
|
||||
# (test runs are silent: the user is often in a call; -Sound plays it)
|
||||
if ($Sound) { Remove-Item Env:HOS_AUDIO -ErrorAction SilentlyContinue } else { $env:HOS_AUDIO = "0" }
|
||||
foreach ($kv in ($Env -split ";")) { if ($kv -match "^([^=]+)=(.*)$") { Set-Item -Path ("Env:" + $matches[1]) -Value $matches[2] } }
|
||||
$t0 = Get-Date
|
||||
$p = Start-Process -FilePath $exe -ArgumentList @("`"$image`"", "--romfs", "`"$Romfs`"", "--user", "`"$user`"") `
|
||||
-WorkingDirectory $out -PassThru -NoNewWindow `
|
||||
-RedirectStandardOutput (Join-Path $out "stdout.txt") -RedirectStandardError (Join-Path $out "log.txt")
|
||||
try { $p.PriorityClass = $Priority } catch {} # (High by default: the user, 2026-11-04; -Priority BelowNormal to yield)
|
||||
$peak = 0
|
||||
$exited = $false
|
||||
for ($t = 0; $t -lt $Seconds; $t += 2) {
|
||||
if ($p.WaitForExit(2000)) { $exited = $true; break }
|
||||
try { $p.Refresh(); $mb = [int]($p.WorkingSet64 / 1MB) } catch { $mb = 0 }
|
||||
if ($mb -gt $peak) { $peak = $mb }
|
||||
if ($mb -gt $MaxGB * 1024) { "memory guard: $mb MB, stopped"; break }
|
||||
}
|
||||
if (-not $exited) { Stop-Process -Id $p.Id -Force -ErrorAction SilentlyContinue }
|
||||
"peak working set: $peak MB"
|
||||
$secs = [int]((Get-Date) - $t0).TotalSeconds
|
||||
"run '$Name': $secs s, " + $(if ($exited) { "exited with $($p.ExitCode)" } else { "stopped after $Seconds s" })
|
||||
226
scripts/validate_spirv.py
Normal file
226
scripts/validate_spirv.py
Normal file
@@ -0,0 +1,226 @@
|
||||
"""The SPIR-V Hookshot's translator kept in hookshot_programs.bin, checked offline before a driver sees it again.
|
||||
|
||||
Each stage goes through the Vulkan SDK's spirv-val, and through the device limits spirv-val does not know: the workgroup's
|
||||
invocations and its workgroup memory (the mesh fallback keeps a mesh program's outputs there, on top of the game's own
|
||||
shared memory). A stage that fails is written to --out as <key>_<n>_<stage>.spv, its disassembly (.spvasm, spirv-dis) and
|
||||
the reasons (.txt); the summary groups the validator's errors by message.
|
||||
|
||||
python -I scripts/validate_spirv.py "D:\\Games\\Hookshot\\hookshot_programs.bin" (the mesh-as-compute programs)
|
||||
python -I scripts/validate_spirv.py <file> --all (every program)
|
||||
python -I scripts/validate_spirv.py <file> --key a5da9f84b72d1471 (the programs whose key starts so)
|
||||
|
||||
The exit status is 1 when a stage failed, 0 when none did.
|
||||
"""
|
||||
import argparse
|
||||
import collections
|
||||
import concurrent.futures
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
CACHE_MAGIC = 0x31505348 # (gpu/vk/vk_shader.cpp kDiskMagic)
|
||||
FALLBACK = 4 # (a record's kind: a task or mesh program run as compute, issue 640)
|
||||
SPV_MAGIC = b"\x03\x02\x23\x07"
|
||||
STAGES = {0x1: "vertex", 0x2: "tesscontrol", 0x4: "tesseval", 0x8: "geometry", 0x10: "fragment", 0x20: "compute",
|
||||
0x40: "task", 0x80: "mesh"}
|
||||
|
||||
|
||||
def records(path, want):
|
||||
"""(key, kind, bytes) of each record want(key, kind) takes, in file order. The layout is disk_init's: a 16-byte head
|
||||
(magic, 0, stamp), then records of a u64 key, a u64 size and that many bytes, which start with the u32 kind. A record
|
||||
cut short ends the walk, as it does there"""
|
||||
with open(path, "rb") as f:
|
||||
head = f.read(16)
|
||||
if len(head) < 16 or struct.unpack_from("<I", head)[0] != CACHE_MAGIC:
|
||||
raise SystemExit("%s: not a Hookshot program cache" % path)
|
||||
while True:
|
||||
rec = f.read(16)
|
||||
if len(rec) < 16:
|
||||
return
|
||||
key, size = struct.unpack("<QQ", rec)
|
||||
at = f.tell()
|
||||
kind = f.read(4)
|
||||
if len(kind) < 4:
|
||||
return
|
||||
kind = struct.unpack("<I", kind)[0]
|
||||
if want(key, kind):
|
||||
f.seek(at)
|
||||
data = f.read(size)
|
||||
if len(data) < size:
|
||||
return
|
||||
yield key, kind, data
|
||||
f.seek(at + size)
|
||||
|
||||
|
||||
def stages(data):
|
||||
"""(nvn stage, vk stage, SPIR-V bytes) of each stage of a record. program_cache_save writes a stage as int nvn_stage,
|
||||
VkShaderStageFlagBits vk_stage, then the SPIR-V as a list (a u64 word count, the words): each is found by its magic
|
||||
word and kept when the three fields before it agree, so the record's other fields (the answers, Shader::Info) need no
|
||||
layout of their own here"""
|
||||
out = []
|
||||
i = data.find(SPV_MAGIC)
|
||||
while i != -1:
|
||||
if i >= 16:
|
||||
nvn, vk = struct.unpack_from("<iI", data, i - 16)
|
||||
n = struct.unpack_from("<Q", data, i - 8)[0]
|
||||
if 0 <= nvn <= 7 and vk in STAGES and 5 <= n and i + 4 * n <= len(data):
|
||||
out.append((nvn, vk, data[i:i + 4 * n]))
|
||||
i = data.find(SPV_MAGIC, i + 4 * n)
|
||||
continue
|
||||
i = data.find(SPV_MAGIC, i + 1)
|
||||
return out
|
||||
|
||||
|
||||
def reflect(spirv):
|
||||
"""(local size or None, workgroup memory bytes) of a module. The bytes are its Workgroup variables' types summed with
|
||||
no padding: a lower bound of what the driver lays out, so a module over the limit by it is over by any layout"""
|
||||
w = struct.unpack("<%dI" % (len(spirv) // 4), spirv[:len(spirv) // 4 * 4])
|
||||
size, const, pointee = {}, {}, {}
|
||||
local, local_ids, shared = None, None, 0
|
||||
i = 5
|
||||
while i < len(w):
|
||||
n, op = w[i] >> 16, w[i] & 0xFFFF
|
||||
if n == 0 or i + n > len(w):
|
||||
break
|
||||
a = w[i + 1:i + n]
|
||||
if op == 16 and len(a) >= 5 and a[1] == 17: # OpExecutionMode LocalSize x y z
|
||||
local = tuple(a[2:5])
|
||||
elif op == 331 and len(a) >= 5 and a[1] == 38: # OpExecutionModeId LocalSizeId (constants defined later)
|
||||
local_ids = a[2:5]
|
||||
elif op in (43, 50) and len(a) >= 3: # OpConstant, OpSpecConstant (its default): the low word
|
||||
const[a[1]] = a[2]
|
||||
elif op == 20: # OpTypeBool
|
||||
size[a[0]] = 4
|
||||
elif op in (21, 22): # OpTypeInt, OpTypeFloat: width in bits
|
||||
size[a[0]] = a[1] // 8
|
||||
elif op in (23, 24): # OpTypeVector, OpTypeMatrix: component type, count
|
||||
size[a[0]] = size.get(a[1], 0) * a[2]
|
||||
elif op == 28: # OpTypeArray: element type, length constant
|
||||
size[a[0]] = size.get(a[1], 0) * const.get(a[2], 0)
|
||||
elif op == 30: # OpTypeStruct: members
|
||||
size[a[0]] = sum(size.get(m, 0) for m in a[1:])
|
||||
elif op == 32: # OpTypePointer: id, storage class, pointee
|
||||
pointee[a[0]] = a[2]
|
||||
elif op == 59 and len(a) >= 3 and a[2] == 4: # OpVariable of storage class Workgroup
|
||||
shared += size.get(pointee.get(a[0]), 0)
|
||||
i += n
|
||||
if local_ids is not None:
|
||||
local = tuple(const.get(x, 0) for x in local_ids)
|
||||
return local, shared
|
||||
|
||||
|
||||
def find_tool(name):
|
||||
"""A Vulkan SDK tool: on PATH, else in %VULKAN_SDK%\\Bin"""
|
||||
found = shutil.which(name)
|
||||
if not found and os.environ.get("VULKAN_SDK"):
|
||||
candidate = os.path.join(os.environ["VULKAN_SDK"], "Bin", name + (".exe" if os.name == "nt" else ""))
|
||||
found = candidate if os.path.isfile(candidate) else None
|
||||
return found
|
||||
|
||||
|
||||
def check(job, a, val):
|
||||
"""The reasons a stage fails ([] when it passes): spirv-val's errors (on stdin), then the device limits"""
|
||||
key, nvn, vk, spirv = job
|
||||
reasons = []
|
||||
r = subprocess.run([val, "--target-env", a.target_env, *a.val_arg, "-"], input=spirv, capture_output=True)
|
||||
if r.returncode:
|
||||
text = (r.stdout + r.stderr).decode("utf-8", "replace").strip()
|
||||
reasons.append(("spirv-val", text or "spirv-val exit status %d" % r.returncode))
|
||||
local, shared = reflect(spirv)
|
||||
if local and local[0] * local[1] * local[2] > a.max_invocations:
|
||||
reasons.append(("limit", "workgroup %dx%dx%d is %d invocations, over maxComputeWorkGroupInvocations %d"
|
||||
% (*local, local[0] * local[1] * local[2], a.max_invocations)))
|
||||
if shared > a.max_shared:
|
||||
reasons.append(("limit", "workgroup memory at least %d bytes, over maxComputeSharedMemorySize %d"
|
||||
% (shared, a.max_shared)))
|
||||
return reasons, local, shared
|
||||
|
||||
|
||||
def first_error(text):
|
||||
"""A validator message with its numbers taken out, so the same fault in many stages counts as one"""
|
||||
line = next((x for x in text.splitlines() if x.startswith("error")), text.splitlines()[0] if text else "")
|
||||
return re.sub(r"\d+", "N", line).strip()
|
||||
|
||||
|
||||
def dump(out, job, reasons, local, shared, dis):
|
||||
key, nvn, vk, spirv = job
|
||||
base = os.path.join(out, "%016x_%d_%s" % (key, nvn, STAGES[vk]))
|
||||
with open(base + ".spv", "wb") as f:
|
||||
f.write(spirv)
|
||||
with open(base + ".txt", "w", encoding="utf-8") as f:
|
||||
f.write("program %016x, NVN stage %d, %s stage, %d words\n" % (key, nvn, STAGES[vk], len(spirv) // 4))
|
||||
f.write("local size %s, workgroup memory at least %d bytes\n\n" % (local, shared))
|
||||
for kind, text in reasons:
|
||||
f.write("[%s]\n%s\n\n" % (kind, text))
|
||||
if dis:
|
||||
subprocess.run([dis, base + ".spv", "-o", base + ".spvasm"], capture_output=True)
|
||||
return base
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
||||
p.add_argument("cache", help="hookshot_programs.bin")
|
||||
p.add_argument("--all", action="store_true", help="every program, not only the mesh-as-compute ones")
|
||||
p.add_argument("--key", action="append", default=[], help="a program key (hex) or its start; may be given again")
|
||||
p.add_argument("--out", help="where failing stages are written (default: spirv_failures beside the cache)")
|
||||
p.add_argument("--target-env", default="vulkan1.3", help="spirv-val's target (Hookshot asks for Vulkan 1.3)")
|
||||
p.add_argument("--val-arg", action="append", default=[], help="an argument passed on to spirv-val")
|
||||
p.add_argument("--max-shared", type=int, default=65536, help="maxComputeSharedMemorySize (RDNA1: 65536)")
|
||||
p.add_argument("--max-invocations", type=int, default=1024, help="maxComputeWorkGroupInvocations (RDNA1: 1024)")
|
||||
p.add_argument("--jobs", type=int, default=os.cpu_count() or 4)
|
||||
a = p.parse_args(argv)
|
||||
val, dis = find_tool("spirv-val"), find_tool("spirv-dis")
|
||||
if not val:
|
||||
raise SystemExit("spirv-val not found: install the Vulkan SDK or put its Bin folder on PATH")
|
||||
keys = [k.lower().removeprefix("0x") for k in a.key]
|
||||
out = a.out or os.path.join(os.path.dirname(os.path.abspath(a.cache)), "spirv_failures")
|
||||
|
||||
def want(key, kind):
|
||||
if keys:
|
||||
return any(("%016x" % key).startswith(k) for k in keys)
|
||||
return a.all or bool(kind & FALLBACK)
|
||||
|
||||
programs, failed, groups = 0, [], collections.defaultdict(list)
|
||||
with concurrent.futures.ThreadPoolExecutor(a.jobs) as pool:
|
||||
pending = collections.deque()
|
||||
|
||||
def settle(future):
|
||||
job, (reasons, local, shared) = future.job, future.result()
|
||||
if not reasons:
|
||||
return
|
||||
os.makedirs(out, exist_ok=True)
|
||||
base = dump(out, job, reasons, local, shared, dis)
|
||||
failed.append(base)
|
||||
for kind, text in reasons:
|
||||
groups[first_error(text) if kind == "spirv-val" else re.sub(r"\d+", "N", text)].append(
|
||||
"%016x %s" % (job[0], STAGES[job[2]]))
|
||||
|
||||
for key, kind, data in records(a.cache, want):
|
||||
programs += 1
|
||||
for nvn, vk, spirv in stages(data):
|
||||
job = (key, nvn, vk, spirv)
|
||||
future = pool.submit(check, job, a, val)
|
||||
future.job = job
|
||||
pending.append(future)
|
||||
while len(pending) > 4 * a.jobs: # (a few batches in flight: --all reads gigabytes)
|
||||
settle(pending.popleft())
|
||||
while pending:
|
||||
settle(pending.popleft())
|
||||
|
||||
checked = "programs" if a.all or keys else "mesh-as-compute programs"
|
||||
print("%s: %d %s, %d stages failed" % (a.cache, programs, checked, len(failed)))
|
||||
for message, where in sorted(groups.items(), key=lambda g: -len(g[1])):
|
||||
print(" %4dx %s" % (len(where), message))
|
||||
print(" e.g. %s" % ", ".join(where[:3]))
|
||||
if failed:
|
||||
print("written to %s (.spv, .spvasm, .txt)" % out)
|
||||
if not programs:
|
||||
print(" (no records matched%s)" % (": is the key right?" if keys else ""))
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
62
scripts/vkprof.py
Normal file
62
scripts/vkprof.py
Normal file
@@ -0,0 +1,62 @@
|
||||
"""The renderer's threads from an all-thread sample (host/mkw_glue.c HOS_SAMPLE_ALL): for the threads whose name starts
|
||||
with a prefix (default mkw_vk), the share of samples running, and the functions by inclusive and self share of the
|
||||
running samples (leaf in hookshot_vk.dll, hookshot.exe or the driver; on Linux hookshot_vk.so, hookshot or the driver).
|
||||
|
||||
python -I scripts/vkprof.py work/runs/<run>_all.txt [top 40] [prefix mkw_vk]
|
||||
(RX_MAPS=<folder>: another build's linker maps, or on Linux the game folder)
|
||||
"""
|
||||
import bisect, collections, os, sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import prof # noqa: E402
|
||||
|
||||
top = int(sys.argv[2]) if len(sys.argv) > 2 else 40
|
||||
prefix = sys.argv[3] if len(sys.argv) > 3 else "mkw_vk"
|
||||
maps = prof.load_symbols()
|
||||
cache = {}
|
||||
|
||||
|
||||
def name(f):
|
||||
if f in cache:
|
||||
return cache[f]
|
||||
mod, _, off = f.rpartition("+")
|
||||
r = mod
|
||||
if mod in maps:
|
||||
a, n = maps[mod]
|
||||
i = bisect.bisect_right(a, int(off, 16)) - 1
|
||||
if i >= 0:
|
||||
r = prof.module_name(mod) + ":" + prof.demangle(n[i])[:70]
|
||||
cache[f] = r
|
||||
return r
|
||||
|
||||
|
||||
RUN = prof.RUN_MODS + ("nvoglv64.dll",)
|
||||
DRIVERS = ("libnvidia-glcore.so", "libvulkan_radeon.so", "libvulkan_intel.so") # (Linux: the driver's file has its version after .so)
|
||||
seen, running = collections.Counter(), collections.Counter()
|
||||
incl, self_ = collections.Counter(), collections.Counter()
|
||||
nrun = 0
|
||||
for line in open(sys.argv[1], encoding="utf-8", errors="replace"):
|
||||
head, _, st = line.rstrip("\n").partition("|")
|
||||
th = head.split(" ", 1)[-1]
|
||||
if not st or not th.startswith(prefix):
|
||||
continue
|
||||
frames = st.split(";")
|
||||
seen[th] += 1
|
||||
leafmod = frames[0].rpartition("+")[0]
|
||||
if leafmod not in RUN and not leafmod.startswith(DRIVERS):
|
||||
continue
|
||||
running[th] += 1
|
||||
nrun += 1
|
||||
names = [name(f) for f in frames]
|
||||
self_[names[0]] += 1
|
||||
for n in set(names):
|
||||
incl[n] += 1
|
||||
print("-- running share by thread")
|
||||
for th in sorted(seen, key=lambda t: -running[t]):
|
||||
print(" %5.1f%% %s (%d samples)" % (100.0 * running[th] / seen[th], th, seen[th]))
|
||||
print("-- inclusive (share of %d running samples)" % nrun)
|
||||
for k, c in incl.most_common(top):
|
||||
print(" %5.1f%% %s" % (100.0 * c / max(1, nrun), k))
|
||||
print("-- self")
|
||||
for k, c in self_.most_common(top):
|
||||
print(" %5.1f%% %s" % (100.0 * c / max(1, nrun), k))
|
||||
Reference in New Issue
Block a user