Files
Triforce-Source/scripts/release_parts.py
ruckus7382sfgsgdf 34b3683441 Triforce v0.2.0
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-10 19:32:48 -06:00

269 lines
12 KiB
Python

"""What the release workflows need from earlier releases, and the Hookshot Builder package folder made in CI
(.github/workflows/windows-builder.yml, linux-appimage.yml). No game data is involved: the parts hold the builder, its tools
and the play profile (addresses and counts made on the development PC).
python -I scripts/release_parts.py previous <out list> --repo <owner/name> [--repo ...] [--skip-tag <tag>]
the newest stable release's hookshot-setup.txt (of the first repository that has one; not --skip-tag's, the release
being made); prints that release's tag. Exits 1 when no repository has one.
python -I scripts/release_parts.py profile <setup list> <work folder>
the play profile (tools/work of the list's app part) into <work folder>
python -I scripts/release_parts.py llvm-mingw <folder> [--name]
llvm-mingw as scripts/make_setup_parts.py names it (its official release, SHA-256 checked) unpacked into <folder>;
prints its folder. --name: only prints its name (a cache key)
python -I scripts/release_parts.py package <out folder> --publish <dotnet publish folder> --setup <setup exe>
[--list <setup list>] [--python-packages <requirements>]
the package folder make_setup_parts.py splits: the published builder, the setup, LICENSE, the tracked gpu, host, nx2
and scripts under tools/, tools/work (the profile) and tools/third_party: the tracked files and, from the list's
third_party part, the prebuilt files the repository does not carry (a64emit.dll, lld-link.exe, SDL2.dll, SDL3.dll,
zbic.dll; rebuilt on the player's PC when their sources' stamps differ). Python, emsdk, CPM and wgpu-native are not
in it: make_setup_parts.py --reuse takes their parts from the list. --python-packages: when the list's Python part
lacks one of those pinned packages, the part goes into tools/python with them installed (by this Python's pip, for
the part's version), and make_setup_parts.py makes a new Python part of it
GitHub's API is read with $GH_TOKEN when it is set. Downloads are retried (GitHub answers 5xx now and then). Progress goes to
stderr: what a command prints on stdout is its answer.
"""
import argparse
import hashlib
import importlib.util
import json
import os
import shutil
import subprocess
import sys
import tempfile
import time
import urllib.error
import urllib.request
import zipfile
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
CACHE = os.path.join(os.environ.get("RUNNER_TEMP") or tempfile.gettempdir(), "release-parts")
# (the prebuilt files of the third_party part: what it holds and git does not)
PREBUILT_EXT = (".dll", ".lib", ".exe", ".sha1")
def get(url, api=False, tries=5):
if not url.startswith("https://"): # (only https: the token is never sent over plain http, nor a file: link read)
raise ValueError("not an https link: " + url)
headers = {"User-Agent": "hookshot-ci"}
token = None
if api:
headers["Accept"] = "application/vnd.github+json"
token = os.environ.get("GH_TOKEN")
for i in range(tries):
try:
req = urllib.request.Request(url, headers=headers)
if token: # (not sent again on a redirect, which may lead to another host: urllib copies plain headers)
req.add_unredirected_header("Authorization", "Bearer " + token)
with urllib.request.urlopen(req, timeout=120) as r:
return r.read()
except urllib.error.HTTPError as e:
if e.code < 500 or i == tries - 1:
raise
except (urllib.error.URLError, TimeoutError):
if i == tries - 1:
raise
print(" %s: retrying in 10 s" % url, file=sys.stderr, flush=True)
time.sleep(10)
def previous(a):
for repo in a.repo:
try:
releases = json.loads(get("https://api.github.com/repos/%s/releases?per_page=30" % repo, api=True))
except urllib.error.HTTPError as e:
print("%s: %s" % (repo, e), file=sys.stderr)
continue
for r in releases:
if r["draft"] or r["prerelease"] or r["tag_name"] == a.skip_tag:
continue
url = next((x["browser_download_url"] for x in r["assets"] if x["name"] == "hookshot-setup.txt"), None)
if url:
with open(a.out, "wb") as f:
f.write(get(url))
print(r["tag_name"])
return 0
print("no release with a hookshot-setup.txt in %s" % ", ".join(a.repo), file=sys.stderr)
return 1
def part_lines(path):
with open(path, encoding="utf-8") as f:
return [l.split() for l in f.read().splitlines() if l.startswith("part ")]
def fetch_part(sha, size, url):
"""A part downloaded once into the cache, its size and SHA-256 checked"""
os.makedirs(CACHE, exist_ok=True)
path = os.path.join(CACHE, url.rsplit("/", 1)[-1])
if not (os.path.isfile(path) and os.path.getsize(path) == size and sha256(path) == sha):
print("downloading %s (%d MB)" % (url, size >> 20), file=sys.stderr, flush=True)
data = get(url)
if len(data) != size or hashlib.sha256(data).hexdigest() != sha:
sys.exit("%s: not the size and SHA-256 the list gives" % url)
with open(path + ".new", "wb") as f:
f.write(data)
os.replace(path + ".new", path)
return path
def sha256(path):
h = hashlib.sha256()
with open(path, "rb") as f:
while b := f.read(1 << 20):
h.update(b)
return h.hexdigest()
def named_part(list_path, name):
for w in part_lines(list_path):
if w[6].rsplit("/", 1)[-1].startswith("HookshotBuilder-%s-" % name):
return fetch_part(w[1], int(w[2]), w[6])
return None
def extract(zip_path, prefix, dest, keep=lambda rel: True):
"""The zip's members under prefix (HookshotBuilder/...), into dest without that prefix; how many"""
n = 0
with zipfile.ZipFile(zip_path) as z:
for m in z.infolist():
if m.is_dir() or not m.filename.startswith(prefix):
continue
rel = m.filename[len(prefix):]
if ".." in rel.split("/") or not keep(rel):
continue
out = os.path.join(dest, *rel.split("/"))
# (a name with a drive or a root, such as "C:/x" on Windows, would leave dest: such members are skipped)
root = os.path.realpath(dest)
if os.path.commonpath([os.path.realpath(out), root]) != root:
continue
os.makedirs(os.path.dirname(out), exist_ok=True)
with z.open(m) as src, open(out, "wb") as f:
shutil.copyfileobj(src, f)
n += 1
return n
def profile(a):
app = named_part(a.list, "app")
if not app:
sys.exit("no app part in %s" % a.list)
n = extract(app, "HookshotBuilder/tools/work/", a.work)
print("%d profile files into %s" % (n, a.work))
return 0
def llvm_spec():
spec = importlib.util.spec_from_file_location("make_setup_parts", os.path.join(ROOT, "scripts", "make_setup_parts.py"))
m = importlib.util.module_from_spec(spec)
spec.loader.exec_module(m)
return m.LLVM # (sha256, size, top folder in the zip, url)
def llvm_mingw(a):
sha, size, name, url = llvm_spec()
if a.name:
print(name)
return 0
folder = os.path.join(a.folder, name)
if not os.path.isfile(os.path.join(folder, "bin", "clang.exe")):
z = fetch_part(sha, size, url)
with zipfile.ZipFile(z) as zf:
zf.extractall(a.folder)
print(folder)
return 0
def tracked(*paths):
r = subprocess.run(["git", "ls-files", "-z", "--"] + list(paths), cwd=ROOT, check=True, stdout=subprocess.PIPE)
return [f for f in r.stdout.decode("utf-8").split("\0") if f]
def copy(src, dst):
os.makedirs(os.path.dirname(dst), exist_ok=True)
shutil.copyfile(src, dst)
def package(a):
out = os.path.abspath(a.out)
if os.path.exists(out):
shutil.rmtree(out)
shutil.copytree(a.publish, out)
copy(a.setup, os.path.join(out, "HookshotBuilder-Setup.exe")) # (the builder's Repair runs it)
copy(os.path.join(ROOT, "LICENSE"), os.path.join(out, "LICENSE"))
tools = os.path.join(out, "tools")
for f in tracked("gpu", "host", "nx2", "scripts", "third_party"):
copy(os.path.join(ROOT, f), os.path.join(tools, *f.split("/")))
if a.list:
if not profile(argparse.Namespace(list=a.list, work=os.path.join(tools, "work"))) == 0:
return 1
tp = named_part(a.list, "third_party")
if tp:
known = set(tracked("third_party"))
prebuilt = lambda rel: rel.startswith("third_party/") and rel not in known and rel.lower().endswith(PREBUILT_EXT)
n = extract(tp, "HookshotBuilder/tools/", tools, keep=prebuilt)
print("%d prebuilt third_party files from %s" % (n, os.path.basename(tp)))
if a.python_packages and python_packages(a.list, a.python_packages, tools) != 0:
return 1
else:
print("no setup list: the package has no play profile and no prebuilt third_party files")
print("package folder: %s" % out)
return 0
def python_packages(list_path, requirements, tools):
"""The list's Python part with requirements' pins (name==version) installed into tools/python, if it lacks one: by this
Python's pip, for the part's version and win_amd64 (the part has no pip)"""
with open(requirements, encoding="utf-8") as f:
pins = [p for p in (l.split("#")[0].strip() for l in f.read().splitlines()) if p]
part = named_part(list_path, "python")
if not part:
sys.exit("no python part in %s" % list_path)
with zipfile.ZipFile(part) as z:
names = [m.split("/") for m in z.namelist()]
have = {m[-2].lower() for m in names if len(m) > 1 and m[-2].endswith(".dist-info")}
missing = [p for p in pins if "%s-%s.dist-info" % tuple(x.strip().lower().replace("-", "_") for x in p.split("==")) not in have]
if not missing:
print("the Python part has %s" % ", ".join(pins))
return 0
dll = [m[-1] for m in names if m[:-1] == ["HookshotBuilder", "tools", "python"] and m[-1].lower().startswith("python3")
and m[-1][7:-4].isdigit()]
if len(dll) != 1:
sys.exit("%s: no pythonXY.dll to give the version" % os.path.basename(part))
version = "3." + dll[0][7:-4]
py = os.path.join(tools, "python")
n = extract(part, "HookshotBuilder/tools/python/", py)
print("%d files of %s (Python %s) into %s, with %s" % (n, os.path.basename(part), version, py, ", ".join(missing)))
subprocess.run([sys.executable, "-m", "pip", "install", "--no-deps", "--only-binary", ":all:", "--no-compile", "--no-cache-dir",
"--disable-pip-version-check", "-q", "--platform", "win_amd64", "--python-version", version,
"--implementation", "cp", "--target", os.path.join(py, "Lib", "site-packages")] + missing, check=True)
return 0
def main():
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
sub = p.add_subparsers(dest="cmd", required=True)
s = sub.add_parser("previous")
s.add_argument("out")
s.add_argument("--repo", action="append", required=True)
s.add_argument("--skip-tag", default="")
s = sub.add_parser("profile")
s.add_argument("list")
s.add_argument("work")
s = sub.add_parser("llvm-mingw")
s.add_argument("folder", nargs="?", default="")
s.add_argument("--name", action="store_true")
s = sub.add_parser("package")
s.add_argument("out")
s.add_argument("--publish", required=True)
s.add_argument("--setup", required=True)
s.add_argument("--list")
s.add_argument("--python-packages")
a = p.parse_args()
return {"previous": previous, "profile": profile, "llvm-mingw": llvm_mingw, "package": package}[a.cmd](a)
if __name__ == "__main__":
sys.exit(main())