Files
Triforce-Source/builder/py/hookshot_builder/setup_parts.py
ruckus7382sfgsgdf 34b3683441 Triforce v0.2.0
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-10 19:32:48 -06:00

338 lines
15 KiB
Python

"""The setup's work (builder/setup/setup.c, the same rules): a release's hookshot-setup.txt names its tag and its parts (one zip each,
with its SHA-256 and size); each part is downloaded (or moved over from the installed version when its checksum did not change),
checked and unpacked into versions\\<tag>.part, which becomes versions\\<tag> once complete. The version installed stays, the others
and the downloads go.
hookshot-setup.txt, one item a line:
tag <tag>
part <sha256> <bytes> <folder under the install> <top folder in the zip to rename, or -> <new name, or -> <url>
run <the WPF builder, relative to the install> (not needed here: the tools are what this app uses)"""
import hashlib
import json
import os
import re
import shutil
import time
import urllib.error
import urllib.request
import zipfile
from . import REPO, paths
UA = "HookshotBuilder-Setup/2.0"
WAITS = (3, 6, 12, 20, 30, 45, 60) # (8 tries over ~3 minutes: GitHub's release downloads answered 504 for hours on 2026-11-08)
OWN = ("python", "emsdk", "cpm", "third_party", "wgpu-native")
class SetupError(Exception):
pass
class Part:
def __init__(self, sha, size, dest, frm, to, url):
self.sha, self.size, self.dest, self.frm, self.to, self.url = sha, int(size), dest, frm, to, url
def retryable(status):
"""an answer worth trying again: none, a server error, a download cut off half way (200), a timeout or the API's rate limit"""
return not status or status >= 500 or status in (200, 408, 429, 403)
class Http:
def __init__(self):
self.status = 0
def get(self, url, sink=None, expect=0, accept=None, progress=None, cancel=None):
"""GET following redirects (never to http); the bytes to sink, or returned. Sets self.status; raises on failure"""
if not url.startswith("https://"):
raise SetupError("Only https downloads are allowed: " + url)
req = urllib.request.Request(url, headers={"User-Agent": UA, **({"Accept": accept} if accept else {})})
self.status = 0
try:
with urllib.request.urlopen(req, timeout=60) as r:
if not r.geturl().startswith("https://"):
raise SetupError("A download was redirected away from https")
self.status = r.status
got, out = 0, []
while True:
if cancel and cancel():
raise SetupError("cancelled")
b = r.read(1 << 16)
if not b:
break
got += len(b)
if sink:
sink(b)
else:
out.append(b)
if progress and expect:
progress(min(1.0, got / expect))
if expect and got != expect:
self.status = 200 # (cut off half way: worth another try)
raise SetupError("The download ended early (%d of %d MB)" % (got >> 20, expect >> 20))
return None if sink else b"".join(out)
except urllib.error.HTTPError as e:
self.status = e.code
raise SetupError("GitHub returned HTTP %d" % e.code) from None
except (urllib.error.URLError, OSError, TimeoutError) as e:
if not isinstance(e, SetupError):
self.status = self.status if self.status and self.status != 200 else 0
raise SetupError("Could not reach GitHub (%s)" % getattr(e, "reason", e)) from None
raise
def api_link_for(http, url):
"""https://github.com/<owner>/<repo>/releases/download/<tag>/<name> (or .../latest/download/<name>) -> the API's own link to
that file, which redirects to a fresh download (GitHub's cached release links answered 504 for hours while its API answered)"""
m = re.match(r"https://github\.com/([^/]+/[^/]+)/releases/(?:download/([^/]+)|latest/download)/([^/?]+)$", url)
if not m:
return None
repo, tag, name = m.groups()
api = "https://api.github.com/repos/%s/releases/%s" % (repo, ("tags/" + tag) if tag else "latest")
try:
doc = json.loads(http.get(api, accept="application/vnd.github+json"))
for a in doc.get("assets", []):
if a.get("name") == name and str(a.get("url", "")).startswith("https://api.github.com/"):
return a["url"]
except (SetupError, ValueError):
pass
return None
def manifest_url(tag=None):
"""(an update to a named release reads THAT release's list: the latest/download link can answer with the release before it for
a while after a publish. HOOKSHOT_SETUP_LIST=<url>: another list, for testing a release before it is the latest)"""
e = os.environ.get("HOOKSHOT_SETUP_LIST")
if e:
return e
if tag:
return "https://github.com/%s/releases/download/%s/hookshot-setup.txt" % (REPO, tag)
return "https://github.com/%s/releases/latest/download/hookshot-setup.txt" % REPO
def read_manifest(status, tag=None, cancel=None):
"""(tag, parts) of the release's list, tried again on a server error or no answer"""
http, url = Http(), manifest_url(tag)
text, last = None, ""
for attempt in range(len(WAITS) + 1):
if attempt:
status("GitHub didn't answer (%s). Trying again in %d s..." % (http.status or "no answer", WAITS[attempt - 1]))
time.sleep(WAITS[attempt - 1])
status("Looking for the latest Hookshot Builder...")
try:
text = http.get(url, cancel=cancel).decode("utf-8", "replace")
break
except SetupError as e:
last = str(e)
first = http.status
if not first or first >= 500: # (the API's link instead)
api = api_link_for(http, url)
if api:
try:
text = http.get(api, accept="application/octet-stream", cancel=cancel).decode("utf-8", "replace")
break
except SetupError:
http.status = first
if not retryable(http.status):
break
if text is None:
if http.status == 404:
raise SetupError("The latest Hookshot release has no setup files yet. Download it from the releases page for now: "
"https://github.com/%s/releases/latest" % REPO)
if http.status:
raise SetupError("GitHub's download servers are failing right now (an error 500 or above, tried 8 times over 3 minutes). "
"This is on GitHub's side: try again in a few minutes, or later today.")
raise SetupError("Could not reach GitHub to get the Hookshot Builder (%s). Check your internet connection and try again." % last)
got_tag, parts = "", []
for line in text.splitlines():
p = line.split()
if len(p) >= 2 and p[0] == "tag":
got_tag = p[1]
elif len(p) == 7 and p[0] == "part":
parts.append(Part(*p[1:]))
if not got_tag or not parts:
raise SetupError("The release's setup list could not be read. Try again later.")
if tag and tag != got_tag:
raise SetupError("GitHub answered with the setup list of %s, not %s (its download links can lag right after a release). "
"Try the update again in a few minutes." % (got_tag, tag))
return got_tag, parts
def part_folder(p):
"""The folder a part unpacks into, relative to a version's folder (None: the part has none of its own)"""
if p.to != "-" and p.dest != ".":
return os.path.join(p.dest, p.to).replace("/", os.sep)
b = p.url.rsplit("/", 1)[-1]
for own in OWN:
if b.startswith("HookshotBuilder-%s-" % own):
return os.path.join("HookshotBuilder", "tools", own)
return None
def reuse_part(tag, p, install):
"""A part whose checksum the installed version lists in its parts.txt (and whose folder it has) is moved over from it instead of
downloaded: that version is deleted after the install anyway"""
rel = part_folder(p)
was = paths.read_small(os.path.join(paths.versions(), "current.txt"))
if not rel or not was or was == tag:
return False
if p.sha not in paths.read_small(os.path.join(paths.versions(), was, "parts.txt")):
return False
src, dst = os.path.join(paths.versions(), was, rel), os.path.join(install, rel)
if not os.path.exists(src):
return False
os.makedirs(os.path.dirname(dst), exist_ok=True)
try:
os.replace(src, dst)
return True
except OSError:
return False
def rmtree(p):
shutil.rmtree(p, ignore_errors=True)
def installed(tag):
d = os.path.join(paths.versions(), tag)
return d if os.path.isfile(os.path.join(d, "installed.txt")) and os.path.isfile(
os.path.join(d, "HookshotBuilder", "tools", "nx2", "make_game.py")) else None
def current():
"""The version installed last, its folder (None: none yet)"""
tag = paths.read_small(os.path.join(paths.versions(), "current.txt"))
return installed(tag) if tag else None
def install(status, progress=lambda f: None, tag=None, repair=False, cancel=None):
"""Installs the release (the latest, or tag) unless it is there already; returns (tag, its folder)"""
tag, parts = read_manifest(status, tag, cancel)
versions, downloads = paths.versions(), os.path.join(paths.base(), "downloads")
final = os.path.join(versions, tag)
if repair or not installed(tag):
work = final + ".part"
rmtree(work)
rmtree(final)
os.makedirs(work, exist_ok=True)
os.makedirs(downloads, exist_ok=True)
n = len(parts)
for i, p in enumerate(parts, 1):
if cancel and cancel():
raise SetupError("cancelled")
if reuse_part(tag, p, work):
status("Part %d of %d unchanged: kept from the installed version" % (i, n))
continue
zpath = os.path.join(downloads, p.sha + ".zip")
if not (os.path.isfile(zpath) and sha256_file(zpath) == p.sha):
download_part(status, progress, p, i, n, zpath, cancel)
status("Unpacking part %d of %d..." % (i, n))
progress(-1)
dest = work if p.dest == "." else os.path.join(work, p.dest.replace("/", os.sep))
os.makedirs(dest, exist_ok=True)
try:
with zipfile.ZipFile(zpath) as z:
z.extractall(dest)
except (OSError, zipfile.BadZipFile) as e:
raise SetupError("Unpacking part %d of %d failed: %s" % (i, n, e)) from None
if p.frm != "-" and p.to != "-": # (llvm-mingw-<version>-ucrt-x86_64 -> llvm-mingw)
src, dst = os.path.join(dest, p.frm), os.path.join(dest, p.to)
if not os.path.exists(dst):
try:
os.replace(src, dst)
except OSError:
raise SetupError("Could not set up part %d of %d (renaming %s)." % (i, n, p.frm)) from None
if not os.path.isfile(os.path.join(work, "HookshotBuilder", "tools", "nx2", "make_game.py")):
raise SetupError("The build tools are missing after unpacking. Your antivirus may have removed them: allow the folder %s "
"and try again." % paths.base())
try:
os.replace(work, final)
except OSError:
raise SetupError("Could not finish installing into %s." % final) from None
paths.write_small(os.path.join(final, "installed.txt"), tag)
write_parts(final, parts)
elif not os.path.isfile(os.path.join(final, "parts.txt")):
write_parts(final, parts)
prune(tag, parts)
rmtree(downloads) # (installed: its zips are not kept; an update moves unchanged folders over instead)
return tag, final
def download_part(status, progress, p, i, n, zpath, cancel):
http, part = Http(), zpath + ".part"
for attempt in range(len(WAITS) + 1):
if attempt:
status("Part %d of %d: GitHub didn't answer (%s). Trying again in %d s..." % (i, n, http.status or "no answer",
WAITS[attempt - 1]))
time.sleep(WAITS[attempt - 1])
status("Downloading part %d of %d (%d MB)..." % (i, n, p.size >> 20))
h = hashlib.sha256()
try:
with open(part, "wb") as f:
def sink(b):
f.write(b)
h.update(b)
url = api_link_for(http, p.url) if attempt % 2 == 1 else None # (the API's link, every other try)
if url:
http.get(url, sink, p.size, "application/octet-stream", progress, cancel)
else:
http.get(p.url, sink, p.size, None, progress, cancel)
except SetupError as e:
if str(e) == "cancelled":
raise
if not retryable(http.status):
break
continue
except OSError:
raise SetupError("Could not write to %s." % os.path.dirname(zpath)) from None
progress(-1)
if h.hexdigest() != p.sha:
os.remove(part)
raise SetupError("Part %d of %d did not download correctly (its checksum differs). Try again." % (i, n))
os.replace(part, zpath)
return
try:
os.remove(part)
except OSError:
pass
raise SetupError(("Downloading part %d of %d failed: GitHub's download servers are failing right now (tried 8 times over 3 "
"minutes). This is on GitHub's side. The parts already downloaded are kept: try again in a few minutes."
if retryable(http.status) else
"Downloading part %d of %d failed. Check your connection and free space, then try again.") % (i, n))
def sha256_file(path):
h = hashlib.sha256()
with open(path, "rb") as f:
for b in iter(lambda: f.read(1 << 20), b""):
h.update(b)
return h.hexdigest()
def write_parts(install_dir, parts):
"""The checksums of the parts this version holds in folders of their own (reuse_part reads them at the next update)"""
paths.write_small(os.path.join(install_dir, "parts.txt"), "".join(p.sha + "\n" for p in parts if part_folder(p)))
def prune(tag, parts):
"""Only the version installed stays (a folder still in use cannot be renamed: it goes at a later run), and downloads the list
no longer names go"""
versions = paths.versions()
was = paths.read_small(os.path.join(versions, "current.txt"))
if was != tag:
if was:
paths.write_small(os.path.join(versions, "previous.txt"), was)
paths.write_small(os.path.join(versions, "current.txt"), tag)
for name in os.listdir(versions):
p = os.path.join(versions, name)
if not os.path.isdir(p) or name == tag:
continue
if name.endswith(".gone"):
rmtree(p)
continue
try:
os.replace(p, p + ".gone")
rmtree(p + ".gone")
except OSError:
pass