mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-16 15:59:59 -04:00
Add the NX2navi launcher: a one-click setup that builds the game from the player's dump, starts it and keeps it up to date.
NX2navi-Setup.exe installs the launcher into %LOCALAPPDATA%\Programs\NX2navi without administrator rights, with shortcuts and an Apps entry. The launcher finds the player's .nup, downloads the build tools itself (the MSYS2 UCRT64 packages toolchain.lock.json pins, checked by sha256 and unpacked without pacman, and zbic from PyPI) and runs tools/make_game.py, whose steps it shows with their progress. Its window follows the game's options menu: colours sampled from it, the menu's filigree, the white glowing cursor, the Triforce in the title band and as the programs' icon, and all text in the game's Cetra Display, decoded from the player's own dump. Mouse, keyboard and an XInput pad drive it. Updates come from GitHub releases (NX2navi-update.json): the new version is downloaded, checked and staged beside the running one, the stub restarts into it and falls back to the previous version when it fails to start, and the game is rebuilt without the dump. make_game.py now keys its exefs and romfs steps on the game files alone, has --rebuild and --events, compiles only the files whose source, headers, flags or compiler changed, and replaces ocarina.exe only once the new one links. tools/make_launcher.py builds the release on Windows or Linux. Tested: tests/launcher_test.py (19 tests), the real toolchain download and install, and under Wine 11 the setup, the stub, the window on the bundled Windows Python, an update from a local feed by clicking Updates, the rollback of a version that fails to start, and the downloaded clang compiling and linking a program against the game's libraries. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Merged onto the current port: make_game.py keeps the launcher's step keys, cached link and in-place swap, and its own runtime manifest (windres), AMD's FidelityFX library and the shader preparation, which becomes a step of its own ("shaders") that the launcher shows in its install, rebuild and update jobs. make_launcher.py ships the shader tools and, with --ffx, AMD's library. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -3,8 +3,9 @@
|
||||
python -I tools/make_game.py --nup "C:\\path\\game.nup" --out "C:\\Games\\Ocarina"
|
||||
python -I tools/make_game.py --exefs "C:\\path\\exefs" --romfs "C:\\path\\romfs" --out "C:\\Games\\Ocarina"
|
||||
python -I tools/make_game.py --nup "C:\\path\\game.nup" --check # only checks the game's version
|
||||
python -I tools/make_game.py --rebuild --out "C:\\Games\\Ocarina" # a newer kit, the files of the last build
|
||||
|
||||
Steps, each skipped when a run before finished it for the same input (state in <out>/build-cache/state.json):
|
||||
Steps, each skipped when a run before finished it for the same inputs (state in <out>/build-cache/state.json):
|
||||
1. check the title (0400486024DEC000) and main's build id (game 1.1.1)
|
||||
2. exefs the five ExeFS files, every module's segments unpacked (ZBIC through the player's own `zbic` package)
|
||||
and stored plainly, as the runtime's loader reads them -> <out>/exefs
|
||||
@@ -16,15 +17,28 @@ Steps, each skipped when a run before finished it for the same input (state in <
|
||||
6. shaders every shader in the game files (tools/prepare_shaders.py, default bind state) translated, and each one's
|
||||
GPU driver build done once, so play does not stop to compile them (`ocarina.exe --prepare-shaders`)
|
||||
|
||||
The exefs and romfs steps depend only on the game files; lift and compile depend on this kit's code too. A newer kit
|
||||
therefore lifts and compiles again but keeps the unpacked game files, and --rebuild needs no game files at all. The
|
||||
compile step keeps every object with the dependency list clang wrote for it (build-cache/obj/objects.json) and
|
||||
compiles a file again only when it, a header it includes, the flags or the compiler changed. The new ocarina.exe
|
||||
replaces the old one only once it links, so a failed build leaves the last good one in place.
|
||||
|
||||
Nothing is written outside --out. The game folder then holds only what this PC built from the player's files: never
|
||||
share it. Needs Python 3.10+ with numpy, lz4, zstandard and zbic (pip), and MSYS2 UCRT64's clang and libraries
|
||||
(README-BuildKit.md lists the commands). Delete <out>/build-cache afterwards to free about 5 GB.
|
||||
(README-BuildKit.md lists the commands). Delete <out>/build-cache afterwards to free about 5 GB; the next build then
|
||||
compiles everything again.
|
||||
|
||||
--events adds one machine-readable line per step and progress change to stdout, each starting with "@@" and followed
|
||||
by a JSON object (launcher/build.py reads them): {"ev": "step", "step": ..., "state": "start" | "done" | "skip"},
|
||||
{"ev": "progress", "step": ..., "done": n, "total": n, "unit": ...}, {"ev": "phase", ...}, {"ev": "error", "message":
|
||||
...} and {"ev": "end", ...}.
|
||||
"""
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import struct
|
||||
import subprocess
|
||||
@@ -41,7 +55,8 @@ MAIN_BUILD_ID = 'e0f3ccfaf50065ae4ea01eb5aeb13b1a' # main of game 1.1.1
|
||||
EXEFS_FILES = ('main', 'main.npdm', 'sdk', 'subsdk0', 'rtld')
|
||||
MODULES = ('main', 'sdk', 'subsdk0', 'rtld')
|
||||
ALIGN = 16
|
||||
STATE_VERSION = 1
|
||||
STATE_VERSION = 2
|
||||
STEPS = ('exefs', 'romfs', 'lift', 'compile', 'shaders')
|
||||
|
||||
# Bring-up fixes in the lifted SDK, each anchored on exact generated text that must occur once.
|
||||
LIFT_FIXES = [
|
||||
@@ -61,17 +76,62 @@ LIFT_FIXES = [
|
||||
LINK_LIBS = ['-lvulkan-1', '-lSDL3', '-lopus', '-lvpx', '-l:libdav1d.a', '-lzstd', '-l:libatomic.a', '-lbcrypt',
|
||||
'-limm32', '-luuid', '-lole32', '-luser32', '-lshell32', '-lgdi32', '-ladvapi32', '-lwinmm']
|
||||
SYSTEM_DLLS_SKIP = {'vulkan-1.dll'} # the graphics driver's loader, never a copy
|
||||
EXE_NAME = 'ocarina.exe'
|
||||
|
||||
# Runs a script the way `python script.py` does, with the script's folder first on sys.path. The launcher's
|
||||
# embeddable Python leaves the script folder off sys.path (its ._pth file), so every script this build starts goes
|
||||
# through this.
|
||||
RUN_SCRIPT = ('import os, runpy, sys; s = sys.argv[1]; sys.argv = sys.argv[1:]; '
|
||||
'sys.path.insert(0, os.path.dirname(os.path.abspath(s))); runpy.run_path(s, run_name="__main__")')
|
||||
|
||||
EVENTS = False
|
||||
|
||||
|
||||
def say(msg):
|
||||
print(msg, flush=True)
|
||||
|
||||
|
||||
def emit(ev, **fields):
|
||||
"""One "@@{json}" line for the launcher (--events); nothing otherwise."""
|
||||
if EVENTS:
|
||||
print('@@' + json.dumps({'ev': ev, **fields}, ensure_ascii=False), flush=True)
|
||||
|
||||
|
||||
def fail(msg):
|
||||
emit('error', message=msg)
|
||||
print(f'\nerror: {msg}', file=sys.stderr, flush=True)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def py_cmd(script, *args):
|
||||
"""The command line that runs a Python script of this kit with this interpreter (see RUN_SCRIPT)."""
|
||||
flags = ['-I'] if sys.flags.isolated else []
|
||||
return [sys.executable, *flags, '-c', RUN_SCRIPT, str(script), *map(str, args)]
|
||||
|
||||
|
||||
def sha256_bytes(data):
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def sha256_file(path):
|
||||
h = hashlib.sha256()
|
||||
with open(path, 'rb') as f:
|
||||
while b := f.read(8 << 20):
|
||||
h.update(b)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def tree_digest(root, patterns):
|
||||
"""sha256 over the relative names and contents of the files under root matching the glob patterns."""
|
||||
h = hashlib.sha256()
|
||||
files = sorted({p for pat in patterns for p in Path(root).glob(pat) if p.is_file() and '__pycache__' not in p.parts})
|
||||
for p in files:
|
||||
h.update(p.relative_to(root).as_posix().encode() + b'\0')
|
||||
h.update(p.read_bytes())
|
||||
h.update(b'\0')
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- input
|
||||
|
||||
class Source:
|
||||
@@ -137,6 +197,27 @@ class Source:
|
||||
yield b
|
||||
|
||||
|
||||
class BuiltSource:
|
||||
"""--rebuild: the ExeFS and RomFS an earlier run unpacked into the game folder, under that run's input identity.
|
||||
The ExeFS modules are stored plainly there but keep their headers, so check() reads the same build id."""
|
||||
|
||||
def __init__(self, out, state):
|
||||
self.out = out
|
||||
if state.get('version') != STATE_VERSION or not state.get('input'):
|
||||
fail(f'{out} holds no finished build of this kit to rebuild from: give --nup, or --exefs and --romfs')
|
||||
for step in ('exefs', 'romfs'):
|
||||
if step not in state.get('steps', {}):
|
||||
fail(f'the last build in {out} did not finish its {step} step: give the game files again')
|
||||
missing = [n for n in EXEFS_FILES if not (out / 'exefs' / n).is_file()]
|
||||
missing += [f'romfs/{n}' for n in ('romfs.bin', 'index.json') if not (out / 'romfs' / n).is_file()]
|
||||
if missing:
|
||||
fail(f'{out} lacks {", ".join(missing)}: give the game files again')
|
||||
self.identity = state['input']
|
||||
|
||||
def read_exefs(self, name):
|
||||
return (self.out / 'exefs' / name).read_bytes()
|
||||
|
||||
|
||||
def check(src):
|
||||
main = src.read_exefs('main')
|
||||
if main[:4] != b'NSO0':
|
||||
@@ -182,11 +263,16 @@ def flat_nso(raw, name):
|
||||
def step_exefs(src, out):
|
||||
dst = out / 'exefs'
|
||||
dst.mkdir(parents=True, exist_ok=True)
|
||||
for name in EXEFS_FILES:
|
||||
for i, name in enumerate(EXEFS_FILES):
|
||||
raw = src.read_exefs(name)
|
||||
data = flat_nso(raw, name) if name in MODULES else raw
|
||||
(dst / name).write_bytes(data)
|
||||
say(f' {name:9} {len(raw) / 1e6:6.1f} MB -> {len(data) / 1e6:6.1f} MB')
|
||||
emit('progress', step='exefs', done=i + 1, total=len(EXEFS_FILES), unit='files')
|
||||
|
||||
|
||||
def exefs_digest(out):
|
||||
return tree_digest(out / 'exefs', EXEFS_FILES)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- romfs
|
||||
@@ -202,6 +288,7 @@ def step_romfs(src, out, jobs):
|
||||
say(f' {len(files)} files, {total / 1e9:.1f} GB')
|
||||
entries, h, pos, done, t0 = [], hashlib.sha256(), 0, 0, time.time()
|
||||
part = dst / 'romfs.bin.part'
|
||||
emit('progress', step='romfs', done=0, total=total, unit='bytes')
|
||||
|
||||
def load(e): # small files are read whole on a worker; big ones are streamed by the writer
|
||||
rel, ref = e
|
||||
@@ -224,6 +311,7 @@ def step_romfs(src, out, jobs):
|
||||
h.update(chunk)
|
||||
pos += size
|
||||
done += size
|
||||
emit('progress', step='romfs', done=done, total=total, unit='bytes')
|
||||
if start // batch % 40 == 0:
|
||||
say(f' {done / 1e9:5.1f} of {total / 1e9:.1f} GB, {time.time() - t0:.0f} s')
|
||||
part.replace(dst / 'romfs.bin')
|
||||
@@ -234,13 +322,31 @@ def step_romfs(src, out, jobs):
|
||||
|
||||
# ---------------------------------------------------------------- lift
|
||||
|
||||
EMITTED = re.compile(r'emitted (\d+)/(\d+) chunks')
|
||||
|
||||
|
||||
def lift_digest():
|
||||
"""The lifter's code, the patch table and the bring-up fixes: what the lifted C depends on besides the ExeFS."""
|
||||
h = hashlib.sha256(tree_digest(KIT / 'recomp', ('*.py', 'patches-v111.txt')).encode())
|
||||
h.update(repr(LIFT_FIXES).encode())
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def step_lift(out, cache):
|
||||
"""gen.py into build-cache/lift. The folder is kept: gen.py rewrites only the chunks whose text changed and
|
||||
removes the ones it no longer writes."""
|
||||
lift = cache / 'lift'
|
||||
if lift.exists():
|
||||
shutil.rmtree(lift)
|
||||
cmd = [sys.executable, str(KIT / 'recomp' / 'gen.py'), '--exefs', str(out / 'exefs'), '--out', str(lift),
|
||||
'--patches', str(KIT / 'recomp' / 'patches-v111.txt')]
|
||||
if subprocess.run(cmd).returncode:
|
||||
cmd = py_cmd(KIT / 'recomp' / 'gen.py', '--exefs', out / 'exefs', '--out', lift,
|
||||
'--patches', KIT / 'recomp' / 'patches-v111.txt')
|
||||
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding='utf-8',
|
||||
errors='replace', bufsize=1)
|
||||
for line in p.stdout:
|
||||
line = line.rstrip('\n')
|
||||
say(f' {line}')
|
||||
m = EMITTED.search(line)
|
||||
if m:
|
||||
emit('progress', step='lift', done=int(m.group(1)), total=int(m.group(2)), unit='chunks')
|
||||
if p.wait():
|
||||
fail('the code generator failed (see above)')
|
||||
for mod, old, new in LIFT_FIXES:
|
||||
hits = [p for p in sorted((lift / mod).glob('*.c')) if old in p.read_text(encoding='utf-8')]
|
||||
@@ -252,14 +358,6 @@ def step_lift(out, cache):
|
||||
|
||||
# ---------------------------------------------------------------- compile
|
||||
|
||||
def sha256_file(path):
|
||||
h = hashlib.sha256()
|
||||
with open(path, 'rb') as f:
|
||||
while b := f.read(8 << 20):
|
||||
h.update(b)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def modules_c(lift, out, gen):
|
||||
layout = json.loads((lift / 'layout.json').read_text(encoding='utf-8'))
|
||||
L = ['#include "dispatch.h"', '#include "host.h"', '#include "layout.h"', '']
|
||||
@@ -286,12 +384,17 @@ def modules_c(lift, out, gen):
|
||||
'const int rt_nvn_api_version[2] = {231, 40}; /* NVN2: main asks for major 231, minor 40 or later */',
|
||||
f'const char rt_port_version[] = {json.dumps(kit_version())};']
|
||||
p = gen / 'modules.c'
|
||||
p.write_text('\n'.join(L) + '\n', encoding='utf-8')
|
||||
(gen / 'shim.c').write_text('/* Defined by the SDL window path only; this build uses the Win32 one. */\n'
|
||||
'void menu_sdl_event(void *e) { (void)e; }\n', encoding='utf-8')
|
||||
write_if_changed(p, '\n'.join(L) + '\n')
|
||||
write_if_changed(gen / 'shim.c', '/* Defined by the SDL window path only; this build uses the Win32 one. */\n'
|
||||
'void menu_sdl_event(void *e) { (void)e; }\n')
|
||||
return p
|
||||
|
||||
|
||||
def write_if_changed(path, text):
|
||||
if not path.is_file() or path.read_text(encoding='utf-8') != text:
|
||||
path.write_text(text, encoding='utf-8')
|
||||
|
||||
|
||||
def kit_version():
|
||||
v = KIT / 'VERSION'
|
||||
return v.read_text(encoding='utf-8').strip() if v.is_file() else '0.0.1-ocarina'
|
||||
@@ -323,6 +426,23 @@ def pe_imports(path):
|
||||
return names
|
||||
|
||||
|
||||
def copy_if_changed(src, dst):
|
||||
"""Copy src over dst unless dst already has its size and modification time. A file the running game holds open
|
||||
cannot be replaced on Windows: that stops the build with a message instead of a traceback."""
|
||||
s = os.stat(src)
|
||||
try:
|
||||
d = os.stat(dst)
|
||||
if d.st_size == s.st_size and int(d.st_mtime) == int(s.st_mtime):
|
||||
return False
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
try:
|
||||
shutil.copy2(src, dst)
|
||||
except PermissionError:
|
||||
fail(f'{dst} is in use: close the game and try again')
|
||||
return True
|
||||
|
||||
|
||||
def copy_dlls(exe, msys_bin, out):
|
||||
seen, todo, copied = set(), [exe], []
|
||||
while todo:
|
||||
@@ -331,21 +451,136 @@ def copy_dlls(exe, msys_bin, out):
|
||||
if name in seen or name in SYSTEM_DLLS_SKIP or not src.is_file():
|
||||
continue
|
||||
seen.add(name)
|
||||
shutil.copy2(src, out / src.name)
|
||||
copy_if_changed(src, out / src.name)
|
||||
copied.append(src.name)
|
||||
todo.append(src)
|
||||
return copied
|
||||
|
||||
|
||||
def step_compile(out, cache, msys, jobs):
|
||||
lift, gen, obj = cache / 'lift', cache / 'gen', cache / 'obj'
|
||||
def parse_deps(text):
|
||||
"""The prerequisites of a make-style dependency file as clang -MMD writes it (the target dropped)."""
|
||||
text = text.replace('\\\r\n', ' ').replace('\\\n', ' ')
|
||||
tokens, cur, i = [], '', 0
|
||||
while i < len(text):
|
||||
c = text[i]
|
||||
if c == '\\' and i + 1 < len(text) and text[i + 1] in ' #':
|
||||
cur += text[i + 1]
|
||||
i += 2
|
||||
continue
|
||||
if c == '$' and text[i + 1:i + 2] == '$':
|
||||
cur += '$'
|
||||
i += 2
|
||||
continue
|
||||
if c.isspace():
|
||||
if cur:
|
||||
tokens.append(cur)
|
||||
cur = ''
|
||||
else:
|
||||
cur += c
|
||||
i += 1
|
||||
if cur:
|
||||
tokens.append(cur)
|
||||
for k, t in enumerate(tokens): # the target ends with ':' (a drive letter's colon is followed by a separator)
|
||||
if t.endswith(':'):
|
||||
return tokens[k + 1:]
|
||||
return tokens
|
||||
|
||||
|
||||
class ObjectCache:
|
||||
"""Which objects are current: per object the key of its command and source, and the sha256 of every file its
|
||||
dependency list names (build-cache/obj/objects.json)."""
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
try:
|
||||
self.entries = json.loads(path.read_text(encoding='utf-8')) if path.is_file() else {}
|
||||
except (OSError, ValueError):
|
||||
self.entries = {}
|
||||
self._sha = {}
|
||||
|
||||
def file_sha(self, p):
|
||||
p = os.path.normcase(os.path.abspath(p))
|
||||
if p not in self._sha:
|
||||
try:
|
||||
self._sha[p] = sha256_file(p)
|
||||
except OSError:
|
||||
self._sha[p] = None
|
||||
return self._sha[p]
|
||||
|
||||
def current(self, obj, key):
|
||||
e = self.entries.get(obj.name)
|
||||
return (e is not None and e.get('key') == key and obj.is_file()
|
||||
and all(self.file_sha(p) == h for p, h in e.get('deps', {}).items()))
|
||||
|
||||
def record(self, obj, key, dep_file, src):
|
||||
try:
|
||||
deps = parse_deps(Path(dep_file).read_text(encoding='utf-8', errors='replace'))
|
||||
except OSError:
|
||||
deps = []
|
||||
deps = deps or [str(src)]
|
||||
self.entries[obj.name] = {'key': key, 'deps': {p: self.file_sha(p) for p in deps}}
|
||||
|
||||
def forget(self, obj):
|
||||
self.entries.pop(obj.name, None)
|
||||
|
||||
def save(self):
|
||||
tmp = self.path.with_suffix('.tmp')
|
||||
tmp.write_text(json.dumps(self.entries), encoding='utf-8')
|
||||
tmp.replace(self.path)
|
||||
|
||||
|
||||
def compiler_id(clang, env):
|
||||
p = subprocess.run([str(clang), '--version'], capture_output=True, text=True, env=env)
|
||||
st = os.stat(clang)
|
||||
return sha256_bytes(f'{p.stdout}\n{st.st_size}\n{int(st.st_mtime)}'.encode())
|
||||
|
||||
|
||||
def compile_all(srcs, cmd_for, cache, jobs, env, progress=None, salt=''):
|
||||
"""Compile each (source, object) whose object is not current in cache. cmd_for(src, dst) -> argv without the
|
||||
dependency-file options; salt identifies the compiler. Returns (compiled, reused, failures) with failures as
|
||||
(src, stderr tail)."""
|
||||
todo, keys = [], {}
|
||||
for src, dst in srcs:
|
||||
cmd = cmd_for(src, dst)
|
||||
flags = [c for c in cmd if c not in (str(src), str(dst))]
|
||||
keys[dst] = sha256_bytes(json.dumps([salt, flags, cache.file_sha(src)]).encode())
|
||||
if not cache.current(dst, keys[dst]):
|
||||
todo.append((src, dst))
|
||||
reused = len(srcs) - len(todo)
|
||||
|
||||
def cc(job):
|
||||
src, dst = job
|
||||
dep = dst.with_suffix('.d')
|
||||
cmd = cmd_for(src, dst)
|
||||
cmd[1:1] = ['-MMD', '-MF', str(dep)]
|
||||
p = subprocess.run(cmd, capture_output=True, text=True, env=env, errors='replace')
|
||||
return src, dst, dep, p.returncode, p.stderr[-1500:]
|
||||
|
||||
done, bad = reused, []
|
||||
if progress:
|
||||
progress(done, len(srcs))
|
||||
with concurrent.futures.ThreadPoolExecutor(max(1, jobs)) as ex:
|
||||
for src, dst, dep, rc, err in ex.map(cc, todo):
|
||||
done += 1
|
||||
if rc:
|
||||
bad.append((src, err))
|
||||
cache.forget(dst)
|
||||
else:
|
||||
cache.record(dst, keys[dst], dep, src)
|
||||
if progress:
|
||||
progress(done, len(srcs))
|
||||
cache.save()
|
||||
return len(todo), reused, bad
|
||||
|
||||
|
||||
def step_compile(out, cache_dir, msys, jobs):
|
||||
lift, gen, obj = cache_dir / 'lift', cache_dir / 'gen', cache_dir / 'obj'
|
||||
gen.mkdir(parents=True, exist_ok=True)
|
||||
obj.mkdir(parents=True, exist_ok=True)
|
||||
rt = KIT / 'runtime'
|
||||
clang, clangxx = msys / 'bin' / 'clang.exe', msys / 'bin' / 'clang++.exe'
|
||||
env = dict(os.environ, PATH=str(msys / 'bin') + os.pathsep + os.environ.get('PATH', ''))
|
||||
if subprocess.run([sys.executable, str(KIT / 'tools' / 'romfs_index.py'), '--romfs', str(out / 'romfs'),
|
||||
'--out', str(gen)]).returncode:
|
||||
if subprocess.run(py_cmd(KIT / 'tools' / 'romfs_index.py', '--romfs', out / 'romfs', '--out', gen)).returncode:
|
||||
fail('the RomFS index could not be written')
|
||||
srcs = [(p, obj / f'{mod}_{p.stem}.o') for mod in ('main', 'sdk', 'subsdk0') for p in sorted((lift / mod).glob('*.c'))]
|
||||
for pat in ('*.c', '*.cpp', 'platform/*.c', 'ui/*.cpp', 'third_party/imgui/*.cpp', 'third_party/imgui/backends/*.cpp'):
|
||||
@@ -356,53 +591,74 @@ def step_compile(out, cache, msys, jobs):
|
||||
srcs += [(modules_c(lift, out, gen), obj / 'modules.o'), (gen / 'shim.c', obj / 'shim.o')]
|
||||
inc = ['-I', str(rt), '-I', str(lift), '-I', str(gen), '-I', str(rt / 'third_party' / 'imgui'),
|
||||
'-I', str(msys / 'include' / 'vma')]
|
||||
cid = compiler_id(clang, env)
|
||||
|
||||
def cc(job):
|
||||
src, dst = job
|
||||
def cmd_for(src, dst):
|
||||
cpp = src.suffix in ('.cpp', '.cxx')
|
||||
# the lifted code at -O1 (its size makes -O2 slow to build), the runtime at -O2; neither may assume strict aliasing:
|
||||
# the lifted code reaches guest memory through casts of one address to several types (tools/build_game.py's flags)
|
||||
guest = dst.name.startswith(('main_', 'sdk_', 'subsdk0_'))
|
||||
cmd = [str(clangxx if cpp else clang), '-c', '-std=gnu++17' if cpp else '-std=gnu11', '-O1' if guest else '-O2',
|
||||
'-fno-strict-aliasing', '-w', *inc, '-o', str(dst), str(src)]
|
||||
p = subprocess.run(cmd, capture_output=True, text=True, env=env)
|
||||
return src, p.returncode, p.stderr[-1500:]
|
||||
return [str(clangxx if cpp else clang), '-c', '-std=gnu++17' if cpp else '-std=gnu11', '-O1' if guest else '-O2',
|
||||
'-fno-strict-aliasing', '-w', *inc, '-o', str(dst), str(src)]
|
||||
|
||||
t0 = time.time()
|
||||
last = [0.0]
|
||||
|
||||
def progress(done, total):
|
||||
now = time.time()
|
||||
if done == total or now - last[0] >= 0.25:
|
||||
last[0] = now
|
||||
emit('progress', step='compile', done=done, total=total, unit='files')
|
||||
if done % 50 == 0 or done == total:
|
||||
say(f' compiled {done} of {total}, {now - t0:.0f} s')
|
||||
|
||||
say(f' {len(srcs)} files on {jobs} threads')
|
||||
t0, done, bad = time.time(), 0, []
|
||||
with concurrent.futures.ThreadPoolExecutor(jobs) as ex:
|
||||
for src, rc, err in ex.map(cc, srcs):
|
||||
done += 1
|
||||
if rc:
|
||||
bad.append((src, err))
|
||||
if done % 50 == 0 or done == len(srcs):
|
||||
say(f' compiled {done} of {len(srcs)}, {time.time() - t0:.0f} s')
|
||||
cache = ObjectCache(obj / 'objects.json')
|
||||
compiled, reused, bad = compile_all(srcs, cmd_for, cache, jobs, env, progress, salt=cid)
|
||||
say(f' {compiled} compiled, {reused} unchanged since the last build')
|
||||
if bad:
|
||||
for src, err in bad[:3]:
|
||||
print(f'--- {src}\n{err}', file=sys.stderr)
|
||||
fail(f'{len(bad)} files did not compile')
|
||||
|
||||
# the manifest (per-monitor DPI awareness: the picture at the monitor's real size, not one Windows stretches) and the
|
||||
# icon, runtime/nx2navi.rc through MSYS2's windres
|
||||
rco = obj / 'nx2navi_rc.o'
|
||||
r = subprocess.run([str(msys / 'bin' / 'windres.exe'), '-I', str(rt), '-O', 'coff', '-i', str(rt / 'nx2navi.rc'),
|
||||
'-o', str(rco)], capture_output=True, text=True, env=env)
|
||||
'-o', str(rco)], capture_output=True, text=True, env=env, errors='replace')
|
||||
if r.returncode:
|
||||
fail(f'windres failed on runtime/nx2navi.rc: {r.stderr.strip()[:400]}')
|
||||
rsp = cache / 'link.rsp'
|
||||
rsp.write_text('\n'.join('"%s"' % o.as_posix() for o in [o for _, o in srcs] + [rco]), encoding='utf-8')
|
||||
exe = out / 'ocarina.exe'
|
||||
say(' linking')
|
||||
p = subprocess.run([str(clangxx), '-o', str(exe), '@' + str(rsp), '-L', str(msys / 'lib'), *LINK_LIBS],
|
||||
capture_output=True, text=True, env=env)
|
||||
if p.returncode:
|
||||
print(p.stderr[-4000:], file=sys.stderr)
|
||||
fail('the link failed')
|
||||
shutil.copy2(gen / 'romfs-index.bin', out / 'romfs-index.bin')
|
||||
objs = [o for _, o in srcs] + [rco]
|
||||
exe = out / EXE_NAME
|
||||
rsp = cache_dir / 'link.rsp'
|
||||
rsp.write_text('\n'.join('"%s"' % o.as_posix() for o in objs), encoding='utf-8')
|
||||
link_key = sha256_bytes(json.dumps([LINK_LIBS, cid] + [[o.name, o.stat().st_size, o.stat().st_mtime_ns]
|
||||
for o in objs]).encode())
|
||||
state_link = cache_dir / 'link.key'
|
||||
if exe.is_file() and state_link.is_file() and state_link.read_text(encoding='utf-8') == link_key:
|
||||
say(' ocarina.exe is current, not linked again')
|
||||
else:
|
||||
emit('phase', step='compile', phase='link')
|
||||
say(' linking')
|
||||
tmp = out / (EXE_NAME + '.new')
|
||||
p = subprocess.run([str(clangxx), '-o', str(tmp), '@' + str(rsp), '-L', str(msys / 'lib'), *LINK_LIBS],
|
||||
capture_output=True, text=True, env=env, errors='replace')
|
||||
if p.returncode:
|
||||
print(p.stderr[-4000:], file=sys.stderr)
|
||||
tmp.unlink(missing_ok=True)
|
||||
fail('the link failed')
|
||||
try:
|
||||
os.replace(tmp, exe)
|
||||
except PermissionError:
|
||||
tmp.unlink(missing_ok=True)
|
||||
fail(f'{exe} is in use: close the game and try again')
|
||||
state_link.write_text(link_key, encoding='utf-8')
|
||||
copy_if_changed(gen / 'romfs-index.bin', out / 'romfs-index.bin')
|
||||
for name in ('nx2navi_xlate.exe', 'nx2navi_xlate.id'):
|
||||
shutil.copy2(KIT / 'bin' / name, out / name)
|
||||
copy_if_changed(KIT / 'bin' / name, out / name)
|
||||
if (KIT / 'bin' / 'amd_fidelityfx_vk.dll').is_file(): # FSR 3.1 (runtime/vk_ffx.cpp)
|
||||
shutil.copy2(KIT / 'bin' / 'amd_fidelityfx_vk.dll', out / 'amd_fidelityfx_vk.dll')
|
||||
say(f' ocarina.exe {exe.stat().st_size / 1e6:.0f} MB; beside it: {", ".join(copy_dlls(exe, msys / "bin", out))}')
|
||||
copy_if_changed(KIT / 'bin' / 'amd_fidelityfx_vk.dll', out / 'amd_fidelityfx_vk.dll')
|
||||
say(f' {EXE_NAME} {exe.stat().st_size / 1e6:.0f} MB; beside it: {", ".join(copy_dlls(exe, msys / "bin", out))}')
|
||||
(out / 'profile').mkdir(exist_ok=True)
|
||||
|
||||
|
||||
@@ -432,56 +688,111 @@ def keep_awake():
|
||||
pass
|
||||
|
||||
|
||||
def load_state(path):
|
||||
try:
|
||||
state = json.loads(path.read_text(encoding='utf-8')) if path.is_file() else {}
|
||||
except (OSError, ValueError):
|
||||
state = {}
|
||||
if state.get('version') == 1: # the first kit: a list of finished steps, all tied to one kit version
|
||||
done = state.get('done', [])
|
||||
state = {'version': STATE_VERSION, 'input': state.get('input'), 'steps': {}}
|
||||
for step in ('exefs', 'romfs'):
|
||||
if step in done:
|
||||
state['steps'][step] = step_key(step, state['input'])
|
||||
return state
|
||||
|
||||
|
||||
def step_key(step, identity, out=None):
|
||||
"""What a step's output depends on. exefs and romfs: the game files only; lift: the unpacked ExeFS and the
|
||||
lifter; compile: the lift, the runtime sources and this script."""
|
||||
if step in ('exefs', 'romfs'):
|
||||
return f'{step}-1|{identity}'
|
||||
lift = f'lift|{exefs_digest(out)}|{lift_digest()}'
|
||||
if step == 'lift':
|
||||
return lift
|
||||
compile_key = f'compile|{lift}|{tree_digest(KIT / "runtime", ("**/*",))}|{sha256_file(Path(__file__))}|{kit_version()}'
|
||||
if step == 'compile':
|
||||
return compile_key
|
||||
# shaders: the built game and the translator that prepared them
|
||||
return f'shaders-1|{compile_key}|{sha256_file(KIT / "tools" / "prepare_shaders.py")}|{sha256_file(KIT / "bin" / "nx2navi_xlate.id")}'
|
||||
|
||||
|
||||
def main():
|
||||
global EVENTS
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument('--nup', help='the game as one .nup file')
|
||||
ap.add_argument('--exefs', help='the game\'s exefs folder (with --romfs, instead of --nup)')
|
||||
ap.add_argument('--romfs', help='the game\'s romfs folder')
|
||||
ap.add_argument('--rebuild', action='store_true',
|
||||
help='build again from the game files the last run unpacked into --out (after a kit update)')
|
||||
ap.add_argument('--out', help='the folder for the finished game')
|
||||
ap.add_argument('--check', action='store_true', help='only check the game\'s version; builds nothing')
|
||||
ap.add_argument('--msys', default=r'C:\msys64\ucrt64', help='MSYS2 UCRT64 folder (default C:\\msys64\\ucrt64)')
|
||||
ap.add_argument('--jobs', type=int, default=os.cpu_count() or 4, help='parallel compiles')
|
||||
ap.add_argument('--events', action='store_true', help='machine-readable progress lines on stdout ("@@" + JSON)')
|
||||
ap.add_argument('--until', choices=STEPS, help=argparse.SUPPRESS) # tests: stop after this step
|
||||
a = ap.parse_args()
|
||||
if bool(a.nup) == bool(a.exefs or a.romfs) or (not a.nup and not (a.exefs and a.romfs)):
|
||||
ap.error('give --nup, or both --exefs and --romfs')
|
||||
src = Source(a)
|
||||
EVENTS = a.events
|
||||
given = bool(a.nup) + bool(a.exefs or a.romfs) + bool(a.rebuild)
|
||||
if given != 1 or (not a.nup and not a.rebuild and not (a.exefs and a.romfs)):
|
||||
ap.error('give --nup, both --exefs and --romfs, or --rebuild')
|
||||
if a.rebuild and not a.out:
|
||||
ap.error('--rebuild needs --out')
|
||||
out = Path(a.out).resolve() if a.out else None
|
||||
cache = out / 'build-cache' if out else None
|
||||
state_path = cache / 'state.json' if cache else None
|
||||
state = load_state(state_path) if state_path else {}
|
||||
|
||||
emit('step', step='check', state='start')
|
||||
src = BuiltSource(out, state) if a.rebuild else Source(a)
|
||||
check(src)
|
||||
say('check: Ocarina of Time 1.1.1 (build e0f3ccfa)')
|
||||
emit('check', version='1.1.1', build=MAIN_BUILD_ID[:8])
|
||||
emit('step', step='check', state='done')
|
||||
if a.check:
|
||||
return
|
||||
if not a.out:
|
||||
if not out:
|
||||
ap.error('--out is needed to build')
|
||||
msys = Path(a.msys)
|
||||
for need in ('bin/clang.exe', 'bin/clang++.exe', 'bin/SDL3.dll', 'lib/libSDL3.dll.a', 'include/vma'):
|
||||
if not (msys / need).exists():
|
||||
fail(f'{msys / need} is missing: install MSYS2 and the packages README-BuildKit.md lists')
|
||||
for mod in ('numpy', 'lz4', 'zbic'):
|
||||
for mod in ('numpy', 'lz4') + (() if a.rebuild else ('zbic',)):
|
||||
try:
|
||||
__import__(mod)
|
||||
except ImportError:
|
||||
fail(f'the Python package {mod} is missing: pip install numpy lz4 zstandard zbic')
|
||||
out = Path(a.out).resolve()
|
||||
cache = out / 'build-cache'
|
||||
cache.mkdir(parents=True, exist_ok=True)
|
||||
state_path = cache / 'state.json'
|
||||
state = json.loads(state_path.read_text(encoding='utf-8')) if state_path.is_file() else {}
|
||||
if state.get('input') != src.identity or state.get('version') != STATE_VERSION or state.get('kit') != kit_version():
|
||||
state = {'input': src.identity, 'version': STATE_VERSION, 'kit': kit_version(), 'done': []}
|
||||
if state.get('version') != STATE_VERSION or state.get('input') != src.identity:
|
||||
state = {'version': STATE_VERSION, 'input': src.identity, 'steps': {}}
|
||||
state['kit'] = kit_version()
|
||||
keep_awake()
|
||||
t0 = time.time()
|
||||
for name, run in (('exefs', lambda: step_exefs(src, out)), ('romfs', lambda: step_romfs(src, out, min(a.jobs, 8))),
|
||||
('lift', lambda: step_lift(out, cache)), ('compile', lambda: step_compile(out, cache, msys, a.jobs)),
|
||||
('shaders', lambda: step_shaders(out))):
|
||||
if name in state['done']:
|
||||
runs = {'exefs': lambda: step_exefs(src, out), 'romfs': lambda: step_romfs(src, out, min(a.jobs, 8)),
|
||||
'lift': lambda: step_lift(out, cache), 'compile': lambda: step_compile(out, cache, msys, a.jobs),
|
||||
'shaders': lambda: step_shaders(out)}
|
||||
for name in STEPS:
|
||||
key = step_key(name, src.identity, out)
|
||||
if state['steps'].get(name) == key:
|
||||
say(f'{name}: done before')
|
||||
continue
|
||||
say(f'{name}:')
|
||||
t = time.time()
|
||||
run()
|
||||
state['done'].append(name)
|
||||
state_path.write_text(json.dumps(state, indent=1), encoding='utf-8')
|
||||
say(f'{name}: {time.time() - t:.0f} s')
|
||||
say(f'\nfinished in {(time.time() - t0) / 60:.1f} min: {out / "ocarina.exe"}')
|
||||
emit('step', step=name, state='skip')
|
||||
else:
|
||||
# a step that runs makes every step after it run too: their keys hash its output, not its key
|
||||
for later in STEPS[STEPS.index(name):]:
|
||||
state['steps'].pop(later, None)
|
||||
say(f'{name}:')
|
||||
emit('step', step=name, state='start')
|
||||
t = time.time()
|
||||
runs[name]()
|
||||
state['steps'][name] = step_key(name, src.identity, out)
|
||||
state_path.write_text(json.dumps(state, indent=1), encoding='utf-8')
|
||||
say(f'{name}: {time.time() - t:.0f} s')
|
||||
emit('step', step=name, state='done', seconds=round(time.time() - t, 1))
|
||||
if name == a.until:
|
||||
break
|
||||
state_path.write_text(json.dumps(state, indent=1), encoding='utf-8')
|
||||
say(f'\nfinished in {(time.time() - t0) / 60:.1f} min: {out / EXE_NAME}')
|
||||
emit('end', exe=str(out / EXE_NAME), seconds=round(time.time() - t0, 1), kit=kit_version())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
301
tools/make_launcher.py
Normal file
301
tools/make_launcher.py
Normal file
@@ -0,0 +1,301 @@
|
||||
"""Build a release of the NX2navi launcher (docs/launcher.md, "Releases").
|
||||
|
||||
python tools/make_launcher.py --xlate build/xlate [--ffx amd_fidelityfx_vk.dll] [--out build/launcher-release]
|
||||
[--feed owner/name]
|
||||
[--notes notes.md] [--publish]
|
||||
|
||||
Needs Python 3.14 (tarfile and zstd for the MSYS2 packages), uv on PATH, network access on the first run, and
|
||||
either Windows (msiexec, and the C compiler of MSYS2 it downloads itself) or Linux with clang, lld and llvm-windres
|
||||
(it cross-compiles against the same MSYS2 packages) plus msitools. Downloads are cached in toolchain/launcher-dl/ and
|
||||
checked against the sha256 in PINS.
|
||||
|
||||
The release folder (--out) holds:
|
||||
|
||||
NX2navi/ the launcher folder exactly as NX2navi-Setup.exe installs it:
|
||||
NX2navi.exe launcher/native/nx2navi.c with the Triforce icon (launcher/ui/art.py)
|
||||
python/ python.org's embeddable Python 3.14.9, tkinter and Tcl/Tk 9.0 from its tcltk.msi,
|
||||
the packages of tools/requirements-launcher.txt, runtime-id.txt
|
||||
app/current.txt, app/<version>/ launcher/ (without native/), runtime/, recomp/, the tools make_game.py uses,
|
||||
bin/nx2navi_xlate.exe and .id (and amd_fidelityfx_vk.dll with --ffx), toolchain.lock.json (the MSYS2 packages of today,
|
||||
pinned), feed.json, VERSION, the licence files
|
||||
NX2navi-Setup-<version>.exe launcher/native/setup.c with the folder above appended as a zip: the one file
|
||||
players download
|
||||
NX2navi-app-<version>.zip app/<version> alone: what the launcher downloads to update itself
|
||||
NX2navi-runtime-<id>.zip python/ alone: downloaded only when a release changes the Python
|
||||
NX2navi-update.json the manifest the launcher reads (launcher/feed.py)
|
||||
|
||||
--publish uploads the four files as GitHub release v<version> of the --feed repository with the gh CLI (a draft
|
||||
first is wiser: add --draft). check_content runs over everything before it is packed: the release holds no game data.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
sys.path.insert(0, str(ROOT / 'tools'))
|
||||
import check_content # noqa: E402
|
||||
from launcher import toolchain # noqa: E402
|
||||
from launcher.paths import Layout # noqa: E402
|
||||
|
||||
MAIN_ROOT = (ROOT / 'build').resolve().parent # build/ and toolchain/ are junctions in a worktree
|
||||
DL = MAIN_ROOT / 'toolchain' / 'launcher-dl'
|
||||
WORK = MAIN_ROOT / 'toolchain' / 'launcher-work'
|
||||
PY_VERSION = '3.14.9'
|
||||
PINS = {
|
||||
'python-embed': (f'https://www.python.org/ftp/python/{PY_VERSION}/python-{PY_VERSION}-embed-amd64.zip',
|
||||
'3c01d7cfc666f746a6f064f60fb2156ab0ec464d3d738ab796371a46a9bb24a2'),
|
||||
# the installer's own Tcl/Tk and tkinter component, published beside it
|
||||
'tcltk': (f'https://www.python.org/ftp/python/{PY_VERSION}/amd64/tcltk.msi',
|
||||
'ac87609db3f715404dd41a989c2f1706ea1512f3d86217f938d11c63af8eb505'),
|
||||
}
|
||||
TOOLS = ('make_game.py', 'nup.py', 'romfs_index.py', 'check_content.py', 'hostmem.py', 'prepare_shaders.py',
|
||||
'shader_census.py', 'shader_list.py', 'zstd_compat.py')
|
||||
TOP = ('LICENSE', 'LICENSE-ADDITIONS', 'THIRD_PARTY_NOTICES.md', 'nx2navi.ini.example')
|
||||
SKIP_DIRS = {'__pycache__', '.pytest_cache', 'native'}
|
||||
SKIP_SUFFIXES = {'.o', '.obj', '.pyc', '.exe', '.dll', '.a', '.lib'}
|
||||
TCLTK_FILES = ('_tkinter.pyd', 'tcl90.dll', 'tcl9tk90.dll', 'zlib1.dll', 'libtommath.dll')
|
||||
DEFAULT_FEED = 'tangerineturtle1/NX2navi'
|
||||
|
||||
|
||||
def say(msg):
|
||||
print(msg, flush=True)
|
||||
|
||||
|
||||
def sha256_file(path):
|
||||
h = hashlib.sha256()
|
||||
with open(path, 'rb') as f:
|
||||
while b := f.read(1 << 20):
|
||||
h.update(b)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def fetch(name):
|
||||
url, want = PINS[name]
|
||||
dest = DL / url.rsplit('/', 1)[1]
|
||||
if dest.is_file() and sha256_file(dest) == want:
|
||||
return dest
|
||||
from launcher import net
|
||||
say(f'download {url}')
|
||||
net.download(url, dest, sha256=want)
|
||||
return dest
|
||||
|
||||
|
||||
def run(cmd, **kw):
|
||||
say(' ' + ' '.join(str(c) for c in cmd))
|
||||
subprocess.run([str(c) for c in cmd], check=True, **kw)
|
||||
|
||||
|
||||
def copy_tree(src, dst, skip_dirs=SKIP_DIRS):
|
||||
for p in sorted(src.rglob('*')):
|
||||
rel = p.relative_to(src)
|
||||
if any(part in skip_dirs for part in rel.parts) or p.is_dir() or p.suffix.lower() in SKIP_SUFFIXES:
|
||||
continue
|
||||
(dst / rel).parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(p, dst / rel)
|
||||
|
||||
|
||||
def version():
|
||||
v = (ROOT / 'VERSION').read_text(encoding='utf-8').strip()
|
||||
r = subprocess.run(['git', '-C', str(ROOT), 'rev-parse', '--short', 'HEAD'], capture_output=True, text=True)
|
||||
return v, f'{v}+{r.stdout.strip()}' if r.returncode == 0 and r.stdout.strip() else v
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- python/
|
||||
|
||||
def tcltk_files():
|
||||
dest = WORK / 'tcltk'
|
||||
if (dest / 'DLLs' / '_tkinter.pyd').is_file():
|
||||
return dest
|
||||
msi = fetch('tcltk')
|
||||
shutil.rmtree(dest, ignore_errors=True)
|
||||
dest.mkdir(parents=True)
|
||||
if os.name == 'nt': # an administrative image: copies the files, installs nothing
|
||||
run(['msiexec', '/a', msi, '/qn', f'TARGETDIR={dest}'])
|
||||
else:
|
||||
run(['msiextract', '-C', dest, msi], stdout=subprocess.DEVNULL)
|
||||
if not (dest / 'DLLs' / '_tkinter.pyd').is_file():
|
||||
sys.exit(f'{dest}: no _tkinter.pyd in {msi.name}')
|
||||
return dest
|
||||
|
||||
|
||||
def runtime_id():
|
||||
req = (ROOT / 'tools' / 'requirements-launcher.txt').read_bytes()
|
||||
return f'py{PY_VERSION}-{hashlib.sha256(req + PINS["tcltk"][1].encode()).hexdigest()[:8]}'
|
||||
|
||||
|
||||
def build_python(py):
|
||||
with zipfile.ZipFile(fetch('python-embed')) as z:
|
||||
z.extractall(py)
|
||||
# The embeddable Python's sys.path is exactly this list; ../pydeps holds zbic (launcher/toolchain.py).
|
||||
(py / 'python314._pth').write_text('python314.zip\n.\nLib/site-packages\n../pydeps\n', encoding='utf-8')
|
||||
env = dict(os.environ, UV_CACHE_DIR=str(MAIN_ROOT / 'toolchain' / 'uv-cache'))
|
||||
run(['uv', 'pip', 'install', '--target', py / 'Lib' / 'site-packages', '--python', sys.executable,
|
||||
'--python-platform', 'windows', '--python-version', '3.14', '--only-binary', ':all:', '-q',
|
||||
'-r', ROOT / 'tools' / 'requirements-launcher.txt'], env=env)
|
||||
shutil.rmtree(py / 'Lib' / 'site-packages' / 'bin', ignore_errors=True)
|
||||
tk = tcltk_files()
|
||||
for dll in TCLTK_FILES:
|
||||
if (tk / 'DLLs' / dll).is_file():
|
||||
shutil.copy2(tk / 'DLLs' / dll, py / dll)
|
||||
copy_tree(tk / 'Lib' / 'tkinter', py / 'tkinter', skip_dirs={'__pycache__', 'test'})
|
||||
copy_tree(tk / 'tcl', py / 'tcl', skip_dirs={'__pycache__'})
|
||||
(py / 'runtime-id.txt').write_text(runtime_id() + '\n', encoding='utf-8')
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- app/<version>/
|
||||
|
||||
def build_app(app, ver, xlate, feed_repo, lock, ffx=None):
|
||||
copy_tree(ROOT / 'launcher', app / 'launcher')
|
||||
for d in ('runtime', 'recomp'):
|
||||
copy_tree(ROOT / d, app / d)
|
||||
(app / 'tools').mkdir(parents=True)
|
||||
for t in TOOLS:
|
||||
shutil.copy2(ROOT / 'tools' / t, app / 'tools' / t)
|
||||
for t in TOP:
|
||||
if (ROOT / t).is_file():
|
||||
shutil.copy2(ROOT / t, app / t)
|
||||
(app / 'VERSION').write_text(ver + '\n', encoding='utf-8')
|
||||
(app / 'bin').mkdir()
|
||||
for n in ('nx2navi_xlate.exe', 'nx2navi_xlate.id'):
|
||||
if not (xlate / n).is_file():
|
||||
sys.exit(f'{xlate / n} is missing: build the shader translator first (tools/build_xlate.py)')
|
||||
shutil.copy2(xlate / n, app / 'bin' / n)
|
||||
if ffx: # FSR 3.1 loads it from beside the game (runtime/vk_ffx.cpp); make_game.py copies it there
|
||||
shutil.copy2(ffx, app / 'bin' / 'amd_fidelityfx_vk.dll')
|
||||
(app / 'toolchain.lock.json').write_text(json.dumps(lock, indent=1), encoding='utf-8')
|
||||
(app / 'feed.json').write_text(json.dumps({'type': 'github', 'repo': feed_repo}, indent=1), encoding='utf-8')
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- the native programs
|
||||
|
||||
def native_compiler(lock):
|
||||
"""(cc argv prefix, windres argv prefix) for x86_64 Windows: MSYS2's own clang on Windows, else the host's clang
|
||||
against the same MSYS2 packages as a sysroot."""
|
||||
lay = Layout(root=MAIN_ROOT / 'toolchain' / 'launcher-msys')
|
||||
if not toolchain.is_current(lay, lock):
|
||||
say('MSYS2 packages for the native programs')
|
||||
toolchain.install(lay, lock, log=say)
|
||||
sysroot = lay.msys
|
||||
if os.name == 'nt':
|
||||
return [str(sysroot / 'bin' / 'clang.exe')], [str(sysroot / 'bin' / 'windres.exe')]
|
||||
for tool in ('clang', 'ld.lld'):
|
||||
if not shutil.which(tool):
|
||||
sys.exit(f'{tool} is needed to cross-compile the launcher programs')
|
||||
windres = shutil.which('llvm-windres') or shutil.which('llvm-windres-18') or '/usr/lib/llvm-18/bin/llvm-windres'
|
||||
if not Path(windres).exists():
|
||||
sys.exit('llvm-windres is needed to compile the launcher programs\' resources')
|
||||
return (['clang', '--target=x86_64-w64-mingw32', f'--sysroot={sysroot}', '-fuse-ld=lld'],
|
||||
[windres, '--target=x86_64-w64-mingw32'])
|
||||
|
||||
|
||||
def build_native(work, cc, windres):
|
||||
from launcher.ui import art
|
||||
work.mkdir(parents=True, exist_ok=True)
|
||||
(work / 'nx2navi.ico').write_bytes(art.triforce_ico())
|
||||
shutil.copy2(ROOT / 'launcher' / 'native' / 'app.manifest', work / 'app.manifest')
|
||||
res = work / 'res.o'
|
||||
run([*windres, '-I', work, ROOT / 'launcher' / 'native' / 'res.rc', '-O', 'coff', '-o', res])
|
||||
flags = ['-municode', '-mwindows', '-O2', '-Wall', '-static', '-s']
|
||||
stub = work / 'NX2navi.exe'
|
||||
setup = work / 'setup-stub.exe'
|
||||
run([*cc, *flags, ROOT / 'launcher' / 'native' / 'nx2navi.c', res, '-o', stub, '-lshell32', '-lole32', '-luuid'])
|
||||
run([*cc, *flags, ROOT / 'launcher' / 'native' / 'setup.c', res, '-o', setup, '-l:libz.a', '-lshell32',
|
||||
'-lole32', '-luuid', '-ladvapi32'])
|
||||
return stub, setup
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- packing
|
||||
|
||||
def zip_dir(src, dest, prefix=''):
|
||||
with zipfile.ZipFile(dest, 'w', zipfile.ZIP_DEFLATED, compresslevel=9) as z:
|
||||
for p in sorted(src.rglob('*')):
|
||||
if p.is_file():
|
||||
z.write(p, (Path(prefix) / p.relative_to(src)).as_posix())
|
||||
return dest
|
||||
|
||||
|
||||
def make_setup(stub, payload, dest, ver):
|
||||
"""setup-stub.exe + the payload zip + the 48-byte trailer launcher/native/setup.c reads."""
|
||||
data = stub.read_bytes()
|
||||
off = len(data)
|
||||
z = payload.read_bytes()
|
||||
trailer = b'NX2NSET1' + off.to_bytes(8, 'little') + len(z).to_bytes(8, 'little') + \
|
||||
ver.encode('utf-8')[:24].ljust(24, b'\0')
|
||||
dest.write_bytes(data + z + trailer)
|
||||
return dest
|
||||
|
||||
|
||||
def check_folder(folder):
|
||||
bad = [f'{p.relative_to(folder)}: {m}' for p in sorted(folder.rglob('*')) if p.is_file()
|
||||
for m in check_content.problems_for(p, allow_large=True)]
|
||||
if bad:
|
||||
sys.exit('check_content refuses these release files:\n ' + '\n '.join(bad[:20]))
|
||||
|
||||
|
||||
def entry(path):
|
||||
return {'name': path.name, 'size': path.stat().st_size, 'sha256': sha256_file(path)}
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument('--xlate', type=Path, required=True, help='folder with nx2navi_xlate.exe and nx2navi_xlate.id')
|
||||
ap.add_argument('--ffx', type=Path, help="AMD's signed amd_fidelityfx_vk.dll (FidelityFX SDK 1.1.4, MIT) for FSR 3.1")
|
||||
ap.add_argument('--out', type=Path, default=MAIN_ROOT / 'build' / 'launcher-release')
|
||||
ap.add_argument('--feed', default=DEFAULT_FEED, help='the GitHub repository the launcher looks for releases in')
|
||||
ap.add_argument('--notes', type=Path, help='release notes (markdown) for the manifest and the GitHub release')
|
||||
ap.add_argument('--lock', type=Path, help='a toolchain.lock.json to reuse instead of resolving today\'s packages')
|
||||
ap.add_argument('--version', help='the release version instead of VERSION\'s (test releases)')
|
||||
ap.add_argument('--publish', action='store_true', help='upload as GitHub release v<version> with gh')
|
||||
ap.add_argument('--draft', action='store_true', help='with --publish: as a draft')
|
||||
a = ap.parse_args()
|
||||
if sys.version_info < (3, 14):
|
||||
sys.exit('Python 3.14 or later is needed (compression.zstd for the MSYS2 packages)')
|
||||
ver, build = version()
|
||||
if a.version:
|
||||
ver, build = a.version, a.version + build[len(ver):]
|
||||
out = a.out
|
||||
shutil.rmtree(out, ignore_errors=True)
|
||||
folder = out / 'NX2navi'
|
||||
t0 = time.time()
|
||||
lock = json.loads(a.lock.read_text(encoding='utf-8')) if a.lock else toolchain.make_lock()
|
||||
say(f'release {build}: toolchain lock with {len(lock["packages"])} packages')
|
||||
build_app(folder / 'app' / ver, ver, a.xlate, a.feed, lock, a.ffx)
|
||||
(folder / 'app' / 'current.txt').write_text(ver + '\n', encoding='utf-8')
|
||||
build_python(folder / 'python')
|
||||
cc, windres = native_compiler(lock)
|
||||
stub, setup_stub = build_native(out / 'native', cc, windres)
|
||||
shutil.copy2(stub, folder / 'NX2navi.exe')
|
||||
check_folder(folder)
|
||||
app_zip = zip_dir(folder / 'app' / ver, out / f'NX2navi-app-{ver}.zip')
|
||||
rt_zip = zip_dir(folder / 'python', out / f'NX2navi-runtime-{runtime_id()}.zip')
|
||||
payload = zip_dir(folder, out / 'native' / 'payload.zip')
|
||||
setup = make_setup(setup_stub, payload, out / f'NX2navi-Setup-{ver}.exe', ver)
|
||||
notes = a.notes.read_text(encoding='utf-8') if a.notes else ''
|
||||
manifest = {'format': 1, 'version': ver, 'build': build, 'notes': notes,
|
||||
'app': entry(app_zip), 'runtime': {'id': runtime_id(), **entry(rt_zip)},
|
||||
'toolchain': toolchain.lock_digest(lock), 'setup': entry(setup),
|
||||
'made': time.strftime('%Y-%m-%d %H:%M')}
|
||||
man = out / 'NX2navi-update.json'
|
||||
man.write_text(json.dumps(manifest, indent=1), encoding='utf-8')
|
||||
for p in (setup, app_zip, rt_zip, man):
|
||||
say(f'{p.name}: {p.stat().st_size / 1e6:.1f} MB')
|
||||
say(f'release folder {out} in {time.time() - t0:.0f} s')
|
||||
if a.publish:
|
||||
cmd = ['gh', 'release', 'create', f'v{ver}', setup, app_zip, rt_zip, man, '--repo', a.feed,
|
||||
'--title', f'NX2navi {ver}']
|
||||
cmd += ['--notes-file', a.notes] if a.notes else ['--notes', f'NX2navi {ver}']
|
||||
if a.draft:
|
||||
cmd.append('--draft')
|
||||
run(cmd)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
6
tools/requirements-launcher.txt
Normal file
6
tools/requirements-launcher.txt
Normal file
@@ -0,0 +1,6 @@
|
||||
# Packages of the launcher release's bundled Python 3.14.9 (tools/make_launcher.py): what the launcher and
|
||||
# tools/make_game.py import. numpy: recomp/gen.py and the window's drawing (launcher/ui/art.py). lz4: NSO segments.
|
||||
# zstandard: tools/nup.py. zbic is not bundled: the launcher downloads it from PyPI on the player's PC.
|
||||
numpy==2.5.4
|
||||
lz4==4.4.5
|
||||
zstandard==0.25.0
|
||||
Reference in New Issue
Block a user