mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-19 19:29:57 -04:00
Import the base tree (MIT, see LICENSE)
This commit is contained in:
618
tools/make_release.py
Normal file
618
tools/make_release.py
Normal file
@@ -0,0 +1,618 @@
|
||||
"""Assemble the release folder of docs/installer.md ("Release layout") from pinned downloads.
|
||||
|
||||
.venv-py314\\Scripts\\python tools\\make_release.py --out build/release/NX2navi-Setup [--zip build/release/NX2navi-Setup.zip]
|
||||
.venv-py314\\Scripts\\python tools\\make_release.py --out build/release/NX2navi-Setup --prove
|
||||
|
||||
Needs Python 3.14 (tarfile reads the MSYS2 .zst packages), uv on PATH and network access on the first run. Downloads are
|
||||
cached in toolchain/release-dl/ and checked against the sha256 in PINS; a mismatch stops the build. The folder holds:
|
||||
|
||||
python/ python.org embeddable 3.14.9 + the packages of tools/requirements-installer.txt + tkinter (the Tcl/Tk 9.0
|
||||
files come out of the python.org installer's own component MSI, unpacked with msiexec /a and expand.exe;
|
||||
the installer is never run)
|
||||
llvm/ llvm-mingw, x86_64 only, the files clang and lld need to compile and link nx2navi.exe
|
||||
runtime/ libnx2navirt.a and libnx2navirt-v2.a (build_game.py --make-runtime-lib with llvm/, x86-64-v3 and v2), the
|
||||
runtime headers, romfs.c, patches.c and ov_zstd.c (compiled per install), deps/ (headers and import
|
||||
libraries of SDL3, Opus, libvpx, Vulkan and VMA)
|
||||
bin/ nx2navi_xlate.exe (+ .id), hactool.exe, SDL3.dll (upstream), libopus-0.dll, ninja.exe
|
||||
recomp/, tools/, data/, installer/, licenses/, THIRD_PARTY_NOTICES.md, nx2navi.ini.example
|
||||
data/pgo-<v>.profdata build/pgo's execution profile of an update (tools/pgo.py) and its .json record: the
|
||||
install compiles that update's code and the runtime libraries with it
|
||||
VERSION the repository's VERSION plus the commit, 0.1.0+973c626; the installer's release string
|
||||
NX2navi-Setup.exe installer/launcher/launcher.c built with llvm/ (-mwindows): starts python/pythonw.exe -m installer.gui
|
||||
|
||||
--prove compiles the generated C and links build/release-check/nx2navi.exe with nothing but the folder's python/, llvm/,
|
||||
runtime/ and bin/ (PATH holds the Windows system directory and those), then runs the four probes there.
|
||||
|
||||
Both need one game version extracted and lifted first (tools/extract.py, recomp/gen.py): generated/v121 or, without
|
||||
it, generated/v143, and for --prove its build/extracted/<version> (they are read, nothing is written to them). The
|
||||
runtime library reads only what both versions' layout.h share (MOD_COUNT, MOD_MAIN_BASE), so either serves.
|
||||
|
||||
A release also needs build/xlate built from the current sources (tools/build_xlate.py); the script stops first if not.
|
||||
|
||||
The folder contains no code, data, text or assets of the game: check_content runs over it before the script finishes.
|
||||
The execution profiles name the lifted functions by their address (f_<address>) and hold branch counts and
|
||||
control-flow hashes of the generated C, nothing of the game's own bytes. Then the folder's own python runs a check-only
|
||||
installer pass (no game files: it must get through the release, toolchain and host checks and stop at the missing game
|
||||
files) and the window's --selftest.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import build_game # noqa: E402
|
||||
import build_xlate # noqa: E402
|
||||
import check_content # noqa: E402
|
||||
import shader_list # noqa: E402
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
MAIN_ROOT = (ROOT / 'build').resolve().parent # build/, generated/, toolchain/ and refs/ are junctions in a worktree
|
||||
DL = MAIN_ROOT / 'toolchain' / 'release-dl'
|
||||
WORK = MAIN_ROOT / 'toolchain' / 'release-work' # unpacked downloads
|
||||
PGO_DIR = MAIN_ROOT / 'build' / 'pgo' # pgo-<version>.profdata + .json (tools/pgo.py merge)
|
||||
|
||||
MSYS2 = 'https://repo.msys2.org/mingw/ucrt64/'
|
||||
SDL_VERSION = '3.4.18'
|
||||
LLVM_VERSION = '20261020'
|
||||
MNG_COMMIT = 'b75be8cfa704df3701e86130c8ced268e79d6b11' # MacNativeGames, the shader list (refs/MacNativeGames)
|
||||
# name: (url, sha256)
|
||||
PINS = {
|
||||
'python-embed': ('https://www.python.org/ftp/python/3.14.9/python-3.14.9-embed-amd64.zip',
|
||||
'3c01d7cfc666f746a6f064f60fb2156ab0ec464d3d738ab796371a46a9bb24a2'),
|
||||
'python-installer': ('https://www.python.org/ftp/python/3.14.9/python-3.14.9-amd64.exe',
|
||||
'a247a0b612244a803f507b39fb7121e3602e4160d59c3b99d98e735289c53394'),
|
||||
'llvm-mingw': (f'https://github.com/mstorsjo/llvm-mingw/releases/download/{LLVM_VERSION}/'
|
||||
f'llvm-mingw-{LLVM_VERSION}-ucrt-x86_64.zip',
|
||||
'fd5462b7f5d93b2ff981fcd3f9c7e427e47f3203c64ea91c27bcc444eebd74cd'),
|
||||
'sdl3-dll': (f'https://github.com/libsdl-org/SDL/releases/download/release-{SDL_VERSION}/SDL3-{SDL_VERSION}-win32-x64.zip',
|
||||
'75c2c0fc74e7d1206aaedc22893a35887332f4f85f5c8b74fddc10d177985f2c'),
|
||||
'sdl3-devel': (f'https://github.com/libsdl-org/SDL/releases/download/release-{SDL_VERSION}/SDL3-devel-{SDL_VERSION}-mingw.tar.gz',
|
||||
'049b9dd711e3fb3f8d84feb998dfec54a75e17cf539c4a80ff4d058d839771cf'),
|
||||
'ninja': ('https://github.com/ninja-build/ninja/releases/download/v1.13.2/ninja-win.zip',
|
||||
'07fc8261b42b20e71d1720b39068c2e14ffcee6396b76fb7a795fb460b78dc65'),
|
||||
'opus': (MSYS2 + 'mingw-w64-ucrt-x86_64-opus-1.6.1-2-any.pkg.tar.zst',
|
||||
'bb114bcf26225f50f874497ebfd6b8224fff2a20bda3f8ee45cd07ce5f02657b'),
|
||||
'libvpx': (MSYS2 + 'mingw-w64-ucrt-x86_64-libvpx-1.17.0-1-any.pkg.tar.zst',
|
||||
'8ff6919dbb88ea26ce28d22fd81d6f0d6e200cf56a4dd81ffc2d33124fe98024'),
|
||||
'libvpx-src': ('https://github.com/webmproject/libvpx/archive/refs/tags/v1.17.0.tar.gz', # LICENSE and PATENTS only
|
||||
'1020f184046187baa2985dbde38e0691f49c44088bca7a1842b0236c6081dc0a'),
|
||||
'vulkan-headers': (MSYS2 + 'mingw-w64-ucrt-x86_64-vulkan-headers-1~1.4.363.0-1-any.pkg.tar.zst',
|
||||
'be602b2657cace531316e78b1eda4087e8f7f84ebb877f08fba198a2db06406a'),
|
||||
'vulkan-loader': (MSYS2 + 'mingw-w64-ucrt-x86_64-vulkan-loader-1~1.4.363.0-1-any.pkg.tar.zst', # import library only
|
||||
'065076fe13d39ab97eea4740596426d5ae741b5044e48e1c9495d04835c92dbe'),
|
||||
'vma': (MSYS2 + 'mingw-w64-ucrt-x86_64-vulkan-memory-allocator-3.4.0-1-any.pkg.tar.zst',
|
||||
'44c81162337915afecda3185f66fd98268a54454580b2717080c9dfef57fe781'),
|
||||
'zstd': (MSYS2 + 'mingw-w64-ucrt-x86_64-zstd-1.5.7-2-any.pkg.tar.zst', # static libzstd.a (runtime/resolution.c)
|
||||
'dbdb8427280046a2b41697780aa4c52983b708082b0da4755951dc3bea96ca89'),
|
||||
}
|
||||
# The component MSI inside the python.org installer that carries Tcl/Tk and tkinter (Burn names its payloads a0..a27).
|
||||
TCLTK_PAYLOAD = ('a12', 'ac87609db3f715404dd41a989c2f1706ea1512f3d86217f938d11c63af8eb505')
|
||||
SDL3_DLL_SHA256 = '2b4b128d8d510cc2d442693cf848c050d700d43dc667afe9e5fd82b0570d667e' # SDL3.dll inside the SDL zip
|
||||
|
||||
# llvm-mingw: what clang and lld need to compile and link nx2navi.exe for x86_64-w64-mingw32 (dynamic LLVM, libc++, compiler-rt).
|
||||
LLVM_BIN = {'clang.exe', 'clang++.exe', 'clang-23.exe', 'ld.lld.exe', 'llvm-ar.exe', 'llvm-ranlib.exe', 'llvm-nm.exe',
|
||||
'llvm-objdump.exe', 'llvm-objcopy.exe', 'llvm-strip.exe', 'llvm-readobj.exe', 'llvm-dlltool.exe',
|
||||
'llvm-rc.exe', 'llvm-cvtres.exe', 'llvm-windres.exe', 'llvm-wrapper.exe', 'mingw32-common.cfg',
|
||||
'x86_64-w64-windows-gnu.cfg', 'x86_64-pc-windows-gnu.cfg', 'x86_64-w64-mingw32-clang.exe',
|
||||
'x86_64-w64-mingw32-clang++.exe', 'libLLVM-23.dll', 'libclang-cpp.dll', 'libc++.dll', 'libunwind.dll',
|
||||
'libwinpthread-1.dll', 'libffi-8.dll'}
|
||||
LLVM_SKIP_DIRS = {'aarch64-w64-mingw32', 'armv7-w64-mingw32', 'i686-w64-mingw32', 'busybox', 'python', 'share'}
|
||||
LLVM_SKIP_SUFFIXES = {'.idl', '.tlb'}
|
||||
|
||||
# tools/ files the installer steps run (and what they import), copied from the repo.
|
||||
TOOLS_FILES = ['build_game.py', 'build_shaders.py', 'button_icons.py', 'check_content.py', 'check_isa.py', 'extract.py',
|
||||
'hostmem.py', 'make_romfs_patches.py', 'ncz.py', 'pack_shader_cache.py', 'prepare_shaders.py',
|
||||
'romfs_index.py', 'shader_census.py', 'shader_list.py', 'ultrawide_layout.py', 'cheats.py',
|
||||
'exefs_patches.py']
|
||||
TOOLS_DIRS = ['vendor/macnativegames', 'data'] # data/: expected hashes
|
||||
RUNTIME_SUFFIXES = {'.h', '.inc', '.ico'} # .ico: the installer window icon
|
||||
|
||||
PROBES = ['--probe-kernel', '--probe-vkexec', '--probe-nvn-tex', '--probe-shader']
|
||||
|
||||
|
||||
def sha256_file(path):
|
||||
h = hashlib.sha256()
|
||||
with open(path, 'rb') as f:
|
||||
for block in iter(lambda: f.read(1 << 20), b''):
|
||||
h.update(block)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def fetch(name):
|
||||
"""The pinned download, from the cache or the network, checked against its sha256."""
|
||||
url, want = PINS[name]
|
||||
DL.mkdir(parents=True, exist_ok=True)
|
||||
path = DL / Path(url.split('?')[0]).name.replace('%7E', '~')
|
||||
if name in ('tcl-linux-src', 'tk-linux-src'):
|
||||
path = DL / f'{name}-9.0.4.tar.gz' # upstream archives have identical basenames
|
||||
if name == 'libvpx-src':
|
||||
path = DL / 'libvpx-1.17.0.tar.gz'
|
||||
if path.is_file():
|
||||
if sha256_file(path) == want:
|
||||
return path
|
||||
print(f'{path}: not the pinned file (a cut download or an older pin); downloading it again')
|
||||
path.unlink()
|
||||
# into <name>.part, renamed once complete: a cut connection leaves a .part that the next run resumes, never a
|
||||
# short file under the final name
|
||||
part = path.with_name(path.name + '.part')
|
||||
have = part.stat().st_size if part.is_file() else 0
|
||||
print(f'download {url}' + (f' (resuming at {have} bytes)' if have else ''))
|
||||
req = urllib.request.Request(url, headers={'Range': f'bytes={have}-'} if have else {})
|
||||
try:
|
||||
with urllib.request.urlopen(req) as r:
|
||||
if have and r.status != 206: # the server ignored the range: start over
|
||||
have = 0
|
||||
with open(part, 'ab' if have else 'wb') as f:
|
||||
shutil.copyfileobj(r, f)
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code != 416: # 416: the .part already holds the whole file
|
||||
raise
|
||||
part.replace(path)
|
||||
got = sha256_file(path)
|
||||
if got != want:
|
||||
path.unlink() # a .part resumed from another upload, or the URL now serves another file: start clean next run
|
||||
sys.exit(f'{path}: sha256 {got}, pinned {want}; deleted, run again to download it anew')
|
||||
return path
|
||||
|
||||
|
||||
def unpack(name, member_filter=None):
|
||||
"""Extract a pinned archive into toolchain/release-work/<name> (once) and return that directory."""
|
||||
dest = WORK / name
|
||||
stamp = dest / '.unpacked'
|
||||
if stamp.is_file():
|
||||
return dest
|
||||
shutil.rmtree(dest, ignore_errors=True)
|
||||
dest.mkdir(parents=True)
|
||||
path = fetch(name)
|
||||
if path.suffix == '.zip':
|
||||
with zipfile.ZipFile(path) as z:
|
||||
z.extractall(dest)
|
||||
else:
|
||||
with tarfile.open(path) as t:
|
||||
t.extractall(dest, filter='data')
|
||||
stamp.write_text('')
|
||||
return dest
|
||||
|
||||
|
||||
def copy_tree(src, dst, skip=lambda rel: False):
|
||||
"""Copy src into dst file by file (sorted), leaving out what skip(relative posix path) names."""
|
||||
for p in sorted(src.rglob('*')):
|
||||
rel = p.relative_to(src)
|
||||
if p.is_dir() or skip(rel.as_posix()):
|
||||
continue
|
||||
(dst / rel).parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(p, dst / rel)
|
||||
|
||||
|
||||
def clean_env(*path_dirs):
|
||||
"""A process environment without MSYS2 and the user's tools: the Windows system directory and path_dirs only."""
|
||||
system = os.environ.get('SystemRoot', r'C:\Windows')
|
||||
keep = ('SYSTEMROOT', 'SYSTEMDRIVE', 'TEMP', 'TMP', 'USERPROFILE', 'LOCALAPPDATA', 'APPDATA', 'COMSPEC', 'PATHEXT',
|
||||
'NUMBER_OF_PROCESSORS') # os.environ keys are upper case on Windows
|
||||
env = {k: v for k, v in os.environ.items() if k.upper() in keep}
|
||||
env['PATH'] = os.pathsep.join([*(str(Path(d)) for d in path_dirs), rf'{system}\System32', system])
|
||||
return env
|
||||
|
||||
|
||||
def run(cmd, **kw):
|
||||
print('+', ' '.join(str(c) for c in cmd), flush=True)
|
||||
return subprocess.run([str(c) for c in cmd], check=True, **kw)
|
||||
|
||||
|
||||
def mib(n):
|
||||
return f'{n / (1 << 20):.1f} MiB'
|
||||
|
||||
|
||||
def dir_size(p):
|
||||
return sum(f.stat().st_size for f in Path(p).rglob('*') if f.is_file())
|
||||
|
||||
|
||||
# ---- python/ -------------------------------------------------------------------------------------------------------
|
||||
|
||||
def tcltk_files():
|
||||
"""Directory holding the python.org installer's tcltk payload, unpacked (not installed) into toolchain/release-work."""
|
||||
dest = WORK / 'tcltk'
|
||||
if (dest / 'DLLs' / '_tkinter.pyd').is_file():
|
||||
return dest
|
||||
exe = fetch('python-installer')
|
||||
data = exe.read_bytes()
|
||||
cabs = []
|
||||
pos = data.find(b'MSCF\0\0\0\0')
|
||||
while pos >= 0:
|
||||
cabs.append((struct.unpack_from('<I', data, pos + 8)[0], pos))
|
||||
pos = data.find(b'MSCF\0\0\0\0', pos + 1)
|
||||
size, pos = max(cabs) # the attached container holds all component MSIs
|
||||
shutil.rmtree(WORK / 'tcltk-src', ignore_errors=True)
|
||||
src = WORK / 'tcltk-src'
|
||||
src.mkdir(parents=True)
|
||||
(src / 'container.cab').write_bytes(data[pos:pos + size])
|
||||
name, want = TCLTK_PAYLOAD
|
||||
expand = Path(os.environ.get('SystemRoot', r'C:\Windows')) / 'System32' / 'expand.exe'
|
||||
run([expand, '-F:' + name, src / 'container.cab', src], stdout=subprocess.DEVNULL)
|
||||
got = sha256_file(src / name)
|
||||
if got != want:
|
||||
sys.exit(f'{src / name}: sha256 {got}, pinned {want} (not the tcltk component of python 3.14.9)')
|
||||
msi = src / (name + '.msi')
|
||||
(src / name).rename(msi)
|
||||
shutil.rmtree(dest, ignore_errors=True)
|
||||
# Administrative image: copies the MSI's files to TARGETDIR, installs nothing and shows no window.
|
||||
run(['msiexec', '/a', msi, '/qn', f'TARGETDIR={dest}'])
|
||||
if not (dest / 'DLLs' / '_tkinter.pyd').is_file():
|
||||
sys.exit(f'{dest}: no _tkinter.pyd after msiexec /a')
|
||||
return dest
|
||||
|
||||
|
||||
def build_python(out):
|
||||
py = out / 'python'
|
||||
with zipfile.ZipFile(fetch('python-embed')) as z:
|
||||
z.extractall(py)
|
||||
(py / 'python314._pth').write_text('python314.zip\n.\nLib/site-packages\n..\n../tools\n../recomp\n', encoding='utf-8')
|
||||
# The embeddable Python ignores the script directory and PYTHONPATH once a ._pth file exists, so the folder's own code
|
||||
# (installer/, and tools/*.py and recomp/*.py importing their siblings) has to be on the list.
|
||||
env = dict(os.environ, UV_CACHE_DIR=str(MAIN_ROOT / 'toolchain' / 'uv-cache'))
|
||||
run(['uv', 'pip', 'install', '--target', py / 'Lib' / 'site-packages', '--python', sys.executable,
|
||||
'--python-platform', 'windows', '--python-version', '3.14', '--only-binary', ':all:',
|
||||
'-r', ROOT / 'tools' / 'requirements-installer.txt'], env=env)
|
||||
shutil.rmtree(py / 'Lib' / 'site-packages' / 'bin', ignore_errors=True) # numpy/cffi console scripts
|
||||
tk = tcltk_files()
|
||||
for dll in ('_tkinter.pyd', 'tcl90.dll', 'tcl9tk90.dll', 'zlib1.dll'):
|
||||
shutil.copy2(tk / 'DLLs' / dll, py / dll)
|
||||
copy_tree(tk / 'Lib' / 'tkinter', py / 'tkinter', lambda r: r.startswith('__pycache__'))
|
||||
copy_tree(tk / 'tcl', py / 'tcl', lambda r: r.endswith(('.lib', '.sh')) or r.startswith('nmake'))
|
||||
check = subprocess.run([str(py / 'python.exe'), '-I', '-c',
|
||||
'import tkinter, numpy, lz4.block, cryptography, cffi, pycparser, fontTools.ttLib; '
|
||||
'print("python ok: Tcl", tkinter.Tcl().eval("info patchlevel"), "numpy", numpy.__version__)'],
|
||||
env=clean_env(py), capture_output=True, text=True)
|
||||
if check.returncode:
|
||||
sys.exit('bundled python check failed:\n' + check.stderr)
|
||||
print(check.stdout.strip())
|
||||
|
||||
|
||||
# ---- llvm/ ---------------------------------------------------------------------------------------------------------
|
||||
|
||||
def build_llvm(out):
|
||||
src = unpack('llvm-mingw') / f'llvm-mingw-{LLVM_VERSION}-ucrt-x86_64'
|
||||
dst = out / 'llvm'
|
||||
|
||||
def skip(rel):
|
||||
top = rel.split('/')[0]
|
||||
if top in LLVM_SKIP_DIRS:
|
||||
return True
|
||||
if top == 'bin':
|
||||
return rel not in {f'bin/{n}' for n in LLVM_BIN}
|
||||
if top == 'lib': # lib/clang/<major>/include and the x86_64 Windows builtins; no sanitizers, no other targets
|
||||
if rel.startswith('lib/clang/') and '/include/' in rel:
|
||||
return False
|
||||
return not rel.endswith('/lib/windows/libclang_rt.builtins-x86_64.a')
|
||||
return Path(rel).suffix.lower() in LLVM_SKIP_SUFFIXES or rel.startswith('include/ddk/')
|
||||
|
||||
copy_tree(src, dst, skip)
|
||||
return dst
|
||||
|
||||
|
||||
# ---- bin/ and deps ---------------------------------------------------------------------------------------------------
|
||||
|
||||
def check_translator():
|
||||
"""Stop on a translator older than the sources. build/xlate is built by hand and only copied here (the Linux release
|
||||
builds its own), so without a rebuild a translator change reaches no Windows release, and nothing fails: the runtime
|
||||
sends job fields the old exe ignores, as happened to gather_bias."""
|
||||
idf = MAIN_ROOT / 'build' / 'xlate' / 'nx2navi_xlate.id'
|
||||
have = idf.read_text(encoding='ascii').strip() if idf.is_file() else 'none'
|
||||
want = build_xlate.identity()
|
||||
if have != want:
|
||||
sys.exit(f'build/xlate has translator {have[:16]}, the sources are {want[:16]}: run tools/build_xlate.py')
|
||||
|
||||
|
||||
def build_bin(out):
|
||||
b = out / 'bin'
|
||||
b.mkdir(parents=True, exist_ok=True)
|
||||
for f in ('nx2navi_xlate.exe', 'nx2navi_xlate.id'):
|
||||
shutil.copy2(MAIN_ROOT / 'build' / 'xlate' / f, b / f)
|
||||
shutil.copy2(MAIN_ROOT / 'toolchain' / 'hactool' / 'hactool.exe', b / 'hactool.exe')
|
||||
dll = unpack('sdl3-dll') / 'SDL3.dll'
|
||||
if sha256_file(dll) != SDL3_DLL_SHA256:
|
||||
sys.exit(f'{dll}: not the pinned SDL3.dll')
|
||||
shutil.copy2(dll, b / 'SDL3.dll')
|
||||
shutil.copy2(unpack('opus') / 'ucrt64' / 'bin' / 'libopus-0.dll', b / 'libopus-0.dll')
|
||||
shutil.copy2(unpack('ninja') / 'ninja.exe', b / 'ninja.exe')
|
||||
|
||||
|
||||
def build_deps(out):
|
||||
"""runtime/deps: the headers and import libraries build_game.py --deps expects (include/, lib/)."""
|
||||
d = out / 'runtime' / 'deps'
|
||||
(d / 'include').mkdir(parents=True, exist_ok=True)
|
||||
(d / 'lib').mkdir(parents=True, exist_ok=True)
|
||||
sdl = unpack('sdl3-devel') / f'SDL3-{SDL_VERSION}' / 'x86_64-w64-mingw32'
|
||||
copy_tree(sdl / 'include', d / 'include')
|
||||
shutil.copy2(sdl / 'lib' / 'libSDL3.dll.a', d / 'lib')
|
||||
for pkg, dirs, libs in (('opus', ['opus'], ['libopus.dll.a']), ('libvpx', ['vpx'], ['libvpx.a']),
|
||||
('vulkan-headers', ['vulkan', 'vk_video'], []), ('vulkan-loader', [], ['libvulkan-1.dll.a']),
|
||||
('vma', ['vma'], [])):
|
||||
root = unpack(pkg) / 'ucrt64'
|
||||
for inc in dirs:
|
||||
copy_tree(root / 'include' / inc, d / 'include' / inc)
|
||||
for lib in libs:
|
||||
shutil.copy2(root / 'lib' / lib, d / 'lib')
|
||||
zstd = unpack('zstd') / 'ucrt64'
|
||||
for name in ('zstd.h', 'zstd_errors.h'):
|
||||
shutil.copy2(zstd / 'include' / name, d / 'include')
|
||||
shutil.copy2(zstd / 'lib' / 'libzstd.a', d / 'lib')
|
||||
|
||||
|
||||
def collect_licenses(out):
|
||||
lic = out / 'licenses'
|
||||
lic.mkdir(parents=True, exist_ok=True)
|
||||
pairs = [
|
||||
(unpack('opus') / 'ucrt64' / 'share' / 'licenses' / 'opus' / 'COPYING', 'opus-COPYING.txt'),
|
||||
(unpack('vulkan-headers') / 'ucrt64' / 'share' / 'licenses' / 'vulkan-headers' / 'LICENSE', 'vulkan-headers-LICENSE.txt'),
|
||||
(unpack('vulkan-loader') / 'ucrt64' / 'share' / 'licenses' / 'vulkan-loader' / 'LICENSE', 'vulkan-loader-LICENSE.txt'),
|
||||
(unpack('vma') / 'ucrt64' / 'share' / 'licenses' / 'VulkanMemoryAllocator' / 'LICENSE', 'vulkan-memory-allocator-LICENSE.txt'),
|
||||
(unpack('sdl3-dll') / 'LICENSE.txt', 'SDL3-LICENSE.txt'),
|
||||
(unpack('zstd') / 'ucrt64' / 'share' / 'licenses' / 'zstd' / 'LICENSE', 'zstd-LICENSE.txt'),
|
||||
(ROOT / 'runtime' / 'shaders' / 'ffx' / 'LICENSE.txt', 'FidelityFX-FSR-LICENSE.txt'),
|
||||
(MAIN_ROOT / 'toolchain' / 'hactool' / 'LICENSE', 'hactool-LICENSE.txt'),
|
||||
(MAIN_ROOT / 'refs' / 'MacNativeGames' / 'LICENSE', 'MacNativeGames-LICENSE.txt'),
|
||||
]
|
||||
for src, name in pairs:
|
||||
shutil.copy2(src, lic / name)
|
||||
with tarfile.open(fetch('libvpx-src')) as t: # the MSYS2 package carries no licence text; the notice needs both
|
||||
for member, name in (('libvpx-1.17.0/LICENSE', 'libvpx-LICENSE.txt'), ('libvpx-1.17.0/PATENTS', 'libvpx-PATENTS.txt')):
|
||||
(lic / name).write_bytes(t.extractfile(member).read())
|
||||
for name in ('LICENSE.TXT',):
|
||||
shutil.copy2(out / 'llvm' / name, lic / 'llvm-mingw-LICENSE.txt')
|
||||
|
||||
|
||||
# ---- runtime/, recomp/, tools/, data/ ---------------------------------------------------------------------------------
|
||||
|
||||
def lifted_version():
|
||||
"""The game version whose generated C the runtime library and --prove build against: v121, else v143."""
|
||||
for version in ('v121', 'v143'):
|
||||
if (MAIN_ROOT / 'generated' / version / 'layout.json').is_file():
|
||||
return version
|
||||
sys.exit('neither generated/v121 nor generated/v143 exists: extract a game version and lift it first '
|
||||
'(tools/extract.py, recomp/gen.py; docs/building.md)')
|
||||
|
||||
|
||||
def profiles():
|
||||
"""{version tag: .profdata} of build/pgo (tools/pgo.py merge). A profile trained on another recomp/ than the one
|
||||
shipped still works but misses the functions whose generated C changed since: warn, so the next release retrains."""
|
||||
tree = subprocess.run(['git', '-C', str(ROOT), 'rev-parse', 'HEAD:recomp'], capture_output=True, text=True).stdout.strip()
|
||||
dirty = subprocess.run(['git', '-C', str(ROOT), 'status', '--porcelain', '--', 'recomp'], capture_output=True,
|
||||
text=True).stdout.strip()
|
||||
found = {}
|
||||
for p in sorted(PGO_DIR.glob('pgo-*.profdata')):
|
||||
rec = p.with_suffix('.json')
|
||||
record = json.loads(rec.read_text(encoding='utf-8')) if rec.is_file() else {}
|
||||
if record.get('recomp_tree') != tree or dirty:
|
||||
print(f'make_release: warning: {p.name} was trained on recomp tree {record.get("recomp_tree", "unknown")[:12]}, '
|
||||
f'this is {tree[:12]}{" with local changes" if dirty else ""}: functions whose generated C changed '
|
||||
'build without a profile; retrain (docs/building.md, "Profile-guided build")', file=sys.stderr)
|
||||
found[p.stem.removeprefix('pgo-')] = p
|
||||
if not found:
|
||||
print(f'make_release: no execution profile in {PGO_DIR}: installs build the game at -O1', file=sys.stderr)
|
||||
return found
|
||||
|
||||
|
||||
def runtime_lib(out, llvm, pgo):
|
||||
"""Build libnx2navirt.a (x86-64-v3) and libnx2navirt-v2.a (x86-64-v2) with the folder's llvm/, and copy them, the headers
|
||||
and the install-side sources into runtime/. pgo: profiles() of the release."""
|
||||
version = lifted_version()
|
||||
gen = MAIN_ROOT / 'generated' / version
|
||||
rt = out / 'runtime'
|
||||
# the runtime is the same for every update: any update's profile covers it (the newest update's, as players run it)
|
||||
newest = sorted(pgo.items())[-1:]
|
||||
for cpu, name in (('v3', 'libnx2navirt.a'), ('v2', 'libnx2navirt-v2.a')):
|
||||
build = MAIN_ROOT / 'build' / 'release-check' / f'rtlib-{cpu}'
|
||||
run([sys.executable, ROOT / 'tools' / 'build_game.py', '--make-runtime-lib', '--toolchain', llvm, '--cpu', cpu,
|
||||
'--version', version, '--deps', out / 'runtime' / 'deps', '--ninja', out / 'bin' / 'ninja.exe',
|
||||
'--gen', gen, '--out', build, *(['--pgo', newest[0][1]] if newest else [])],
|
||||
env=clean_env(llvm / 'bin', out / 'bin'))
|
||||
shutil.copy2(build / 'libnx2navirt.a', rt / name)
|
||||
for p in sorted((ROOT / 'runtime').rglob('*')):
|
||||
if p.suffix in RUNTIME_SUFFIXES and p.is_file():
|
||||
(rt / p.relative_to(ROOT / 'runtime')).parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(p, rt / p.relative_to(ROOT / 'runtime'))
|
||||
for name in sorted(build_game.INSTALL_SIDE): # compiled at install, so left out of libnx2navirt.a
|
||||
shutil.copy2(ROOT / 'runtime' / name, rt / name)
|
||||
|
||||
|
||||
def build_sources(out, pgo):
|
||||
copy_tree(ROOT / 'recomp', out / 'recomp', lambda r: not r.endswith(('.py', '.txt')) or '__pycache__' in r)
|
||||
for f in TOOLS_FILES:
|
||||
(out / 'tools').mkdir(exist_ok=True)
|
||||
shutil.copy2(ROOT / 'tools' / f, out / 'tools' / f)
|
||||
for d in TOOLS_DIRS:
|
||||
copy_tree(ROOT / 'tools' / d, out / 'tools' / d, lambda r: '__pycache__' in r or r.endswith('.md'))
|
||||
(out / 'data').mkdir(exist_ok=True)
|
||||
text = shader_list.history(MAIN_ROOT / 'refs' / 'MacNativeGames', MNG_COMMIT)
|
||||
(out / 'data' / 'shader_list.tsv').write_text(text, encoding='utf-8', newline='\n')
|
||||
for p in pgo.values():
|
||||
for f in (p, p.with_suffix('.json')):
|
||||
if f.is_file():
|
||||
shutil.copy2(f, out / 'data' / f.name)
|
||||
copy_tree(ROOT / 'installer', out / 'installer', lambda r: '__pycache__' in r or r.startswith('launcher/'))
|
||||
shutil.copy2(ROOT / 'nx2navi.ini.example', out / 'nx2navi.ini.example')
|
||||
shutil.copy2(ROOT / 'THIRD_PARTY_NOTICES.md', out / 'THIRD_PARTY_NOTICES.md')
|
||||
(out / 'VERSION').write_text(version() + '\n', encoding='utf-8')
|
||||
|
||||
|
||||
def version():
|
||||
"""The port version from VERSION with the commit as build metadata: 0.1.0+973c626. The commit keeps two releases of
|
||||
one version apart, so the installer reruns the steps an earlier release made."""
|
||||
r = subprocess.run(['git', '-C', ROOT, 'rev-parse', '--short', 'HEAD'], capture_output=True, text=True)
|
||||
if r.returncode or not r.stdout.strip():
|
||||
sys.exit('no git version: not a git checkout')
|
||||
return (ROOT / 'VERSION').read_text(encoding='utf-8').strip() + '+' + r.stdout.strip()
|
||||
|
||||
|
||||
# ---- NX2navi-Setup.exe --------------------------------------------------------------------------------------------------
|
||||
|
||||
def c_wide(text):
|
||||
"""A C wide string literal of text: ASCII as is, everything else as a universal character name."""
|
||||
bs = chr(92)
|
||||
out = []
|
||||
for ch in text:
|
||||
if ch in (bs, '"'):
|
||||
out.append(bs + ch)
|
||||
elif 32 <= ord(ch) < 127:
|
||||
out.append(ch)
|
||||
elif ord(ch) < 0x10000:
|
||||
out.append(f'{bs}u{ord(ch):04x}')
|
||||
else:
|
||||
sys.exit(f'launcher text: {ch!r} is outside the BMP')
|
||||
return 'L"' + ''.join(out) + '"'
|
||||
|
||||
|
||||
def launcher_header(path):
|
||||
"""launcher_text.h: the launcher's two messages in English, Russian, Brazilian Portuguese and Simplified Chinese,
|
||||
taken from the installer's catalog (LAUNCHER_TITLE_PT_BR for pt-BR.json, LAUNCHER_TITLE_ZH_HANS for zh-Hans.json)."""
|
||||
lines = ['/* generated by tools/make_release.py from installer/text/{en,ru,pt-BR,zh-Hans}.json */']
|
||||
for lang in ('en', 'ru', 'pt-BR', 'zh-Hans'):
|
||||
texts = json.loads((ROOT / 'installer' / 'text' / f'{lang}.json').read_text(encoding='utf-8'))
|
||||
for key in ('title', 'python_missing'):
|
||||
macro = f'LAUNCHER_{key.upper()}_{lang.upper().replace("-", "_")}'
|
||||
lines.append(f'#define {macro} {c_wide(texts["launcher." + key])}')
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text('\n'.join(lines) + '\n', encoding='ascii')
|
||||
|
||||
|
||||
def build_launcher(out, llvm):
|
||||
work = MAIN_ROOT / 'build' / 'release-check' / 'launcher'
|
||||
launcher_header(work / 'launcher_text.h')
|
||||
env = clean_env(llvm / 'bin')
|
||||
res = work / 'launcher_rc.o'
|
||||
run([llvm / 'bin' / 'llvm-windres.exe', '-I', ROOT / 'runtime', '-O', 'coff',
|
||||
ROOT / 'installer' / 'launcher' / 'launcher.rc', '-o', res], env=env)
|
||||
run([llvm / 'bin' / 'clang.exe', '-O2', '-mwindows', '-Wall', '-Wl,-s', '-I', work,
|
||||
ROOT / 'installer' / 'launcher' / 'launcher.c', res, '-o', out / 'NX2navi-Setup.exe'], env=env)
|
||||
|
||||
|
||||
# ---- checks ----------------------------------------------------------------------------------------------------------
|
||||
|
||||
def check_folder(out):
|
||||
"""check_content over every file of the release folder; a hit stops the build."""
|
||||
bad = []
|
||||
for p in sorted(out.rglob('*')):
|
||||
if p.is_file():
|
||||
bad += [f'{p.relative_to(out)}: {problem}' for problem in check_content.problems_for(p, allow_large=True)]
|
||||
if bad:
|
||||
sys.exit('game-derived content in the release folder:\n' + '\n'.join(bad))
|
||||
print(f'check_content: {sum(1 for p in out.rglob("*") if p.is_file())} files, no hit')
|
||||
|
||||
|
||||
def check_bundle(out):
|
||||
"""The folder's own python runs the installer without game files and the window's selftest."""
|
||||
chk = MAIN_ROOT / 'build' / 'release-check' / 'bundle'
|
||||
shutil.rmtree(chk, ignore_errors=True)
|
||||
env = clean_env(out / 'python')
|
||||
py = out / 'python' / 'python.exe'
|
||||
r = subprocess.run([py, '-m', 'installer', '--lang', 'en-US', '--dest', chk / 'install', '--temp', chk / 'temp'],
|
||||
cwd=out, env=env, capture_output=True, text=True, timeout=600)
|
||||
events = [json.loads(line) for line in r.stdout.splitlines() if line.startswith('{')]
|
||||
errors = [e['id'] for e in events if e['type'] == 'error']
|
||||
started = any(e['type'] == 'start' and e.get('release') == (out / 'VERSION').read_text().strip() for e in events)
|
||||
steps = [e['phase'] for e in events if e['type'] == 'progress']
|
||||
if errors != ['check.inputs_needed'] or not started or 'inputs' not in steps:
|
||||
sys.exit(f'bundle check: exit {r.returncode}, errors {errors}, release read {started}\n{r.stdout[-2000:]}\n{r.stderr[-2000:]}')
|
||||
print(f'bundle check: the installer ran the host check from the folder and stopped at the game files (exit {r.returncode})')
|
||||
r = subprocess.run([py, '-m', 'installer.gui', '--selftest'], cwd=out, env=env, capture_output=True, text=True, timeout=300)
|
||||
if r.returncode:
|
||||
sys.exit(f'bundle check: window selftest exit {r.returncode}\n{r.stdout[-2000:]}\n{r.stderr[-2000:]}')
|
||||
print('bundle check: the window built every screen in both languages and closed')
|
||||
|
||||
|
||||
def sizes(out):
|
||||
rows = [(p.name if p.is_dir() else p.name, dir_size(p) if p.is_dir() else p.stat().st_size) for p in sorted(out.iterdir())]
|
||||
for name, n in rows:
|
||||
print(f' {name:26} {mib(n):>12}')
|
||||
print(f' {"total":26} {mib(sum(n for _, n in rows)):>12}')
|
||||
|
||||
|
||||
def write_zip(out, path):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with zipfile.ZipFile(path, 'w', zipfile.ZIP_DEFLATED, compresslevel=6) as z:
|
||||
for p in sorted(out.rglob('*')):
|
||||
if p.is_file():
|
||||
info = zipfile.ZipInfo((Path(out.name) / p.relative_to(out)).as_posix(), (1980, 1, 1, 0, 0, 0))
|
||||
info.compress_type = zipfile.ZIP_DEFLATED
|
||||
info.external_attr = 0o644 << 16
|
||||
z.writestr(info, p.read_bytes(), compresslevel=6)
|
||||
print(f'{path}: {mib(path.stat().st_size)}')
|
||||
|
||||
|
||||
def prove(out, cpu='v3'):
|
||||
"""Compile the generated C and link nx2navi.exe with the folder's tools only, then run the probes next to it."""
|
||||
out = out.resolve()
|
||||
version = lifted_version()
|
||||
gen = MAIN_ROOT / 'generated' / version
|
||||
ext = MAIN_ROOT / 'build' / 'extracted' / version
|
||||
chk = MAIN_ROOT / 'build' / 'release-check'
|
||||
env = clean_env(out / 'python', out / 'llvm' / 'bin', out / 'bin')
|
||||
pgo = out / 'data' / f'pgo-{version}.profdata'
|
||||
t0 = time.time()
|
||||
run([out / 'python' / 'python.exe', out / 'tools' / 'build_game.py', '--toolchain', out / 'llvm', '--cpu', cpu,
|
||||
'--deps', out / 'runtime' / 'deps', '--runtime', out / 'runtime',
|
||||
'--runtime-lib', out / 'runtime' / ('libnx2navirt-v2.a' if cpu == 'v2' else 'libnx2navirt.a'),
|
||||
'--ninja', out / 'bin' / 'ninja.exe', '--version', version, '--gen', gen, '--out', chk, '--exefs', ext / 'exefs',
|
||||
'--romfs', ext / 'romfs', '--profile', chk / 'profile', *(['--pgo', pgo] if pgo.is_file() else [])],
|
||||
env=env)
|
||||
print(f'prove: build {time.time() - t0:.0f} s')
|
||||
for dll in ('SDL3.dll', 'libopus-0.dll'):
|
||||
shutil.copy2(out / 'bin' / dll, chk / dll)
|
||||
failed = []
|
||||
for probe in PROBES:
|
||||
r = subprocess.run([str(chk / 'nx2navi.exe'), probe], cwd=chk, env=clean_env(chk), capture_output=True, text=True,
|
||||
timeout=900)
|
||||
tail = (r.stdout.strip().splitlines() or [''])[-1]
|
||||
print(f'{probe}: exit {r.returncode}: {tail}')
|
||||
if r.returncode:
|
||||
failed.append(probe)
|
||||
if failed:
|
||||
sys.exit(f'probes failed: {failed}')
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument('--out', type=Path, required=True, help='release folder (replaced)')
|
||||
ap.add_argument('--zip', type=Path, help='also write the folder as this zip')
|
||||
ap.add_argument('--prove', action='store_true', help='only run the compile-and-probe check on an existing folder')
|
||||
ap.add_argument('--cpu', choices=('v2', 'v3'), default='v3',
|
||||
help='--prove: build and run the check for this CPU level (default v3)')
|
||||
args = ap.parse_args()
|
||||
out = args.out.resolve()
|
||||
if args.prove:
|
||||
prove(out, args.cpu)
|
||||
return
|
||||
check_translator()
|
||||
if out.exists():
|
||||
shutil.rmtree(out)
|
||||
out.mkdir(parents=True)
|
||||
build_python(out)
|
||||
llvm = build_llvm(out)
|
||||
build_bin(out)
|
||||
build_deps(out)
|
||||
pgo = profiles()
|
||||
runtime_lib(out, llvm, pgo)
|
||||
build_sources(out, pgo)
|
||||
build_launcher(out, llvm)
|
||||
collect_licenses(out)
|
||||
check_folder(out)
|
||||
check_bundle(out)
|
||||
sizes(out)
|
||||
if args.zip:
|
||||
write_zip(out, args.zip.resolve())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user