NX2navi/tools/make_launcher.py

"""Build a release of the NX2navi launcher (docs/launcher.md, "Releases").

    python tools/make_launcher.py --xlate build/xlate [--ffx amd_fidelityfx_vk.dll] [--out build/launcher-release]
                                  [--feed owner/name]
                                  [--notes notes.md] [--publish]

Needs Python 3.14 (tarfile and zstd for the MSYS2 packages), uv on PATH, network access on the first run, and
either Windows (msiexec, and the C compiler of MSYS2 it downloads itself) or Linux with clang, lld and llvm-windres
(it cross-compiles against the same MSYS2 packages) plus msitools. Downloads are cached in toolchain/launcher-dl/ and
checked against the sha256 in PINS.

The release folder (--out) holds:

    NX2navi/                      the launcher folder exactly as NX2navi-Setup.exe installs it:
      NX2navi.exe                 launcher/native/nx2navi.c with the Triforce icon (launcher/ui/art.py)
      python/                     python.org's embeddable Python 3.14.9, tkinter and Tcl/Tk 9.0 from its tcltk.msi,
                                  the packages of tools/requirements-launcher.txt, runtime-id.txt
      app/current.txt, app/<version>/   launcher/ (without native/), runtime/, recomp/, the tools make_game.py uses,
                                  bin/nx2navi_xlate.exe and .id (and amd_fidelityfx_vk.dll with --ffx), toolchain.lock.json (the MSYS2 packages of today,
                                  pinned), feed.json, VERSION, the licence files
    NX2navi-Setup-<version>.exe   launcher/native/setup.c with the folder above appended as a zip: the one file
                                  players download
    NX2navi-app-<version>.zip     app/<version> alone: what the launcher downloads to update itself
    NX2navi-runtime-<id>.zip      python/ alone: downloaded only when a release changes the Python
    NX2navi-update.json           the manifest the launcher reads (launcher/feed.py)

--publish uploads the four files as GitHub release v<version> of the --feed repository with the gh CLI (a draft
first is wiser: add --draft). check_content runs over everything before it is packed: the release holds no game data.
"""
import argparse
import hashlib
import json
import os
import shutil
import subprocess
import sys
import time
import zipfile
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
sys.path.insert(0, str(ROOT / 'tools'))
import check_content  # noqa: E402
from launcher import toolchain  # noqa: E402
from launcher.paths import Layout  # noqa: E402

MAIN_ROOT = (ROOT / 'build').resolve().parent   # build/ and toolchain/ are junctions in a worktree
DL = MAIN_ROOT / 'toolchain' / 'launcher-dl'
WORK = MAIN_ROOT / 'toolchain' / 'launcher-work'
PY_VERSION = '3.14.9'
PINS = {
    'python-embed': (f'https://www.python.org/ftp/python/{PY_VERSION}/python-{PY_VERSION}-embed-amd64.zip',
                     '3c01d7cfc666f746a6f064f60fb2156ab0ec464d3d738ab796371a46a9bb24a2'),
    # the installer's own Tcl/Tk and tkinter component, published beside it
    'tcltk': (f'https://www.python.org/ftp/python/{PY_VERSION}/amd64/tcltk.msi',
              'ac87609db3f715404dd41a989c2f1706ea1512f3d86217f938d11c63af8eb505'),
}
TOOLS = ('make_game.py', 'nup.py', 'romfs_index.py', 'check_content.py', 'hostmem.py', 'prepare_shaders.py',
         'shader_census.py', 'shader_list.py', 'zstd_compat.py')
TOP = ('LICENSE', 'LICENSE-ADDITIONS', 'THIRD_PARTY_NOTICES.md', 'nx2navi.ini.example')
SKIP_DIRS = {'__pycache__', '.pytest_cache', 'native'}
SKIP_SUFFIXES = {'.o', '.obj', '.pyc', '.exe', '.dll', '.a', '.lib'}
TCLTK_FILES = ('_tkinter.pyd', 'tcl90.dll', 'tcl9tk90.dll', 'zlib1.dll', 'libtommath.dll')
DEFAULT_FEED = 'tangerineturtle1/NX2navi'


def say(msg):
    print(msg, flush=True)


def sha256_file(path):
    h = hashlib.sha256()
    with open(path, 'rb') as f:
        while b := f.read(1 << 20):
            h.update(b)
    return h.hexdigest()


def fetch(name):
    url, want = PINS[name]
    dest = DL / url.rsplit('/', 1)[1]
    if dest.is_file() and sha256_file(dest) == want:
        return dest
    from launcher import net
    say(f'download {url}')
    net.download(url, dest, sha256=want)
    return dest


def run(cmd, **kw):
    say('  ' + ' '.join(str(c) for c in cmd))
    subprocess.run([str(c) for c in cmd], check=True, **kw)


def copy_tree(src, dst, skip_dirs=SKIP_DIRS):
    for p in sorted(src.rglob('*')):
        rel = p.relative_to(src)
        if any(part in skip_dirs for part in rel.parts) or p.is_dir() or p.suffix.lower() in SKIP_SUFFIXES:
            continue
        (dst / rel).parent.mkdir(parents=True, exist_ok=True)
        shutil.copy2(p, dst / rel)


def version():
    v = (ROOT / 'VERSION').read_text(encoding='utf-8').strip()
    r = subprocess.run(['git', '-C', str(ROOT), 'rev-parse', '--short', 'HEAD'], capture_output=True, text=True)
    return v, f'{v}+{r.stdout.strip()}' if r.returncode == 0 and r.stdout.strip() else v


# ---------------------------------------------------------------- python/

def tcltk_files():
    dest = WORK / 'tcltk'
    if (dest / 'DLLs' / '_tkinter.pyd').is_file():
        return dest
    msi = fetch('tcltk')
    shutil.rmtree(dest, ignore_errors=True)
    dest.mkdir(parents=True)
    if os.name == 'nt':   # an administrative image: copies the files, installs nothing
        run(['msiexec', '/a', msi, '/qn', f'TARGETDIR={dest}'])
    else:
        run(['msiextract', '-C', dest, msi], stdout=subprocess.DEVNULL)
    if not (dest / 'DLLs' / '_tkinter.pyd').is_file():
        sys.exit(f'{dest}: no _tkinter.pyd in {msi.name}')
    return dest


def runtime_id():
    req = (ROOT / 'tools' / 'requirements-launcher.txt').read_bytes()
    return f'py{PY_VERSION}-{hashlib.sha256(req + PINS["tcltk"][1].encode()).hexdigest()[:8]}'


def build_python(py):
    with zipfile.ZipFile(fetch('python-embed')) as z:
        z.extractall(py)
    # The embeddable Python's sys.path is exactly this list; ../pydeps holds zbic (launcher/toolchain.py).
    (py / 'python314._pth').write_text('python314.zip\n.\nLib/site-packages\n../pydeps\n', encoding='utf-8')
    env = dict(os.environ, UV_CACHE_DIR=str(MAIN_ROOT / 'toolchain' / 'uv-cache'))
    run(['uv', 'pip', 'install', '--target', py / 'Lib' / 'site-packages', '--python', sys.executable,
         '--python-platform', 'windows', '--python-version', '3.14', '--only-binary', ':all:', '-q',
         '-r', ROOT / 'tools' / 'requirements-launcher.txt'], env=env)
    shutil.rmtree(py / 'Lib' / 'site-packages' / 'bin', ignore_errors=True)
    tk = tcltk_files()
    for dll in TCLTK_FILES:
        if (tk / 'DLLs' / dll).is_file():
            shutil.copy2(tk / 'DLLs' / dll, py / dll)
    copy_tree(tk / 'Lib' / 'tkinter', py / 'tkinter', skip_dirs={'__pycache__', 'test'})
    copy_tree(tk / 'tcl', py / 'tcl', skip_dirs={'__pycache__'})
    (py / 'runtime-id.txt').write_text(runtime_id() + '\n', encoding='utf-8')


# ---------------------------------------------------------------- app/<version>/

def build_app(app, ver, xlate, feed_repo, lock, ffx=None):
    copy_tree(ROOT / 'launcher', app / 'launcher')
    for d in ('runtime', 'recomp'):
        copy_tree(ROOT / d, app / d)
    (app / 'tools').mkdir(parents=True)
    for t in TOOLS:
        shutil.copy2(ROOT / 'tools' / t, app / 'tools' / t)
    for t in TOP:
        if (ROOT / t).is_file():
            shutil.copy2(ROOT / t, app / t)
    (app / 'VERSION').write_text(ver + '\n', encoding='utf-8')
    (app / 'bin').mkdir()
    for n in ('nx2navi_xlate.exe', 'nx2navi_xlate.id'):
        if not (xlate / n).is_file():
            sys.exit(f'{xlate / n} is missing: build the shader translator first (tools/build_xlate.py)')
        shutil.copy2(xlate / n, app / 'bin' / n)
    if ffx:   # FSR 3.1 loads it from beside the game (runtime/vk_ffx.cpp); make_game.py copies it there
        shutil.copy2(ffx, app / 'bin' / 'amd_fidelityfx_vk.dll')
    (app / 'toolchain.lock.json').write_text(json.dumps(lock, indent=1), encoding='utf-8')
    (app / 'feed.json').write_text(json.dumps({'type': 'github', 'repo': feed_repo}, indent=1), encoding='utf-8')


# ---------------------------------------------------------------- the native programs

def native_compiler(lock):
    """(cc argv prefix, windres argv prefix) for x86_64 Windows: MSYS2's own clang on Windows, else the host's clang
    against the same MSYS2 packages as a sysroot."""
    lay = Layout(root=MAIN_ROOT / 'toolchain' / 'launcher-msys')
    if not toolchain.is_current(lay, lock):
        say('MSYS2 packages for the native programs')
        toolchain.install(lay, lock, log=say)
    sysroot = lay.msys
    if os.name == 'nt':
        return [str(sysroot / 'bin' / 'clang.exe')], [str(sysroot / 'bin' / 'windres.exe')]
    for tool in ('clang', 'ld.lld'):
        if not shutil.which(tool):
            sys.exit(f'{tool} is needed to cross-compile the launcher programs')
    windres = shutil.which('llvm-windres') or shutil.which('llvm-windres-18') or '/usr/lib/llvm-18/bin/llvm-windres'
    if not Path(windres).exists():
        sys.exit('llvm-windres is needed to compile the launcher programs\' resources')
    return (['clang', '--target=x86_64-w64-mingw32', f'--sysroot={sysroot}', '-fuse-ld=lld'],
            [windres, '--target=x86_64-w64-mingw32'])


def build_native(work, cc, windres):
    from launcher.ui import art
    work.mkdir(parents=True, exist_ok=True)
    (work / 'nx2navi.ico').write_bytes(art.triforce_ico())
    shutil.copy2(ROOT / 'launcher' / 'native' / 'app.manifest', work / 'app.manifest')
    res = work / 'res.o'
    run([*windres, '-I', work, ROOT / 'launcher' / 'native' / 'res.rc', '-O', 'coff', '-o', res])
    flags = ['-municode', '-mwindows', '-O2', '-Wall', '-static', '-s']
    stub = work / 'NX2navi.exe'
    setup = work / 'setup-stub.exe'
    run([*cc, *flags, ROOT / 'launcher' / 'native' / 'nx2navi.c', res, '-o', stub, '-lshell32', '-lole32', '-luuid'])
    run([*cc, *flags, ROOT / 'launcher' / 'native' / 'setup.c', res, '-o', setup, '-l:libz.a', '-lshell32',
         '-lole32', '-luuid', '-ladvapi32'])
    return stub, setup


# ---------------------------------------------------------------- packing

def zip_dir(src, dest, prefix=''):
    with zipfile.ZipFile(dest, 'w', zipfile.ZIP_DEFLATED, compresslevel=9) as z:
        for p in sorted(src.rglob('*')):
            if p.is_file():
                z.write(p, (Path(prefix) / p.relative_to(src)).as_posix())
    return dest


def make_setup(stub, payload, dest, ver):
    """setup-stub.exe + the payload zip + the 48-byte trailer launcher/native/setup.c reads."""
    data = stub.read_bytes()
    off = len(data)
    z = payload.read_bytes()
    trailer = b'NX2NSET1' + off.to_bytes(8, 'little') + len(z).to_bytes(8, 'little') + \
        ver.encode('utf-8')[:24].ljust(24, b'\0')
    dest.write_bytes(data + z + trailer)
    return dest


def check_folder(folder):
    bad = [f'{p.relative_to(folder)}: {m}' for p in sorted(folder.rglob('*')) if p.is_file()
           for m in check_content.problems_for(p, allow_large=True)]
    if bad:
        sys.exit('check_content refuses these release files:\n  ' + '\n  '.join(bad[:20]))


def entry(path):
    return {'name': path.name, 'size': path.stat().st_size, 'sha256': sha256_file(path)}


def main():
    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    ap.add_argument('--xlate', type=Path, required=True, help='folder with nx2navi_xlate.exe and nx2navi_xlate.id')
    ap.add_argument('--ffx', type=Path, help="AMD's signed amd_fidelityfx_vk.dll (FidelityFX SDK 1.1.4, MIT) for FSR 3.1")
    ap.add_argument('--out', type=Path, default=MAIN_ROOT / 'build' / 'launcher-release')
    ap.add_argument('--feed', default=DEFAULT_FEED, help='the GitHub repository the launcher looks for releases in')
    ap.add_argument('--notes', type=Path, help='release notes (markdown) for the manifest and the GitHub release')
    ap.add_argument('--lock', type=Path, help='a toolchain.lock.json to reuse instead of resolving today\'s packages')
    ap.add_argument('--version', help='the release version instead of VERSION\'s (test releases)')
    ap.add_argument('--publish', action='store_true', help='upload as GitHub release v<version> with gh')
    ap.add_argument('--draft', action='store_true', help='with --publish: as a draft')
    a = ap.parse_args()
    if sys.version_info < (3, 14):
        sys.exit('Python 3.14 or later is needed (compression.zstd for the MSYS2 packages)')
    ver, build = version()
    if a.version:
        ver, build = a.version, a.version + build[len(ver):]
    out = a.out
    shutil.rmtree(out, ignore_errors=True)
    folder = out / 'NX2navi'
    t0 = time.time()
    lock = json.loads(a.lock.read_text(encoding='utf-8')) if a.lock else toolchain.make_lock()
    say(f'release {build}: toolchain lock with {len(lock["packages"])} packages')
    build_app(folder / 'app' / ver, ver, a.xlate, a.feed, lock, a.ffx)
    (folder / 'app' / 'current.txt').write_text(ver + '\n', encoding='utf-8')
    build_python(folder / 'python')
    cc, windres = native_compiler(lock)
    stub, setup_stub = build_native(out / 'native', cc, windres)
    shutil.copy2(stub, folder / 'NX2navi.exe')
    check_folder(folder)
    app_zip = zip_dir(folder / 'app' / ver, out / f'NX2navi-app-{ver}.zip')
    rt_zip = zip_dir(folder / 'python', out / f'NX2navi-runtime-{runtime_id()}.zip')
    payload = zip_dir(folder, out / 'native' / 'payload.zip')
    setup = make_setup(setup_stub, payload, out / f'NX2navi-Setup-{ver}.exe', ver)
    notes = a.notes.read_text(encoding='utf-8') if a.notes else ''
    manifest = {'format': 1, 'version': ver, 'build': build, 'notes': notes,
                'app': entry(app_zip), 'runtime': {'id': runtime_id(), **entry(rt_zip)},
                'toolchain': toolchain.lock_digest(lock), 'setup': entry(setup),
                'made': time.strftime('%Y-%m-%d %H:%M')}
    man = out / 'NX2navi-update.json'
    man.write_text(json.dumps(manifest, indent=1), encoding='utf-8')
    for p in (setup, app_zip, rt_zip, man):
        say(f'{p.name}: {p.stat().st_size / 1e6:.1f} MB')
    say(f'release folder {out} in {time.time() - t0:.0f} s')
    if a.publish:
        cmd = ['gh', 'release', 'create', f'v{ver}', setup, app_zip, rt_zip, man, '--repo', a.feed,
               '--title', f'NX2navi {ver}']
        cmd += ['--notes-file', a.notes] if a.notes else ['--notes', f'NX2navi {ver}']
        if a.draft:
            cmd.append('--draft')
        run(cmd)


if __name__ == '__main__':
    main()