From c403547c8b12c78c68c3b245f2008aa9c852e7cf Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 7 Nov 2026 19:14:39 +0000 Subject: [PATCH] Add the NX2navi launcher: a one-click setup that builds the game from the player's dump, starts it and keeps it up to date. NX2navi-Setup.exe installs the launcher into %LOCALAPPDATA%\Programs\NX2navi without administrator rights, with shortcuts and an Apps entry. The launcher finds the player's .nup, downloads the build tools itself (the MSYS2 UCRT64 packages toolchain.lock.json pins, checked by sha256 and unpacked without pacman, and zbic from PyPI) and runs tools/make_game.py, whose steps it shows with their progress. Its window follows the game's options menu: colours sampled from it, the menu's filigree, the white glowing cursor, the Triforce in the title band and as the programs' icon, and all text in the game's Cetra Display, decoded from the player's own dump. Mouse, keyboard and an XInput pad drive it. Updates come from GitHub releases (NX2navi-update.json): the new version is downloaded, checked and staged beside the running one, the stub restarts into it and falls back to the previous version when it fails to start, and the game is rebuilt without the dump. make_game.py now keys its exefs and romfs steps on the game files alone, has --rebuild and --events, compiles only the files whose source, headers, flags or compiler changed, and replaces ocarina.exe only once the new one links. tools/make_launcher.py builds the release on Windows or Linux. Tested: tests/launcher_test.py (19 tests), the real toolchain download and install, and under Wine 11 the setup, the stub, the window on the bundled Windows Python, an update from a local feed by clicking Updates, the rollback of a version that fails to start, and the downloaded clang compiling and linking a program against the game's libraries. Co-Authored-By: Claude Opus 5.5 Merged onto the current port: make_game.py keeps the launcher's step keys, cached link and in-place swap, and its own runtime manifest (windres), AMD's FidelityFX library and the shader preparation, which becomes a step of its own ("shaders") that the launcher shows in its install, rebuild and update jobs. make_launcher.py ships the shader tools and, with --ffx, AMD's library. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- README.md | 10 +- THIRD_PARTY_NOTICES.md | 20 + docs/README-BuildKit.md | 3 + docs/building.md | 4 +- docs/launcher.md | 160 ++++ launcher/__init__.py | 8 + launcher/__main__.py | 6 + launcher/build.py | 135 ++++ launcher/cli.py | 131 ++++ launcher/context.py | 48 ++ launcher/feed.py | 144 ++++ launcher/gamefiles.py | 322 ++++++++ launcher/jobs.py | 368 +++++++++ launcher/native/app.manifest | 24 + launcher/native/common.h | 166 ++++ launcher/native/nx2navi.c | 117 +++ launcher/native/res.rc | 4 + launcher/native/setup.c | 382 +++++++++ launcher/net.py | 128 +++ launcher/paths.py | 115 +++ launcher/settings.py | 43 + launcher/start.pyw | 45 ++ launcher/toolchain.py | 288 +++++++ launcher/ui/__init__.py | 1 + launcher/ui/app.py | 1297 +++++++++++++++++++++++++++++++ launcher/ui/art.py | 538 +++++++++++++ launcher/ui/selftest.py | 173 +++++ launcher/updater.py | 158 ++++ launcher/versions.py | 25 + launcher/winsys.py | 192 +++++ tests/launcher_test.py | 678 ++++++++++++++++ tools/make_game.py | 463 +++++++++-- tools/make_launcher.py | 301 +++++++ tools/requirements-launcher.txt | 6 + 35 files changed, 6422 insertions(+), 83 deletions(-) create mode 100644 docs/launcher.md create mode 100644 launcher/__init__.py create mode 100644 launcher/__main__.py create mode 100644 launcher/build.py create mode 100644 launcher/cli.py create mode 100644 launcher/context.py create mode 100644 launcher/feed.py create mode 100644 launcher/gamefiles.py create mode 100644 launcher/jobs.py create mode 100644 launcher/native/app.manifest create mode 100644 launcher/native/common.h create mode 100644 launcher/native/nx2navi.c create mode 100644 launcher/native/res.rc create mode 100644 launcher/native/setup.c create mode 100644 launcher/net.py create mode 100644 launcher/paths.py create mode 100644 launcher/settings.py create mode 100644 launcher/start.pyw create mode 100644 launcher/toolchain.py create mode 100644 launcher/ui/__init__.py create mode 100644 launcher/ui/app.py create mode 100644 launcher/ui/art.py create mode 100644 launcher/ui/selftest.py create mode 100644 launcher/updater.py create mode 100644 launcher/versions.py create mode 100644 launcher/winsys.py create mode 100644 tests/launcher_test.py create mode 100644 tools/make_launcher.py create mode 100644 tools/requirements-launcher.txt diff --git a/AGENTS.md b/AGENTS.md index f2146c34..fd951bf3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,6 +1,6 @@ # NX2navi -Static recompilation of *Ocarina of Time* (Switch 2, ARM64, game version 1.1.1) into a native Windows x86-64 exe with a Vulkan renderer. Design: [docs/architecture.md](docs/architecture.md). Toolchain, build commands and tests: [docs/building.md](docs/building.md). +Static recompilation of *Ocarina of Time* (Switch 2, ARM64, game version 1.1.1) into a native Windows x86-64 exe with a Vulkan renderer. Design: [docs/architecture.md](docs/architecture.md). Toolchain, build commands and tests: [docs/building.md](docs/building.md). The players' launcher, setup and updates: [docs/launcher.md](docs/launcher.md). ## Game content boundary diff --git a/README.md b/README.md index d23ae9ae..ec72632f 100644 --- a/README.md +++ b/README.md @@ -45,10 +45,12 @@ Report problems as issues, following [CONTRIBUTING.md](CONTRIBUTING.md). ## Installing -The build kit builds the game on your PC from your own dump (a `.nup` file, or ExeFS and RomFS folders) in about 10 to -15 minutes: [docs/README-BuildKit.md](docs/README-BuildKit.md). [docs/building.md](docs/building.md) describes building -from source. Settings are in the in-game menu and in `nx2navi.ini` ([nx2navi.ini.example](nx2navi.ini.example) lists every -key). Controls: [docs/controls.md](docs/controls.md). +Download `NX2navi-Setup-.exe` from the releases and start it next to your `.nup` dump. The launcher it +installs finds the dump, downloads the build tools, builds the game on your PC (10 to 25 minutes, once) and starts +it; afterwards it keeps the port up to date and rebuilds only what changed ([docs/launcher.md](docs/launcher.md)). +[docs/README-BuildKit.md](docs/README-BuildKit.md) describes the same build by hand, [docs/building.md](docs/building.md) +building from source. Settings are in the in-game menu and in `nx2navi.ini` ([nx2navi.ini.example](nx2navi.ini.example) +lists every key). Controls: [docs/controls.md](docs/controls.md). Requirements: Windows 10 or later, an x86-64 CPU with SSE4.2 (AVX2/FMA machines get a faster build), a GPU with Vulkan 1.3, `VK_EXT_external_memory_host` and `VK_EXT_mesh_shader`. diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 8581497a..1b8a977f 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -107,3 +107,23 @@ folder's `licenses/` and in each component's own folder. | MacNativeGames shader variant list | as in the installer release row above | MIT | `data/shader_list.tsv` | | Weeping of the Empire menu cursor: eui's focus box read through ScreenMgr, BoxCursorMgr and the screens' node lists, AnimButton's hit test, the stale-hit-pane fallback to the parts panes, the three writes of moveBoxCursor, the structure offsets (1.0.0; the ScreenMgr instance placed in 1.4.3 by us) | https://github.com/daniel-madrid-07/weeping-of-the-empire @ `df0586cdc5702ca19bbebfa59697e0b6980f6237` (`engine/runtime/nx2navi_menu_cursor.cpp`), Copyright (c) 2026 daniel-madrid-07 | MIT | `runtime/menu_cursor.c` | | AMD FidelityFX API headers (FSR 3.1 upscaling and frame generation) and its signed Vulkan library | https://github.com/GPUOpen-LibrariesAndSDKs/FidelityFX-SDK @ `v1.1.4` (`ffx-api/include/ffx_api/`, `PrebuiltSignedDLL/amd_fidelityfx_vk.dll`), Copyright (C) 2024 Advanced Micro Devices, Inc. | MIT | `runtime/third_party/ffx_api/` (unmodified, with `LICENSE.txt`), used by `runtime/vk_ffx.cpp`; the library ships in the build kit's `bin/` and beside the built game (never in this repository) | + +## Distributed in the launcher release + +`tools/make_launcher.py` assembles these into `NX2navi-Setup-.exe` and the update archives of +`docs/launcher.md`. Versions and sha256 of the downloads are pinned in that script and in +`tools/requirements-launcher.txt`. + +| Component | Source | License | Where in the release | +|---|---|---|---| +| CPython 3.14.9 embeddable package; includes `vcruntime140*.dll` (Microsoft, redistributable with Python), OpenSSL, libffi, SQLite, zlib, libmpdec | https://www.python.org/downloads/release/python-3149/ | PSF-2.0 | `python/` | +| `tkinter` and Tcl/Tk 9.0 (`_tkinter.pyd`, `tcl90.dll`, `tcl9tk90.dll`, `zlib1.dll`, `libtommath.dll`, `tcl/`), from python.org's 3.14.9 `amd64/tcltk.msi`, unpacked, never run | https://www.python.org/ftp/python/3.14.9/amd64/ | PSF-2.0; Tcl/Tk licence (BSD-style); libtommath public domain or BSD; zlib licence | `python/` | +| numpy 2.5.4 (with its bundled OpenBLAS), lz4 4.4.5, zstandard 0.25.0 | PyPI binary wheels, pins in `tools/requirements-launcher.txt` | BSD-3-Clause; lz4 BSD-3-Clause; zstandard BSD-3-Clause | `python/Lib/site-packages/` | +| zlib 1.4 (MSYS2 `mingw-w64-ucrt-x86_64-zlib`), static library | https://zlib.net, Copyright (C) Jean-loup Gailly and Mark Adler | zlib | linked into `NX2navi-Setup.exe` (its unzip) | +| mingw-w64 CRT startup code and winpthreads 14.0.0, GCC 16.2.0's `libgcc` (MSYS2 packages), static | https://www.mingw-w64.org, https://gcc.gnu.org | mingw-w64: ZPL-2.1 / MIT / public domain; libgcc: GPL-3.0 with the GCC Runtime Library Exception | linked into `NX2navi.exe` and `NX2navi-Setup.exe` | + +Downloaded on the player's PC by the launcher, not distributed: the MSYS2 UCRT64 packages that +`toolchain.lock.json` pins (clang and LLVM, Apache-2.0 with LLVM exception; GCC and binutils, GPL-3.0; the +libraries of the "Compiler runtime in a source build" section and of `README-BuildKit.md`), from +https://repo.msys2.org/mingw/ucrt64/; and zbic 1.0.1.post1 (GPL-2.0), from PyPI. The game build links them as a +source build does. diff --git a/docs/README-BuildKit.md b/docs/README-BuildKit.md index 2b310c25..3f3f5302 100644 --- a/docs/README-BuildKit.md +++ b/docs/README-BuildKit.md @@ -4,6 +4,9 @@ This kit builds *The Legend of Zelda: Ocarina of Time* (Nintendo Switch 2, game program **on your PC, from your own copy of the game**. The kit contains no game code or data. Everything the build writes (`ocarina.exe`, the RomFS image, the shader cache, your saves) comes from your game: keep it to yourself. +The NX2navi launcher (`NX2navi-Setup.exe`, [launcher.md](launcher.md)) does everything below for you, the +downloads included, and keeps the game up to date. This page is the same build by hand. + ## What you need - 64-bit Windows 10 or 11, 16 GB of RAM, a Vulkan 1.3 graphics card with mesh shaders and a current driver (tested on diff --git a/docs/building.md b/docs/building.md index e51a423f..7e955617 100644 --- a/docs/building.md +++ b/docs/building.md @@ -5,8 +5,8 @@ default build targets x86-64-v3 (AVX2/FMA); a CPU without them builds with `--cp "Build"). Allow about 60 GB for the work folder. Extraction wants about 55 GB free while it runs and keeps about 35 GB afterwards. -Players who only want to play can use the [installer](installer.md) instead: it builds the game on their machine -without this toolchain. +Players who only want to play use the [launcher](launcher.md) instead: it installs its own build tools, builds the +game on their machine and keeps it up to date. ## Toolchain diff --git a/docs/launcher.md b/docs/launcher.md new file mode 100644 index 00000000..0e8f73b9 --- /dev/null +++ b/docs/launcher.md @@ -0,0 +1,160 @@ +# Launcher + +NX2navi's launcher installs everything the build needs, builds the game from the player's own dump, starts it, and +keeps the port up to date. Players download one file, `NX2navi-Setup-.exe`; nothing of the game ships with +it. The code is `launcher/`, the release is made by `tools/make_launcher.py`, and the build itself is +`tools/make_game.py` (the build kit's script, [README-BuildKit.md](README-BuildKit.md)). + +## What the player does + +1. Starts `NX2navi-Setup.exe` (from the folder that holds their `.nup` dump, ideally). It installs the launcher into + `%LOCALAPPDATA%\Programs\NX2navi` without administrator rights, adds Start menu and Desktop shortcuts and an entry + in Settings > Apps, and opens the launcher. Windows SmartScreen warns once: the programs are not signed. +2. The launcher finds the dump by itself in the setup's folder, Downloads, Desktop, Documents or the top of each + drive, and proposes `\game` as the install folder (or `X:\Games\NX2navi` on the drive with the most + free space when that drive has less than 40 GB). Both rows can be changed. +3. Selects **Begin Installation**. The launcher downloads the build tools (about 170 MB, once), checks the dump, + unpacks it, translates and compiles the game: 10 to 25 minutes. It keeps the PC awake meanwhile. +4. Selects **Play**. The launcher hides while the game runs and comes back when it exits. + +From then on the launcher looks for a new version each time it opens (Settings can turn that off). When one is out, +the Updates row turns orange; selecting it downloads the new version, restarts the launcher in it and rebuilds the +game. Saves and settings are never touched. + +## Look + +The window follows the game's options menu, with colours sampled from it: a wooden title band with the Triforce and +the name, bronze label cells, dark value cells, orange On/Off choices, a white glowing cursor around the selected +row, the menu's filigree dividers and a description box for the selected row. All text is in the game's own Cetra +Display: the launcher decodes it from the player's dump or built game into `data/cache/fonts` (as +`runtime/menu_assets.c` does for the in-game menu) and uses the system's interface font until then. Every picture is +drawn in code (`launcher/ui/art.py`); no texture, font or sound of the game is in the repository or the release. + +The Triforce is also the icon of `NX2navi.exe`, `NX2navi-Setup.exe`, the window and the taskbar +(`art.triforce_ico`, written into the release by `tools/make_launcher.py`). + +Screens (`launcher/ui/app.py`): first run (dump, install folder, Desktop Shortcut, Check for Updates at Start, +Update Channel, Begin Installation); the steps of an install, update or rebuild with their progress, the build log's +last lines and Cancel, and after a failure Try Again and Copy Report; home (Play, Updates, Game Folder, Rebuild the +Game, Settings); settings (update check, Hide Launcher While Playing, Keep Build Cache, Update Channel, the folders, +Open Logs Folder, Uninstall, Back); uninstall (keep or remove the built game). + +Controls: the mouse (hover moves the cursor, a click selects); the keyboard (arrows, Enter or Space, Esc, F5 checks +for updates); an XInput gamepad (d-pad or left stick, A, B, Y). + +`python -m launcher --selftest --shots DIR` draws every screen with made-up states and saves a PNG of each. + +## The launcher's folder + +``` +NX2navi.exe launcher/native/nx2navi.c: swaps in a staged Python, starts app\\launcher\start.pyw +python/ python.org's embeddable Python 3.14.9 with tkinter, numpy, lz4 and zstandard +app/current.txt the version the stub starts; app/previous.txt the one before it +app// launcher/, runtime/, recomp/, tools/, bin/nx2navi_xlate.exe, VERSION, + toolchain.lock.json (the pinned build tools), feed.json (where updates come from) +pydeps/ zbic, downloaded from PyPI on this PC +toolchain/msys64/ucrt64 MSYS2's clang and libraries, downloaded from the MSYS2 repository on this PC +toolchain/pkgs/ the downloaded package files +data/launcher.json settings; data/logs/launcher.log (and crash.log); data/cache/fonts +game/ the built game, unless another folder was chosen (its layout: README-BuildKit.md) +``` + +A path the Windows ANSI code page cannot hold breaks the MSYS2 tools, so the toolchain then goes to +`C:\NX2navi\toolchain` instead (`launcher/paths.py`). + +## Installing + +`jobs.InstallJob`, one row per step: + +1. **Gather Build Tools** (`launcher/toolchain.py`): the packages `toolchain.lock.json` pins (MSYS2 UCRT64 clang, + gcc, SDL3, Opus, libvpx, dav1d, zstd, Vulkan headers and loader, VMA and their dependencies: 31 packages, 170 MB), + each checked against its sha256 and unpacked into `toolchain/msys64.new`, which replaces `toolchain/msys64` once + complete. No MSYS2 shell or pacman runs. Then zbic: the wheel for the bundled Python from PyPI, checked against + PyPI's sha256, into `pydeps/`. +2. **Inspect Your Game** to **Compile**: `tools/make_game.py --nup --out --msys --events`, + whose `@@` progress lines drive the rows (check, exefs, romfs, lift, compile). +3. The game's Cetra Display is decoded for the window. + +## Updating + +A release carries `NX2navi-update.json` (version, notes, and the size and sha256 of each file). The feed +(`app//feed.json`) names a GitHub repository: the launcher reads its newest release (or pre-release, on the +Preview channel) that has the manifest. A private repository answers only with a token (`github_token` in +`data/launcher.json`); players need a public one. `feed_repo` in `data/launcher.json` points one launcher at another +repository for testing; `{"type": "url", "url": ".../NX2navi-update.json"}` in feed.json reads one manifest +directly. + +`jobs.UpdateDownloadJob`, before the restart (`launcher/updater.py`): + +1. **Download** `NX2navi-app-.zip` (about 3 MB plus the shader translator) to `data/downloads`, resuming a cut + download, and check its size and sha256. When the manifest names a Python other than `python/runtime-id.txt`, also + `NX2navi-runtime-.zip`, unpacked into `python.next`. +2. **Verify**: the archive unpacks into `app/.part`, refusing names that leave the folder, holds the expected + `VERSION` and a launcher, and becomes `app/`. +3. `app/previous.txt` gets the running version, `app/current.txt` the new one, and the launcher starts the stub with + `--continue-update ` and closes. + +The stub swaps `python.next` in while no Python runs, then starts the new version, which finishes in +`jobs.ContinueUpdateJob`: + +4. **Update Build Tools**: only when the new release pins other packages. +5. **Translate the Code** and **Compile Changed Files**: `make_game.py --rebuild`. The exefs and romfs steps depend + only on the game files, so they are kept and the dump is not needed again; the lift runs again and the compile + step compiles only the files whose source, included headers, flags or compiler changed (`build-cache/obj/ + objects.json`, from clang's dependency files). +6. **Swap In the New Build**: the link writes `ocarina.exe.new`, which replaces `ocarina.exe` only once it is + complete. A failed rebuild leaves the old build playable; Rebuild the Game tries again. +7. Every version but the current and the previous one is deleted. + +When the new version fails to start (exit code 86 from `start.pyw`, or any error within six seconds), the stub sets +`current.txt` back to `previous.txt`'s version and starts it with `--rolled-back`. The game is never rebuilt while it +runs. Keep Build Cache off deletes the lifted C and the objects (about 5 GB) after each build but keeps +`build-cache/state.json`, so updates still rebuild without the dump (compiling everything). + +## Uninstalling + +Settings > Uninstall NX2navi, or Settings > Apps > NX2navi (which runs `NX2navi.exe --uninstall`): the launcher +folder, its toolchain, the shortcuts and the Apps entry go; the built game and its saves stay unless "Remove the +Built Game" is On. The dump is never touched. + +## Command line + +`launcher/cli.py` runs the same jobs without the window (`python -m launcher.cli`, or in an installed launcher +`python\python.exe app\\launcher\cli.py`): `check`, `update`, `tools`, `install --dump F --game D`, +`rebuild [--game D]`, `fonts`. `NX2navi.exe --desktop-shortcut on|off` and `--shortcuts on|off` are the stub's own. +`NX2navi-Setup.exe` takes `--dest DIR`, `--no-shortcuts`, `--no-launch` and `--quiet`. + +## Releases + +1. Bump `VERSION`; build the shader translator (`tools/build_xlate.py`). +2. With Python 3.14 and uv: + + ```powershell + .venv-py314\Scripts\python tools\make_launcher.py --xlate build\xlate --feed owner/name --notes notes.md + ``` + + On Linux it cross-compiles the two programs with clang, lld and llvm-windres against the same MSYS2 packages and + unpacks Tcl/Tk with msiextract (msitools). It writes into `build/launcher-release`: `NX2navi-Setup-.exe` + (the setup program with the launcher folder appended as a zip), `NX2navi-app-.zip`, + `NX2navi-runtime-.zip` and `NX2navi-update.json`. `check_content` runs over the folder first. +3. Upload those four files to GitHub release `v` of the feed repository, by hand or with `--publish` + (`gh release create`; add `--draft` to look before it goes out). Launchers see it at their next start. + +The release resolves the MSYS2 packages of the day into `toolchain.lock.json` (`--lock` reuses an older lock). The +repository keeps old package files, so a release keeps building with the compiler it was tested with. + +## Tests + +`python tests/launcher_test.py` (Python 3.14): versions, settings, the toolchain resolver and installer against a +local fake repository, resumed and checked downloads, the GitHub and URL feeds, staging, switching, rollback and +pruning, unsafe archives, the font decoding on a system font wrapped like the game's, `make_game.py` on a synthetic +`.nup` (events, skipped steps, `--rebuild` without the dump, step keys), the compile step's object cache with the +system's C compiler, the install job with a fake `make_game.py`, and the window's selftest when a display is there. + +## Requirements and limits + +- Windows 10 or 11, x86-64; the build's own requirements (README-BuildKit.md): 16 GB of RAM, about 30 GB free for + the game and 1.5 GB for the launcher and its tools, a Vulkan 1.3 GPU. +- The programs are not signed: SmartScreen warns on the first start of the setup. +- English only for now. +- The launcher builds the Windows game only; Linux builds use `docs/building.md`. diff --git a/launcher/__init__.py b/launcher/__init__.py new file mode 100644 index 00000000..95636a8c --- /dev/null +++ b/launcher/__init__.py @@ -0,0 +1,8 @@ +"""The NX2navi launcher: installs the build tools, builds the game from the player's own dump, starts it and keeps +the port up to date (docs/launcher.md). + +Standard library only, plus numpy for drawing the window (launcher/ui/art.py) and zstandard where Python lacks +compression.zstd. tkinter draws the window; everything that does work runs without it (launcher/cli.py). +""" + +NAME = 'NX2navi' diff --git a/launcher/__main__.py b/launcher/__main__.py new file mode 100644 index 00000000..f6d6db36 --- /dev/null +++ b/launcher/__main__.py @@ -0,0 +1,6 @@ +"""python -m launcher [--selftest [--shots DIR]] — the launcher window (launcher/ui/app.py).""" +import sys + +from .ui.app import main + +sys.exit(main()) diff --git a/launcher/build.py b/launcher/build.py new file mode 100644 index 00000000..c1f17e9e --- /dev/null +++ b/launcher/build.py @@ -0,0 +1,135 @@ +"""Running tools/make_game.py for the launcher: the command line, its "@@" progress lines (make_game.py --events) +and cancelling it with every compiler it started.""" +import json +import os +import signal +import subprocess +import threading + +# Runs make_game.py with pydeps/ (zbic) and the script's folder first on sys.path; the embeddable Python's ._pth +# file leaves both off. +RUN = ('import os, runpy, sys; p = sys.argv[1]; s = sys.argv[2]; sys.argv = sys.argv[2:]; ' + 'sys.path[:0] = [x for x in p.split(os.pathsep) if x] + [os.path.dirname(os.path.abspath(s))]; ' + 'runpy.run_path(s, run_name="__main__")') +CREATE_NO_WINDOW = 0x08000000 +CREATE_NEW_PROCESS_GROUP = 0x00000200 + + +def command(layout, args): + return [layout.python_for_builds(), '-I', '-c', RUN, str(layout.pydeps), str(layout.make_game), *map(str, args), + '--events'] + + +def install_args(layout, dump, game_dir): + dump = str(dump) + if os.path.isdir(dump): + src = ['--exefs', os.path.join(dump, 'exefs'), '--romfs', os.path.join(dump, 'romfs')] + else: + src = ['--nup', dump] + return [*src, '--out', str(game_dir), '--msys', str(layout.msys)] + + +def rebuild_args(layout, game_dir): + return ['--rebuild', '--out', str(game_dir), '--msys', str(layout.msys)] + + +def check_args(dump): + dump = str(dump) + if os.path.isdir(dump): + return ['--exefs', os.path.join(dump, 'exefs'), '--romfs', os.path.join(dump, 'romfs'), '--check'] + return ['--nup', dump, '--check'] + + +class BuildError(Exception): + pass + + +class Run: + """One make_game.py process. on_event(dict) for each "@@" line, on_line(str) for every other line; wait() + returns the exit code. cancel() ends the process and its children.""" + + def __init__(self, cmd, on_event, on_line, cwd=None): + self.error = None + kw = {} + if os.name == 'nt': + kw['creationflags'] = CREATE_NO_WINDOW | CREATE_NEW_PROCESS_GROUP + else: + kw['start_new_session'] = True + env = dict(os.environ, PYTHONUNBUFFERED='1', PYTHONIOENCODING='utf-8') + self.proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL, + cwd=cwd, env=env, text=True, encoding='utf-8', errors='replace', bufsize=1, **kw) + self.cancelled = False + self._reader = threading.Thread(target=self._read, args=(on_event, on_line), daemon=True) + self._reader.start() + + def _read(self, on_event, on_line): + for line in self.proc.stdout: + line = line.rstrip('\r\n') + if line.startswith('@@'): + try: + ev = json.loads(line[2:]) + except ValueError: + on_line(line) + continue + if ev.get('ev') == 'error': + self.error = ev.get('message') + on_event(ev) + elif line.startswith('error: ') and not self.error: + self.error = line[7:] + on_line(line) + else: + on_line(line) + + def wait(self): + code = self.proc.wait() + self._reader.join(5) + return code + + def cancel(self): + self.cancelled = True + if self.proc.poll() is not None: + return + if os.name == 'nt': + subprocess.run(['taskkill', '/T', '/F', '/PID', str(self.proc.pid)], capture_output=True, + creationflags=CREATE_NO_WINDOW) + else: + try: + os.killpg(self.proc.pid, signal.SIGTERM) + except (ProcessLookupError, PermissionError): + pass + + +def game_running(game_dir, exe='ocarina.exe'): + """True when the built game runs from game_dir: Windows refuses to open a running exe for writing.""" + path = os.path.join(str(game_dir), exe) + if os.name != 'nt' or not os.path.isfile(path): + return False + try: + with open(path, 'r+b'): + return False + except PermissionError: + return True + except OSError: + return False + + +def build_state(game_dir): + """The game folder's build-cache/state.json (make_game.py), or {}.""" + try: + with open(os.path.join(str(game_dir), 'build-cache', 'state.json'), encoding='utf-8') as f: + return json.load(f) + except (OSError, ValueError): + return {} + + +def built_version(game_dir): + """The kit version that last built the game folder ('' when none finished).""" + st = build_state(game_dir) + if 'compile' in st.get('steps', {}) and os.path.isfile(os.path.join(str(game_dir), 'ocarina.exe')): + return st.get('kit', '') + return '' + + +def can_rebuild(game_dir): + st = build_state(game_dir) + return st.get('version') == 2 and all(s in st.get('steps', {}) for s in ('exefs', 'romfs')) diff --git a/launcher/cli.py b/launcher/cli.py new file mode 100644 index 00000000..02eb3bba --- /dev/null +++ b/launcher/cli.py @@ -0,0 +1,131 @@ +"""The launcher without its window, for scripts, tests and anyone who prefers a console: + + python -m launcher.cli check is there a newer release? + python -m launcher.cli update download and stage it, then finish in the new version + python -m launcher.cli tools install the build tools and zbic + python -m launcher.cli install --dump F --game D build the game (the window's Begin Installation) + python -m launcher.cli rebuild [--game D] rebuild after an update (the window's Rebuild the Game) + python -m launcher.cli fonts [--dump F] decode the game's font for the window + python -m launcher.cli continue-update OLD what the new version runs after `update` + +In an installed launcher: python\\python.exe app\\\\launcher\\cli.py (this file puts the release +folder on sys.path itself, as start.pyw does). Exit code 0 on success, 1 on failure, 3 when cancelled. +""" +import argparse +import os +import subprocess +import sys +import time + +if __name__ == '__main__' and not __package__: + sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + __package__ = 'launcher' # noqa: A001 — lets the relative imports below work when run as a script + +from . import build, gamefiles, jobs, versions # noqa: E402 +from .context import Context # noqa: E402 +from .paths import Layout # noqa: E402 + + +def follow(job): + """Run a job here, printing each step's state as it changes.""" + seen = {} + job.start() + while job.result is None: + for s in job.steps: + key = (s.state, s.value if s.state != jobs.RUNNING else int(s.fraction * 20)) + if seen.get(s.id) != key: + seen[s.id] = key + print(f'{s.label}: {s.value}' + (f' ({s.detail})' if s.detail and s.state == jobs.RUNNING else ''), + flush=True) + time.sleep(0.2) + for s in job.steps: + if seen.get(s.id, (None,))[0] != s.state: + print(f'{s.label}: {s.value}', flush=True) + if job.result == 'failed': + print(f'error: {job.error}', file=sys.stderr, flush=True) + return {'ok': 0, 'failed': 1, 'cancelled': 3}[job.result] + + +def main(argv=None): + ap = argparse.ArgumentParser(prog='launcher.cli', description=__doc__.splitlines()[0]) + ap.add_argument('--root', help='the launcher folder (default: this release\'s, or build/launcher-dev)') + sub = ap.add_subparsers(dest='cmd', required=True) + sub.add_parser('check') + sub.add_parser('update') + sub.add_parser('tools') + p = sub.add_parser('install') + p.add_argument('--dump', required=True) + p.add_argument('--game', required=True) + p = sub.add_parser('rebuild') + p.add_argument('--game') + p = sub.add_parser('fonts') + p.add_argument('--dump', default='') + p.add_argument('--game', default='') + p = sub.add_parser('continue-update') + p.add_argument('old') + a = ap.parse_args(argv) + ctx = Context(Layout(root=a.root) if a.root else None, echo=False) + s = ctx.settings + + if a.cmd == 'check': + job = jobs.CheckJob(ctx).run_inline() + if job.result != 'ok': + print(f'error: {job.error}', file=sys.stderr) + return 1 + r = job.release + if not r: + print('no release found') + elif jobs.update_available(ctx, r): + print(f'update available: {versions.display(r.version)} (you have {ctx.version})') + else: + print(f'up to date: {ctx.version} (newest release {versions.display(r.version)})') + return 0 + + if a.cmd == 'update': + check = jobs.CheckJob(ctx).run_inline() + if check.result != 'ok' or not jobs.update_available(ctx, check.release): + print(check.error or f'up to date: {ctx.version}') + return 0 if check.result == 'ok' else 1 + job = jobs.UpdateDownloadJob(ctx, check.release) + code = follow(job) + if code or job.after != 'restart': + return code + new = ctx.layout.apps / versions.display(check.release.version) + cli = new / 'launcher' / 'cli.py' + print(f'finishing in {versions.display(check.release.version)}', flush=True) + return subprocess.call([ctx.layout.python_for_builds(), str(cli), '--root', str(ctx.layout.root), + 'continue-update', ctx.version]) + + if a.cmd == 'continue-update': + gd = s.get('game_dir') + return follow(jobs.ContinueUpdateJob(ctx, gd if gd and build.built_version(gd) else '', a.old, + s.get('dump'))) + + if a.cmd == 'tools': + job = jobs.RebuildJob(ctx, '') + job.steps = job.steps[:1] + job.run = lambda: job.ensure_tools('tools_update') + return follow(job) + + if a.cmd == 'install': + s['dump'], s['game_dir'] = os.path.abspath(a.dump), os.path.abspath(a.game) + s.save() + return follow(jobs.InstallJob(ctx, s['dump'], s['game_dir'])) + + if a.cmd == 'rebuild': + gd = os.path.abspath(a.game) if a.game else s.get('game_dir') + if not gd: + print('error: no game folder (give --game)', file=sys.stderr) + return 1 + return follow(jobs.RebuildJob(ctx, gd, s.get('dump'))) + + if a.cmd == 'fonts': + got = gamefiles.extract_fonts(ctx.layout, a.game or s.get('game_dir'), a.dump or s.get('dump')) + for role, (path, fam) in got.items(): + print(f'{role}: {fam} ({path})') + return 0 if got else 1 + return 2 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/launcher/context.py b/launcher/context.py new file mode 100644 index 00000000..332cf300 --- /dev/null +++ b/launcher/context.py @@ -0,0 +1,48 @@ +"""What every part of the launcher shares: the folder layout, the settings and the log (data/logs/launcher.log, +the run before it in launcher.1.log).""" +import os +import threading +import time + +from .paths import Layout +from .settings import Settings + +LOG_LIMIT = 4 << 20 + + +class Context: + def __init__(self, layout=None, echo=False): + self.layout = (layout or Layout()).ensure() + self.settings = Settings(self.layout.settings) + self.version = self.layout.version + self.echo = echo + self._lock = threading.Lock() + path = self.layout.logs / 'launcher.log' + try: + if path.is_file() and path.stat().st_size > LOG_LIMIT: + os.replace(path, path.with_name('launcher.1.log')) + except OSError: + pass + try: + self._file = open(path, 'a', encoding='utf-8', errors='replace') + except OSError: + self._file = None + self.log(f'--- NX2navi launcher {self.version} ({"checkout" if self.layout.dev else "release"}) in ' + f'{self.layout.root}') + + def log(self, line): + stamp = time.strftime('%H:%M:%S') + with self._lock: + if self._file: + for part in str(line).splitlines() or ['']: + self._file.write(f'{stamp} {part}\n') + self._file.flush() + if self.echo: + print(line, flush=True) + + def log_tail(self, n=80): + try: + with open(self.layout.logs / 'launcher.log', encoding='utf-8', errors='replace') as f: + return f.readlines()[-n:] + except OSError: + return [] diff --git a/launcher/feed.py b/launcher/feed.py new file mode 100644 index 00000000..dc469080 --- /dev/null +++ b/launcher/feed.py @@ -0,0 +1,144 @@ +"""Where updates come from. + +Every release carries NX2navi-update.json (tools/make_launcher.py writes it): + + {"format": 1, "version": "0.3.11", "build": "0.3.11+abc1234", "notes": "...", + "app": {"name": "NX2navi-app-0.3.11.zip", "size": n, "sha256": "..."}, + "runtime": {"id": "py3.14.9-1", "name": "NX2navi-runtime-py3.14.9-1.zip", "size": n, "sha256": "..."}, + "toolchain": ""} + +The feed (app//feed.json, or the player's override in settings) is either + {"type": "github", "repo": "owner/name"} the newest GitHub release (or pre-release, on the preview channel) + that has the manifest; a token reads a private repository + {"type": "url", "url": "https://.../NX2navi-update.json"} one manifest; its files sit beside it +""" +import json +import urllib.parse + +from . import net, versions + +MANIFEST = 'NX2navi-update.json' +DEFAULT = {'type': 'github', 'repo': 'tangerineturtle1/NX2navi'} +GITHUB_API = 'https://api.github.com' + + +class FeedError(Exception): + pass + + +class Release: + def __init__(self, manifest, notes='', page='', assets=None, headers=None, prerelease=False): + self.manifest = manifest + self.version = manifest['version'] + self.build = manifest.get('build', self.version) + self.notes = (manifest.get('notes') or notes or '').strip() + self.page = page + self.assets = assets or {} # name -> url + self.headers = headers or {} # extra request headers for the asset urls (a private repository's token) + self.prerelease = prerelease + + def asset(self, key): + """(url, headers, size, sha256) of the manifest's 'app' or 'runtime' file.""" + entry = self.manifest.get(key) + if not entry: + raise FeedError(f'the {self.version} manifest has no {key} file') + url = self.assets.get(entry['name']) + if not url: + raise FeedError(f'the {self.version} release lacks {entry["name"]}') + return url, self.headers, entry.get('size'), entry.get('sha256') + + +def load(layout, settings): + feed = dict(DEFAULT) + try: + if layout.feed_file.is_file(): + feed.update(json.loads(layout.feed_file.read_text(encoding='utf-8'))) + except (OSError, ValueError): + pass + if settings.get('feed_repo'): + feed = {'type': 'github', 'repo': settings['feed_repo']} + return feed + + +def _check_manifest(m, where): + if not isinstance(m, dict) or m.get('format') != 1 or 'version' not in m or 'app' not in m: + raise FeedError(f'{where}: not an NX2navi update manifest') + versions.parse(m['version']) + return m + + +def _github(feed, channel, token, version): + api = feed.get('api', GITHUB_API).rstrip('/') + headers = {'Accept': 'application/vnd.github+json', 'X-GitHub-Api-Version': '2022-11-28'} + if token: + headers['Authorization'] = f'Bearer {token}' + try: + releases = net.get_json(f'{api}/repos/{feed["repo"]}/releases?per_page=20', headers, version=version) + except Exception as e: + code = getattr(e, 'code', None) + if code == 404: + raise FeedError(f'no releases found for {feed["repo"]} (a private repository needs a token)') from e + if code == 403: + raise FeedError('GitHub refused the request (rate limit); try again in an hour') from e + raise FeedError(f'GitHub could not be reached: {e}') from e + # The API lists the newest release first: the first one the channel takes that has a manifest is the answer. + for r in releases: + if r.get('draft') or (r.get('prerelease') and channel != 'preview'): + continue + assets = {a['name']: a for a in r.get('assets', [])} + if MANIFEST not in assets: + continue + if token: # a private repository's files come through the API with the token + urls = {n: a['url'] for n, a in assets.items()} + asset_headers = {'Accept': 'application/octet-stream', 'Authorization': f'Bearer {token}'} + else: + urls = {n: a['browser_download_url'] for n, a in assets.items()} + asset_headers = {} + try: + m = _check_manifest(json.loads(net.get_bytes(urls[MANIFEST], asset_headers, version=version)), + f'{r.get("tag_name")}/{MANIFEST}') + except (FeedError, ValueError, OSError) as e: + raise FeedError(f'{r.get("tag_name")}: {e}') from e + return Release(m, r.get('body') or '', r.get('html_url', ''), urls, asset_headers, bool(r.get('prerelease'))) + return None + + +def _url(feed, version): + url = feed['url'] + try: + m = _check_manifest(net.get_json(url, version=version), url) + except FeedError: + raise + except Exception as e: + raise FeedError(f'{url}: {e}') from e + base = url.rsplit('/', 1)[0] + '/' + assets = {e['name']: urllib.parse.urljoin(base, e['name']) for e in (m.get('app'), m.get('runtime')) if e} + return Release(m, assets=assets) + + +def latest(feed, channel='stable', token='', version='dev'): + """The newest release the feed offers on the channel, or None when it has none.""" + if feed.get('type') == 'url': + return _url(feed, version) + if feed.get('type') == 'github': + return _github(feed, channel, token, version) + raise FeedError(f'unknown feed type {feed.get("type")!r}') + + +def plain_notes(text, limit=12): + """Release notes as short lines for the window: markdown bullets and headings without their marks.""" + out = [] + for line in (text or '').splitlines(): + s = line.strip() + if not s or s.startswith((' + + + + + + + + + + + true/pm + PerMonitorV2 + UTF-8 + + + + + + + + diff --git a/launcher/native/common.h b/launcher/native/common.h new file mode 100644 index 00000000..68e2300f --- /dev/null +++ b/launcher/native/common.h @@ -0,0 +1,166 @@ +/* Shared by NX2navi.exe (nx2navi.c) and NX2navi-Setup.exe (setup.c): paths, small files, shortcuts and the + uninstall entry. Wide-character Win32 throughout, so folder names in any language work. */ +#pragma once +#define COBJMACROS +#define WIN32_LEAN_AND_MEAN +#include +#include +#include +#include +#include +#include +#include +#include + +#define PATHN 1024 +#define APP_NAME L"NX2navi" +#define UNINSTALL_KEY L"Software\\Microsoft\\Windows\\CurrentVersion\\Uninstall\\NX2navi" + +static inline void exe_dir(wchar_t *out, size_t n) +{ + DWORD len = GetModuleFileNameW(NULL, out, (DWORD)n); + if (len == 0 || len >= n) { out[0] = 0; return; } + while (len > 0 && out[len - 1] != L'\\' && out[len - 1] != L'/') len--; + out[len > 0 ? len - 1 : 0] = 0; +} + +static inline int path_exists(const wchar_t *p) { return GetFileAttributesW(p) != INVALID_FILE_ATTRIBUTES; } + +static inline int is_dir(const wchar_t *p) +{ + DWORD a = GetFileAttributesW(p); + return a != INVALID_FILE_ATTRIBUTES && (a & FILE_ATTRIBUTE_DIRECTORY); +} + +/* The first line of a small UTF-8 text file, trimmed, as wide text; 0 when absent or empty. */ +static inline int read_line(const wchar_t *path, wchar_t *out, size_t n) +{ + char buf[256]; + DWORD got = 0; + HANDLE f = CreateFileW(path, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_DELETE, NULL, OPEN_EXISTING, 0, NULL); + if (f == INVALID_HANDLE_VALUE) return 0; + if (!ReadFile(f, buf, sizeof buf - 1, &got, NULL)) got = 0; + CloseHandle(f); + buf[got] = 0; + char *s = buf; + if ((unsigned char)s[0] == 0xEF && (unsigned char)s[1] == 0xBB && (unsigned char)s[2] == 0xBF) s += 3; + while (*s == ' ' || *s == '\t') s++; + size_t len = strcspn(s, "\r\n"); + while (len > 0 && (s[len - 1] == ' ' || s[len - 1] == '\t')) len--; + s[len] = 0; + if (!len) return 0; + return MultiByteToWideChar(CP_UTF8, 0, s, -1, out, (int)n) > 1; +} + +static inline int write_line(const wchar_t *path, const wchar_t *text) +{ + char buf[512]; + int len = WideCharToMultiByte(CP_UTF8, 0, text, -1, buf, sizeof buf - 2, NULL, NULL); + if (len <= 0) return 0; + buf[len - 1] = '\n'; + HANDLE f = CreateFileW(path, GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + if (f == INVALID_HANDLE_VALUE) return 0; + DWORD put = 0; + BOOL ok = WriteFile(f, buf, (DWORD)len, &put, NULL); + CloseHandle(f); + return ok && put == (DWORD)len; +} + +/* Compare dotted versions ("0.3.11" > "0.3.9"); build metadata after '+' is ignored. */ +static inline int version_cmp(const wchar_t *a, const wchar_t *b) +{ + for (;;) { + long x = 0, y = 0; + while (*a >= L'0' && *a <= L'9') x = x * 10 + (*a++ - L'0'); + while (*b >= L'0' && *b <= L'9') y = y * 10 + (*b++ - L'0'); + if (x != y) return x < y ? -1 : 1; + int ea = *a != L'.', eb = *b != L'.'; + if (ea && eb) return 0; + if (!ea) a++; + if (!eb) b++; + if (ea && !eb) { while (*b == L'0' || *b == L'.') b++; return (*b >= L'1' && *b <= L'9') ? -1 : 0; } + if (eb && !ea) { while (*a == L'0' || *a == L'.') a++; return (*a >= L'1' && *a <= L'9') ? 1 : 0; } + } +} + +static inline int known_folder(REFKNOWNFOLDERID id, wchar_t *out, size_t n) +{ + PWSTR p = NULL; + if (FAILED(SHGetKnownFolderPath(id, 0, NULL, &p)) || !p) return 0; + int ok = wcslen(p) < n; + if (ok) wcscpy(out, p); + CoTaskMemFree(p); + return ok; +} + +/* A .lnk at lnk that starts target (with its own icon). */ +static inline int make_shortcut(const wchar_t *lnk, const wchar_t *target, const wchar_t *workdir, const wchar_t *desc) +{ + IShellLinkW *sl = NULL; + IPersistFile *pf = NULL; + int ok = 0; + HRESULT init = CoInitializeEx(NULL, COINIT_APARTMENTTHREADED); + if (SUCCEEDED(CoCreateInstance(&CLSID_ShellLink, NULL, CLSCTX_INPROC_SERVER, &IID_IShellLinkW, (void **)&sl))) { + IShellLinkW_SetPath(sl, target); + IShellLinkW_SetWorkingDirectory(sl, workdir); + IShellLinkW_SetIconLocation(sl, target, 0); + IShellLinkW_SetDescription(sl, desc); + if (SUCCEEDED(IShellLinkW_QueryInterface(sl, &IID_IPersistFile, (void **)&pf))) { + ok = SUCCEEDED(IPersistFile_Save(pf, lnk, TRUE)); + IPersistFile_Release(pf); + } + IShellLinkW_Release(sl); + } + if (SUCCEEDED(init)) CoUninitialize(); + return ok; +} + +/* The Desktop or Start menu shortcut's path. */ +static inline int shortcut_path(int desktop, wchar_t *out, size_t n) +{ + wchar_t dir[PATHN]; + if (!known_folder(desktop ? &FOLDERID_Desktop : &FOLDERID_Programs, dir, PATHN)) return 0; + return swprintf(out, n, L"%ls\\%ls.lnk", dir, APP_NAME) > 0; +} + +/* Create (on) or remove (off) the Desktop and/or Start menu shortcut to root\NX2navi.exe. */ +static inline int set_shortcut(const wchar_t *root, int desktop, int on) +{ + wchar_t lnk[PATHN], target[PATHN]; + if (!shortcut_path(desktop, lnk, PATHN)) return 0; + if (!on) return DeleteFileW(lnk) || GetLastError() == ERROR_FILE_NOT_FOUND; + swprintf(target, PATHN, L"%ls\\NX2navi.exe", root); + return make_shortcut(lnk, target, root, L"Play and update NX2navi"); +} + +static inline void reg_str(HKEY k, const wchar_t *name, const wchar_t *value) +{ + RegSetValueExW(k, name, 0, REG_SZ, (const BYTE *)value, (DWORD)((wcslen(value) + 1) * sizeof(wchar_t))); +} + +static inline void reg_dword(HKEY k, const wchar_t *name, DWORD value) +{ + RegSetValueExW(k, name, 0, REG_DWORD, (const BYTE *)&value, sizeof value); +} + +/* The entry in Settings > Apps that runs "NX2navi.exe --uninstall". */ +static inline int write_uninstall_entry(const wchar_t *root, const wchar_t *version, DWORD size_kb) +{ + HKEY k; + wchar_t buf[PATHN + 64]; + if (RegCreateKeyExW(HKEY_CURRENT_USER, UNINSTALL_KEY, 0, NULL, 0, KEY_SET_VALUE, NULL, &k, NULL) != ERROR_SUCCESS) + return 0; + reg_str(k, L"DisplayName", APP_NAME); + reg_str(k, L"DisplayVersion", version); + reg_str(k, L"Publisher", APP_NAME); + reg_str(k, L"InstallLocation", root); + swprintf(buf, PATHN + 64, L"\"%ls\\NX2navi.exe\",0", root); + reg_str(k, L"DisplayIcon", buf); + swprintf(buf, PATHN + 64, L"\"%ls\\NX2navi.exe\" --uninstall", root); + reg_str(k, L"UninstallString", buf); + reg_dword(k, L"NoModify", 1); + reg_dword(k, L"NoRepair", 1); + reg_dword(k, L"EstimatedSize", size_kb); + RegCloseKey(k); + return 1; +} diff --git a/launcher/native/nx2navi.c b/launcher/native/nx2navi.c new file mode 100644 index 00000000..b57dc81b --- /dev/null +++ b/launcher/native/nx2navi.c @@ -0,0 +1,117 @@ +/* NX2navi.exe: the launcher's own program, in the launcher folder (launcher/paths.py). + + 1. A Python staged by an update (python.next) replaces python\ while nothing runs from it. + 2. The release named by app\current.txt starts: python\pythonw.exe app\\launcher\start.pyw with this + program's arguments, the launcher folder as working folder and in NX2NAVI_LAUNCHER_ROOT. + 3. When that release fails to start (exit code 86 from start.pyw, or any non-zero exit within six seconds) and + app\previous.txt names another complete release, current.txt goes back to it and it starts with + --rolled-back . + + Its own commands, for the launcher and the setup: --desktop-shortcut on|off, --shortcuts on|off (Desktop and Start + menu). No console window (-mwindows). */ +#include "common.h" + +#define STARTUP_FAILED 86 +#define QUICK_MS 6000 + +static void message(const wchar_t *text) +{ + MessageBoxW(NULL, text, APP_NAME, MB_OK | MB_ICONERROR); +} + +static int release_ok(const wchar_t *root, const wchar_t *ver) +{ + wchar_t p[PATHN]; + swprintf(p, PATHN, L"%ls\\app\\%ls\\launcher\\start.pyw", root, ver); + return ver[0] && path_exists(p); +} + +static void swap_python(const wchar_t *root) +{ + wchar_t next[PATHN], cur[PATHN], old[PATHN]; + swprintf(next, PATHN, L"%ls\\python.next", root); + if (!is_dir(next)) return; + swprintf(cur, PATHN, L"%ls\\python", root); + swprintf(old, PATHN, L"%ls\\python.old-%lu", root, (unsigned long)GetTickCount()); + if (is_dir(cur) && !MoveFileExW(cur, old, 0)) return; /* in use: try again at the next start */ + if (!MoveFileExW(next, cur, 0)) MoveFileExW(old, cur, 0); +} + +/* Start the release; returns its exit code if it ended within QUICK_MS, else -1 (still running). */ +static long start_release(const wchar_t *root, const wchar_t *ver, const wchar_t *args, const wchar_t *extra) +{ + static wchar_t cmd[32768]; + wchar_t python[PATHN]; + swprintf(python, PATHN, L"%ls\\python\\pythonw.exe", root); + if (!path_exists(python)) { + message(L"The launcher's Python is missing. Run NX2navi-Setup.exe again to repair the installation."); + return 1; + } + swprintf(cmd, 32768, L"\"%ls\" \"%ls\\app\\%ls\\launcher\\start.pyw\" %ls %ls", python, root, ver, args, extra); + SetEnvironmentVariableW(L"NX2NAVI_LAUNCHER_ROOT", root); + STARTUPINFOW si = {.cb = sizeof si}; + PROCESS_INFORMATION pi; + if (!CreateProcessW(python, cmd, NULL, NULL, FALSE, 0, NULL, root, &si, &pi)) { + message(L"The launcher could not be started. Run NX2navi-Setup.exe again to repair the installation."); + return 1; + } + CloseHandle(pi.hThread); + DWORD code = 0; + long result = -1; + if (WaitForSingleObject(pi.hProcess, QUICK_MS) == WAIT_OBJECT_0 && GetExitCodeProcess(pi.hProcess, &code)) + result = (long)code; + CloseHandle(pi.hProcess); + return result; +} + +static int shortcut_command(const wchar_t *root, int argc, wchar_t **argv) +{ + for (int i = 1; i + 1 < argc; i++) { + int on = !wcscmp(argv[i + 1], L"on"); + if (!wcscmp(argv[i], L"--desktop-shortcut")) return set_shortcut(root, 1, on) ? 0 : 1; + if (!wcscmp(argv[i], L"--shortcuts")) { + int a = set_shortcut(root, 1, on), b = set_shortcut(root, 0, on); + return a && b ? 0 : 1; + } + } + return -1; +} + +int WINAPI wWinMain(HINSTANCE instance, HINSTANCE previous, PWSTR args, int show) +{ + wchar_t root[PATHN], file[PATHN], ver[128], prev[128], extra[160]; + (void)instance; (void)previous; (void)show; + exe_dir(root, PATHN); + if (!root[0]) return 1; + int argc = 0; + wchar_t **argv = CommandLineToArgvW(GetCommandLineW(), &argc); + if (argv) { + int r = shortcut_command(root, argc, argv); + LocalFree(argv); + if (r >= 0) return r; + } + swap_python(root); + swprintf(file, PATHN, L"%ls\\app\\current.txt", root); + if (!read_line(file, ver, 128) || !release_ok(root, ver)) { + swprintf(file, PATHN, L"%ls\\app\\previous.txt", root); + if (!read_line(file, ver, 128) || !release_ok(root, ver)) { + message(L"NX2navi's files are incomplete. Run NX2navi-Setup.exe again to repair the installation."); + return 1; + } + } + long code = start_release(root, ver, args ? args : L"", L""); + if (code == -1 || code == 0) return 0; + swprintf(file, PATHN, L"%ls\\app\\previous.txt", root); + if ((code == STARTUP_FAILED || code > 0) && read_line(file, prev, 128) && wcscmp(prev, ver) && + release_ok(root, prev)) { + swprintf(file, PATHN, L"%ls\\app\\current.txt", root); + write_line(file, prev); + swprintf(file, PATHN, L"%ls\\app\\previous.txt", root); + DeleteFileW(file); + swprintf(extra, 160, L"--rolled-back %ls", ver); + start_release(root, prev, args ? args : L"", extra); + return 0; + } + if (code == STARTUP_FAILED) message(L"NX2navi could not start. data\\logs\\crash.log in its folder says why."); + return (int)code; +} diff --git a/launcher/native/res.rc b/launcher/native/res.rc new file mode 100644 index 00000000..585e16c2 --- /dev/null +++ b/launcher/native/res.rc @@ -0,0 +1,4 @@ +/* The icon (the Triforce, written by tools/make_launcher.py from launcher/ui/art.py triforce_ico into the build + folder, found through -I) and the manifest of NX2navi.exe and NX2navi-Setup.exe. */ +1 ICON "nx2navi.ico" +1 24 "app.manifest" diff --git a/launcher/native/setup.c b/launcher/native/setup.c new file mode 100644 index 00000000..43299e55 --- /dev/null +++ b/launcher/native/setup.c @@ -0,0 +1,382 @@ +/* NX2navi-Setup.exe: the one-click setup. This program with the launcher's folder appended as a zip. + + It unpacks the zip into %LOCALAPPDATA%\Programs\NX2navi (no administrator rights), adds the Start menu and Desktop + shortcuts and the entry in Settings > Apps, and starts NX2navi.exe with --from , where the + launcher looks for the player's dump first. A small window in the launcher's colours shows while it unpacks. + When the folder already holds a newer version than the setup carries, the setup leaves the files alone (the + launcher updates itself) and only repairs the shortcuts. + + The end of the file is a trailer: "NX2NSET1", the zip's offset and size (u64 little endian) and the version (24 + bytes, NUL padded), written by tools/make_launcher.py. The zip is read here with zlib's inflate, so the setup + needs nothing from Windows beyond its own DLLs. + + Options: --dest DIR, --no-shortcuts, --no-launch, --quiet (no window; for tests). */ +#include "common.h" +#include +#include +#include + +#define TRAILER 48 +#define WM_DONE (WM_APP + 1) + +typedef struct { + wchar_t exe[PATHN], dest[PATHN], src_dir[PATHN]; + uint64_t off, size; + wchar_t version[32]; + int shortcuts, launch, quiet; + int result; /* 0 ok, else an error message id */ + wchar_t error[512]; + uint64_t written; +} Setup; + +static Setup S; +static HWND g_wnd; + +static void fail_msg(const wchar_t *fmt, const wchar_t *arg) +{ + swprintf(S.error, 512, fmt, arg ? arg : L""); + S.result = 1; +} + +/* ---------------------------------------------------------------- reading our own file */ + +typedef struct { HANDLE f; uint64_t base, size; } Src; + +static int src_read(Src *s, uint64_t at, void *buf, DWORD n) +{ + LARGE_INTEGER li; + DWORD got = 0; + if (at + n > s->size) return 0; + li.QuadPart = (LONGLONG)(s->base + at); + if (!SetFilePointerEx(s->f, li, NULL, FILE_BEGIN)) return 0; + return ReadFile(s->f, buf, n, &got, NULL) && got == n; +} + +static uint16_t le16(const unsigned char *p) { return (uint16_t)(p[0] | p[1] << 8); } +static uint32_t le32(const unsigned char *p) { return (uint32_t)p[0] | (uint32_t)p[1] << 8 | (uint32_t)p[2] << 16 | (uint32_t)p[3] << 24; } +static uint64_t le64(const unsigned char *p) { return (uint64_t)le32(p) | (uint64_t)le32(p + 4) << 32; } + +static int read_trailer(void) +{ + unsigned char t[TRAILER]; + HANDLE f = CreateFileW(S.exe, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, 0, NULL); + if (f == INVALID_HANDLE_VALUE) return 0; + LARGE_INTEGER size, li; + DWORD got = 0; + int ok = GetFileSizeEx(f, &size) && size.QuadPart > TRAILER; + if (ok) { + li.QuadPart = size.QuadPart - TRAILER; + ok = SetFilePointerEx(f, li, NULL, FILE_BEGIN) && ReadFile(f, t, TRAILER, &got, NULL) && got == TRAILER && + !memcmp(t, "NX2NSET1", 8); + } + CloseHandle(f); + if (!ok) return 0; + S.off = le64(t + 8); + S.size = le64(t + 16); + char v[25]; + memcpy(v, t + 24, 24); + v[24] = 0; + MultiByteToWideChar(CP_UTF8, 0, v, -1, S.version, 32); + return S.off + S.size <= (uint64_t)size.QuadPart; +} + +/* ---------------------------------------------------------------- unzip */ + +static int make_dirs(wchar_t *path) /* every folder above path (path is a file) */ +{ + for (wchar_t *p = path + 3; *p; p++) { + if (*p == L'\\') { + *p = 0; + if (!is_dir(path) && !CreateDirectoryW(path, NULL) && GetLastError() != ERROR_ALREADY_EXISTS) { + *p = L'\\'; + return 0; + } + *p = L'\\'; + } + } + return 1; +} + +static int safe_name(const char *name) +{ + if (!name[0] || name[0] == '/' || name[0] == '\\' || strchr(name, ':')) return 0; + for (const char *p = name; *p; p++) + if ((p == name || p[-1] == '/' || p[-1] == '\\') && p[0] == '.' && p[1] == '.' && + (p[2] == '/' || p[2] == '\\' || !p[2])) + return 0; + return 1; +} + +static int extract_entry(Src *z, uint64_t local, uint32_t csize, uint32_t usize, uint16_t method, uint32_t crc, + const wchar_t *target) +{ + unsigned char lh[30]; + if (!src_read(z, local, lh, 30) || le32(lh) != 0x04034b50) return 0; + uint64_t data = local + 30 + le16(lh + 26) + le16(lh + 28); + HANDLE out = CreateFileW(target, GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + if (out == INVALID_HANDLE_VALUE) return 0; + static unsigned char in[1 << 16], buf[1 << 17]; + uLong check = crc32(0L, Z_NULL, 0); + int ok = 1; + uint64_t left = csize, pos = data, total = 0; + z_stream st; + memset(&st, 0, sizeof st); + if (method == 8 && inflateInit2(&st, -MAX_WBITS) != Z_OK) ok = 0; + while (ok && (left || (method == 8 && st.avail_in))) { + DWORD n = (DWORD)(left < sizeof in ? left : sizeof in); + if (n && !src_read(z, pos, in, n)) { ok = 0; break; } + pos += n; + left -= n; + if (method == 0) { + DWORD put; + check = crc32(check, in, n); + ok = WriteFile(out, in, n, &put, NULL) && put == n; + total += n; + continue; + } + st.next_in = in; + st.avail_in = n; + do { + st.next_out = buf; + st.avail_out = sizeof buf; + int r = inflate(&st, Z_NO_FLUSH); + if (r != Z_OK && r != Z_STREAM_END && !(r == Z_BUF_ERROR && st.avail_in == 0)) { ok = 0; break; } + DWORD have = (DWORD)(sizeof buf - st.avail_out), put; + if (have) { + check = crc32(check, buf, have); + if (!WriteFile(out, buf, have, &put, NULL) || put != have) { ok = 0; break; } + total += have; + } + if (r == Z_STREAM_END) { left = 0; st.avail_in = 0; break; } + } while (st.avail_out == 0); + } + if (method == 8) inflateEnd(&st); + CloseHandle(out); + S.written += total; + return ok && total == usize && check == crc; +} + +static int unzip(void) +{ + Src z; + unsigned char eocd[22]; + z.f = CreateFileW(S.exe, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, 0, NULL); + if (z.f == INVALID_HANDLE_VALUE) { fail_msg(L"The setup cannot read itself.", NULL); return 0; } + z.base = S.off; + z.size = S.size; + /* the end-of-central-directory record: the zip has no comment, so it is the last 22 bytes */ + if (!src_read(&z, z.size - 22, eocd, 22) || le32(eocd) != 0x06054b50) { + CloseHandle(z.f); + fail_msg(L"The setup is damaged. Download it again.", NULL); + return 0; + } + uint32_t count = le16(eocd + 10), cd_size = le32(eocd + 12), cd_off = le32(eocd + 16); + unsigned char *cd = malloc(cd_size); + if (!cd || !src_read(&z, cd_off, cd, cd_size)) { + free(cd); + CloseHandle(z.f); + fail_msg(L"The setup is damaged. Download it again.", NULL); + return 0; + } + wchar_t target[PATHN], wname[PATHN]; + char name[PATHN]; + uint32_t at = 0; + int ok = 1; + for (uint32_t i = 0; ok && i < count; i++) { + if (at + 46 > cd_size || le32(cd + at) != 0x02014b50) { ok = 0; break; } + uint16_t method = le16(cd + at + 10), nlen = le16(cd + at + 28), xlen = le16(cd + at + 30), + clen = le16(cd + at + 32); + uint32_t crc = le32(cd + at + 16), csize = le32(cd + at + 20), usize = le32(cd + at + 24), + local = le32(cd + at + 42); + if (nlen >= PATHN || at + 46 + nlen > cd_size || (method != 0 && method != 8)) { ok = 0; break; } + memcpy(name, cd + at + 46, nlen); + name[nlen] = 0; + at += 46 + nlen + xlen + clen; + if (!safe_name(name)) { ok = 0; break; } + MultiByteToWideChar(CP_UTF8, 0, name, -1, wname, PATHN); + for (wchar_t *p = wname; *p; p++) + if (*p == L'/') *p = L'\\'; + swprintf(target, PATHN, L"%ls\\%ls", S.dest, wname); + size_t tl = wcslen(target); + if (tl && target[tl - 1] == L'\\') { + target[tl - 1] = 0; + ok = make_dirs(wcscat(target, L"\\")) && (is_dir(target) || CreateDirectoryW(target, NULL)); + continue; + } + if (!make_dirs(target)) { ok = 0; break; } + if (!extract_entry(&z, local, csize, usize, method, crc, target)) { + fail_msg(L"%ls could not be written. Close NX2navi if it is open, then run the setup again.", target); + free(cd); + CloseHandle(z.f); + return 0; + } + if (g_wnd && (i & 31) == 0) InvalidateRect(g_wnd, NULL, FALSE); + } + free(cd); + CloseHandle(z.f); + if (!ok) fail_msg(L"The setup is damaged. Download it again.", NULL); + return ok; +} + +/* ---------------------------------------------------------------- the work */ + +static DWORD WINAPI work(void *arg) +{ + wchar_t cur[PATHN], have[64]; + (void)arg; + swprintf(cur, PATHN, L"%ls\\app\\current.txt", S.dest); + int newer_installed = read_line(cur, have, 64) && version_cmp(have, S.version) > 0; + wchar_t probe[PATHN]; + swprintf(probe, PATHN, L"%ls\\x", S.dest); + if (!make_dirs(probe)) { + fail_msg(L"The folder %ls cannot be created.", S.dest); + } else if (!newer_installed) { + unzip(); + } + if (!S.result) { + if (S.shortcuts) { + set_shortcut(S.dest, 0, 1); + set_shortcut(S.dest, 1, 1); + } + write_uninstall_entry(S.dest, newer_installed ? have : S.version, (DWORD)(S.written / 1024 + 1)); + } + if (g_wnd) PostMessageW(g_wnd, WM_DONE, 0, 0); + return 0; +} + +/* ---------------------------------------------------------------- the window */ + +static LRESULT CALLBACK wndproc(HWND w, UINT m, WPARAM wp, LPARAM lp) +{ + switch (m) { + case WM_PAINT: { + PAINTSTRUCT ps; + HDC dc = BeginPaint(w, &ps); + RECT r; + GetClientRect(w, &r); + HBRUSH bg = CreateSolidBrush(RGB(0x0c, 0x2e, 0x1b)), edge = CreateSolidBrush(RGB(0xa9, 0x87, 0x4f)); + FillRect(dc, &r, edge); + RECT in = {r.left + 2, r.top + 2, r.right - 2, r.bottom - 2}; + FillRect(dc, &in, bg); + /* the Triforce: three gold triangles */ + HBRUSH gold = CreateSolidBrush(RGB(0xe8, 0xaf, 0x3d)); + HPEN none = CreatePen(PS_NULL, 0, 0); + HGDIOBJ ob = SelectObject(dc, gold), op = SelectObject(dc, none); + int cx = 52, top = 28, s = 44; + POINT t1[3] = {{cx, top}, {cx + s / 4, top + s * 433 / 1000}, {cx - s / 4, top + s * 433 / 1000}}; + POINT t2[3] = {{cx - s / 4, top + s * 433 / 1000 + 1}, {cx, top + s * 866 / 1000}, {cx - s / 2, top + s * 866 / 1000}}; + POINT t3[3] = {{cx + s / 4, top + s * 433 / 1000 + 1}, {cx + s / 2, top + s * 866 / 1000}, {cx, top + s * 866 / 1000}}; + Polygon(dc, t1, 3); + Polygon(dc, t2, 3); + Polygon(dc, t3, 3); + SelectObject(dc, ob); + SelectObject(dc, op); + HFONT big = CreateFontW(-26, 0, 0, 0, FW_SEMIBOLD, 0, 0, 0, DEFAULT_CHARSET, 0, 0, CLEARTYPE_QUALITY, 0, L"Segoe UI"); + HFONT small = CreateFontW(-16, 0, 0, 0, FW_NORMAL, 0, 0, 0, DEFAULT_CHARSET, 0, 0, CLEARTYPE_QUALITY, 0, L"Segoe UI"); + SetBkMode(dc, TRANSPARENT); + SetTextColor(dc, RGB(0xf3, 0xec, 0xdc)); + HGDIOBJ of = SelectObject(dc, big); + TextOutW(dc, 96, 30, APP_NAME, (int)wcslen(APP_NAME)); + SelectObject(dc, small); + SetTextColor(dc, RGB(0xe9, 0xdc, 0xc0)); + wchar_t line[96]; + swprintf(line, 96, L"Installing version %ls\x2026 %llu MB", S.version, (unsigned long long)(S.written >> 20)); + TextOutW(dc, 96, 66, line, (int)wcslen(line)); + SelectObject(dc, of); + DeleteObject(big); + DeleteObject(small); + DeleteObject(gold); + DeleteObject(none); + DeleteObject(bg); + DeleteObject(edge); + EndPaint(w, &ps); + return 0; + } + case WM_TIMER: + InvalidateRect(w, NULL, FALSE); + return 0; + case WM_DONE: + DestroyWindow(w); + return 0; + case WM_DESTROY: + PostQuitMessage(0); + return 0; + } + return DefWindowProcW(w, m, wp, lp); +} + +static void run_with_window(HINSTANCE inst) +{ + WNDCLASSW wc = {0}; + wc.lpfnWndProc = wndproc; + wc.hInstance = inst; + wc.hCursor = LoadCursor(NULL, IDC_WAIT); + wc.hIcon = LoadIconW(inst, MAKEINTRESOURCEW(1)); + wc.lpszClassName = L"NX2naviSetup"; + RegisterClassW(&wc); + int w = 440, h = 120; + int x = (GetSystemMetrics(SM_CXSCREEN) - w) / 2, y = (GetSystemMetrics(SM_CYSCREEN) - h) / 3; + g_wnd = CreateWindowExW(WS_EX_APPWINDOW, wc.lpszClassName, L"NX2navi Setup", WS_POPUP | WS_VISIBLE, x, y, w, h, + NULL, NULL, inst, NULL); + SetTimer(g_wnd, 1, 200, NULL); + HANDLE t = CreateThread(NULL, 0, work, NULL, 0, NULL); + MSG msg; + while (GetMessageW(&msg, NULL, 0, 0) > 0) { + TranslateMessage(&msg); + DispatchMessageW(&msg); + } + WaitForSingleObject(t, INFINITE); + CloseHandle(t); +} + +int WINAPI wWinMain(HINSTANCE inst, HINSTANCE prev, PWSTR cmdline, int show) +{ + (void)prev; (void)cmdline; (void)show; + GetModuleFileNameW(NULL, S.exe, PATHN); + exe_dir(S.src_dir, PATHN); + S.shortcuts = S.launch = 1; + S.dest[0] = 0; + int argc = 0; + wchar_t **argv = CommandLineToArgvW(GetCommandLineW(), &argc); + for (int i = 1; argv && i < argc; i++) { + if (!wcscmp(argv[i], L"--dest") && i + 1 < argc) { + GetFullPathNameW(argv[++i], PATHN, S.dest, NULL); + } else if (!wcscmp(argv[i], L"--no-shortcuts")) S.shortcuts = 0; + else if (!wcscmp(argv[i], L"--no-launch")) S.launch = 0; + else if (!wcscmp(argv[i], L"--quiet")) S.quiet = 1; + } + if (argv) LocalFree(argv); + if (!S.dest[0]) { + wchar_t base[PATHN]; + if (!known_folder(&FOLDERID_UserProgramFiles, base, PATHN)) { + if (!GetEnvironmentVariableW(L"LOCALAPPDATA", base, PATHN)) return 1; + wcscat(base, L"\\Programs"); + } + swprintf(S.dest, PATHN, L"%ls\\%ls", base, APP_NAME); + } + size_t dl = wcslen(S.dest); + while (dl > 3 && (S.dest[dl - 1] == L'\\' || S.dest[dl - 1] == L'/')) S.dest[--dl] = 0; + if (!read_trailer()) { + MessageBoxW(NULL, L"This setup is incomplete. Download NX2navi-Setup.exe again.", APP_NAME, MB_ICONERROR); + return 1; + } + if (S.quiet) work(NULL); + else run_with_window(inst); + if (S.result) { + if (!S.quiet) MessageBoxW(NULL, S.error, APP_NAME, MB_OK | MB_ICONERROR); + return 2; + } + if (S.launch) { + static wchar_t cmd[PATHN * 3]; + wchar_t exe[PATHN]; + swprintf(exe, PATHN, L"%ls\\NX2navi.exe", S.dest); + swprintf(cmd, PATHN * 3, L"\"%ls\" --from \"%ls\"", exe, S.src_dir); + STARTUPINFOW si = {.cb = sizeof si}; + PROCESS_INFORMATION pi; + if (CreateProcessW(exe, cmd, NULL, NULL, FALSE, 0, NULL, S.dest, &si, &pi)) { + CloseHandle(pi.hThread); + CloseHandle(pi.hProcess); + } + } + return 0; +} diff --git a/launcher/net.py b/launcher/net.py new file mode 100644 index 00000000..15b7a101 --- /dev/null +++ b/launcher/net.py @@ -0,0 +1,128 @@ +"""HTTPS for the launcher: JSON requests and file downloads that resume, report progress, can be cancelled and are +checked against a sha256. Standard library only (urllib); Windows' own certificate store verifies the servers.""" +import hashlib +import json +import os +import ssl +import time +import urllib.error +import urllib.request +from pathlib import Path + +from . import NAME + +CHUNK = 1 << 20 + + +class Cancelled(Exception): + pass + + +class DownloadError(Exception): + pass + + +def user_agent(version='dev'): + return f'{NAME}-Launcher/{version}' + + +_CTX = None + + +def _context(): + global _CTX + if _CTX is None: + _CTX = ssl.create_default_context() + return _CTX + + +def request(url, headers=None, timeout=30, version='dev'): + h = {'User-Agent': user_agent(version)} + h.update(headers or {}) + req = urllib.request.Request(url, headers=h) + ctx = _context() if url.startswith('https:') else None + return urllib.request.urlopen(req, timeout=timeout, context=ctx) + + +def get_bytes(url, headers=None, timeout=30, version='dev', limit=64 << 20): + with request(url, headers, timeout, version) as r: + data = r.read(limit + 1) + if len(data) > limit: + raise DownloadError(f'{url}: more than {limit >> 20} MB') + return data + + +def get_json(url, headers=None, timeout=30, version='dev'): + return json.loads(get_bytes(url, headers, timeout, version).decode('utf-8')) + + +def sha256_file(path): + h = hashlib.sha256() + with open(path, 'rb') as f: + while b := f.read(CHUNK): + h.update(b) + return h.hexdigest() + + +def download(url, dest, sha256=None, size=None, progress=None, cancel=None, headers=None, retries=4, version='dev'): + """Fetch url to dest through dest.part, resuming a part file a cut connection left. progress(done, total) after + every chunk; cancel() returning True raises Cancelled. A wrong size or sha256 deletes the file and raises + DownloadError. A dest already holding the expected sha256 is kept without a request.""" + dest = Path(dest) + dest.parent.mkdir(parents=True, exist_ok=True) + if sha256 and dest.is_file() and sha256_file(dest) == sha256.lower(): + if progress: + progress(dest.stat().st_size, dest.stat().st_size) + return dest + part = dest.with_name(dest.name + '.part') + last_error = None + for attempt in range(retries): + have = part.stat().st_size if part.is_file() else 0 + h = dict(headers or {}) + if have: + h['Range'] = f'bytes={have}-' + try: + with request(url, h, 60, version) as r: + status = getattr(r, 'status', 200) + if have and status != 206: # the server ignores Range: start over + have = 0 + total = int(r.headers.get('Content-Length') or 0) + have or size or 0 + with open(part, 'ab' if have else 'wb') as f: + done = have + while True: + if cancel and cancel(): + raise Cancelled() + b = r.read(CHUNK) + if not b: + break + f.write(b) + done += len(b) + if progress: + progress(done, total or done) + break + except Cancelled: + raise + except urllib.error.HTTPError as e: + if e.code == 416 and have: # the part file is complete or longer than the file: start over + part.unlink(missing_ok=True) + last_error = e + continue + if e.code in (401, 403, 404, 410): + raise DownloadError(f'{url}: HTTP {e.code}') from e + last_error = e + except (urllib.error.URLError, OSError, TimeoutError) as e: + last_error = e + time.sleep(min(2 ** attempt, 16)) + else: + raise DownloadError(f'{url}: {last_error}') + got = part.stat().st_size + if size and got != size: + part.unlink(missing_ok=True) + raise DownloadError(f'{url}: {got} bytes, expected {size}') + if sha256: + digest = sha256_file(part) + if digest != sha256.lower(): + part.unlink(missing_ok=True) + raise DownloadError(f'{url}: sha256 {digest}, expected {sha256.lower()}') + os.replace(part, dest) + return dest diff --git a/launcher/paths.py b/launcher/paths.py new file mode 100644 index 00000000..ba6ee7a0 --- /dev/null +++ b/launcher/paths.py @@ -0,0 +1,115 @@ +"""Where the launcher keeps everything. + +An installed launcher (NX2navi-Setup.exe puts it in %LOCALAPPDATA%\\Programs\\NX2navi) is one folder: + + NX2navi.exe launcher/native/nx2navi.c: starts python\\pythonw.exe on app\\\\launcher\\start.pyw + python/ the embeddable Python with tkinter, numpy, lz4 and zstandard (tools/make_launcher.py) + app/current.txt the version the stub starts; app/previous.txt the one before it, kept for a rollback + app// one release: launcher/, runtime/, recomp/, tools/, bin/, VERSION, toolchain.lock.json + pydeps/ zbic, downloaded from PyPI on this PC (launcher/toolchain.py ensure_zbic) + toolchain/msys64/ucrt64 MSYS2's clang and libraries, downloaded from the MSYS2 repository on this PC + toolchain/pkgs/ the downloaded package files, kept while the lock names them + data/launcher.json settings; data/logs/ logs; data/cache/ the fonts read from the player's game + game/ the built game, unless the player picks another folder + +Run from a checkout of the repository (python -m launcher), the app folder is the repository and everything else goes +to build/launcher-dev/, which git ignores. +""" +import os +import sys +from pathlib import Path + +APP = Path(__file__).resolve().parents[1] + + +def _is_release_app(app): + return app.parent.name == 'app' and (app.parent / 'current.txt').is_file() + + +def ansi_safe(path): + """True when the path survives the Windows ANSI code page: the MSYS2 tools and the game's own build read their + arguments through it, so a folder name outside it breaks the build. Always True elsewhere.""" + if os.name != 'nt': + return True + try: + str(path).encode('mbcs', errors='strict') + return True + except (UnicodeEncodeError, LookupError): + return False + + +class Layout: + def __init__(self, root=None, app=None): + self.app = Path(app or APP) + self.dev = not _is_release_app(self.app) + env = os.environ.get('NX2NAVI_LAUNCHER_ROOT') + if root: + self.root = Path(root) + elif env: + self.root = Path(env) + elif not self.dev: + self.root = self.app.parents[1] + else: + self.root = self.app / 'build' / 'launcher-dev' + self.apps = self.root / 'app' + self.python = self.root / 'python' + self.stub = self.root / 'NX2navi.exe' + self.pydeps = self.root / 'pydeps' + self.data = self.root / 'data' + self.logs = self.data / 'logs' + self.cache = self.data / 'cache' + self.downloads = self.data / 'downloads' + self.settings = self.data / 'launcher.json' + tool_root = self.root + if not ansi_safe(tool_root): + tool_root = Path(os.environ.get('SystemDrive', 'C:') + '\\') / 'NX2navi' + self.toolchain = tool_root / 'toolchain' + self.msys_root = self.toolchain / 'msys64' + self.msys = self.msys_root / 'ucrt64' + self.pkgs = self.toolchain / 'pkgs' + self.tools = self.app / 'tools' + self.make_game = self.tools / 'make_game.py' + self.lock_file = self.app / 'toolchain.lock.json' + self.feed_file = self.app / 'feed.json' + + @property + def version(self): + v = self.app / 'VERSION' + return v.read_text(encoding='utf-8').strip() if v.is_file() else '0.0.0' + + def default_game_dir(self): + """/game, or a Games folder on the fixed drive with the most free space when the root's drive has + less than 40 GB free or the root's path would break the build.""" + here = self.root / 'game' + try: + import shutil + if shutil.disk_usage(self.root.anchor or self.root).free >= 40e9 and ansi_safe(here): + return here + except OSError: + return here + best = None + if os.name == 'nt': + import shutil + import string + for letter in string.ascii_uppercase[2:]: + drive = Path(f'{letter}:\\') + try: + free = shutil.disk_usage(drive).free + except OSError: + continue + if best is None or free > best[0]: + best = (free, drive / 'Games' / 'NX2navi') + return best[1] if best else here + + def python_for_builds(self): + """The console python.exe beside the launcher's pythonw.exe (no window: the build starts it with + CREATE_NO_WINDOW), or this interpreter in a checkout.""" + exe = Path(sys.executable) + if exe.name.lower() == 'pythonw.exe' and (exe.parent / 'python.exe').is_file(): + return str(exe.parent / 'python.exe') + return str(exe) + + def ensure(self): + for d in (self.data, self.logs, self.cache, self.downloads): + d.mkdir(parents=True, exist_ok=True) + return self diff --git a/launcher/settings.py b/launcher/settings.py new file mode 100644 index 00000000..d9cd89ee --- /dev/null +++ b/launcher/settings.py @@ -0,0 +1,43 @@ +"""data/launcher.json: the player's choices and what the launcher remembers between runs.""" +import json +import os +from pathlib import Path + +DEFAULTS = { + 'game_dir': '', # the built game; empty until the first install picks one + 'dump': '', # the .nup (or the folder with exefs/ and romfs/) the game was built from + 'auto_check': True, # look for a new version when the launcher opens + 'channel': 'stable', # 'stable' or 'preview' (GitHub pre-releases) + 'hide_while_playing': True, # hide the window while the game runs, show it again when it exits + 'keep_cache': True, # keep build-cache/ (the lift and objects) so an update recompiles only changes + 'desktop_shortcut': True, + 'feed_repo': '', # owner/name overriding feed.json, for testing a fork's releases + 'github_token': '', # only for a private release repository; stored as plain text + 'last_check': 0.0, + 'last_seen': '', # the newest version the player was told about +} + + +class Settings(dict): + def __init__(self, path): + super().__init__(DEFAULTS) + self.path = Path(path) + try: + data = json.loads(self.path.read_text(encoding='utf-8')) + if isinstance(data, dict): + self.update({k: v for k, v in data.items() if k in DEFAULTS and type(v) is type(DEFAULTS[k])}) + except (OSError, ValueError): + pass + + def save(self): + self.path.parent.mkdir(parents=True, exist_ok=True) + tmp = self.path.with_name(self.path.name + '.tmp') + tmp.write_text(json.dumps(dict(self), indent=1), encoding='utf-8') + os.replace(tmp, self.path) + + def reset(self): + """Default Settings: the choices go back to their defaults; the game folder, dump and history stay.""" + keep = {k: self[k] for k in ('game_dir', 'dump', 'last_check', 'last_seen', 'feed_repo', 'github_token')} + self.clear() + self.update(DEFAULTS) + self.update(keep) diff --git a/launcher/start.pyw b/launcher/start.pyw new file mode 100644 index 00000000..5339fda3 --- /dev/null +++ b/launcher/start.pyw @@ -0,0 +1,45 @@ +"""What NX2navi.exe runs: python\\pythonw.exe app\\\\launcher\\start.pyw [arguments]. + +The embeddable Python puts only its ._pth entries on sys.path, so the release folder (the parent of launcher/) goes +first here. A failure before the window is up is written to data/logs/crash.log, shown in a message box and ends the +process with exit code 86, which tells the stub to go back to the previous version (launcher/native/nx2navi.c). +""" +import os +import sys +import traceback + +APP = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, APP) +STARTUP_FAILED = 86 + + +def crash(text): + root = os.environ.get('NX2NAVI_LAUNCHER_ROOT') or os.path.dirname(os.path.dirname(APP)) + try: + logs = os.path.join(root, 'data', 'logs') + os.makedirs(logs, exist_ok=True) + with open(os.path.join(logs, 'crash.log'), 'a', encoding='utf-8') as f: + f.write(text + '\n') + except OSError: + pass + try: + import ctypes + ctypes.windll.user32.MessageBoxW(None, 'NX2navi could not start.\n\n' + text[-1500:], 'NX2navi', 0x10) + except Exception: + print(text, file=sys.stderr) + + +try: + from launcher.ui.app import main +except Exception: + crash(traceback.format_exc()) + sys.exit(STARTUP_FAILED) + +try: + code = main() +except SystemExit: + raise +except Exception: + crash(traceback.format_exc()) + sys.exit(1) +sys.exit(code or 0) diff --git a/launcher/toolchain.py b/launcher/toolchain.py new file mode 100644 index 00000000..79382ecf --- /dev/null +++ b/launcher/toolchain.py @@ -0,0 +1,288 @@ +"""The build tools, installed on the player's PC: MSYS2's UCRT64 clang and the libraries tools/make_game.py links +(the packages README-BuildKit.md lists), and the zbic Python package. + +No MSYS2 shell and no pacman: the packages and everything they depend on are read from the repository's database, +downloaded, checked against the sha256 the database (or the release's lock) names, and unpacked into +toolchain/msys64/ucrt64 as pacman would lay them out. A release pins the exact package files in +toolchain.lock.json (make_lock, written by tools/make_launcher.py), so every player builds with the compiler the +release was tested with; the repository keeps old package files. A checkout without a lock resolves the newest +packages instead. + +zbic (GPL-2.0) unpacks the game's compressed program files. The release does not carry it: the launcher downloads the +wheel for its own Python from PyPI and unpacks it into pydeps/, as `pip install zbic` would. +""" +import hashlib +import io +import json +import os +import re +import shutil +import sys +import tarfile +import time +import zipfile +from pathlib import Path + +from . import net + +REPO = 'https://repo.msys2.org/mingw/ucrt64/' +MIRRORS = (REPO, 'https://mirror.msys2.org/mingw/ucrt64/') +PREFIX = 'mingw-w64-ucrt-x86_64-' +ROOTS = ('clang', 'gcc', 'sdl3', 'opus', 'libvpx', 'dav1d', 'zstd', 'vulkan-headers', 'vulkan-loader', + 'vulkan-memory-allocator') +REQUIRED = ('bin/clang.exe', 'bin/clang++.exe', 'bin/SDL3.dll', 'lib/libSDL3.dll.a', 'include/vma') +SKIP_SHARE = ('ucrt64/share/doc/', 'ucrt64/share/man/', 'ucrt64/share/info/', 'ucrt64/share/locale/', + 'ucrt64/share/gtk-doc/', 'ucrt64/share/licenses/') +ZBIC = ('zbic', '1.0.1.post1') +LOCK_FORMAT = 1 + + +class ToolchainError(Exception): + pass + + +# ---------------------------------------------------------------- the package database + +def _zstd_decompress_stream(data): + try: + from compression import zstd # Python 3.14 + return zstd.decompress(data) + except ImportError: + import zstandard + return zstandard.ZstdDecompressor().decompressobj().decompress(data) + + +def _open_tar(path_or_bytes): + """A tarfile over a .tar.zst (or plain/gzip tar) file or bytes.""" + data = path_or_bytes if isinstance(path_or_bytes, (bytes, bytearray)) else Path(path_or_bytes).read_bytes() + if data[:4] == b'\x28\xb5\x2f\xfd': + data = _zstd_decompress_stream(data) + return tarfile.open(fileobj=io.BytesIO(data), mode='r:*') + + +def parse_db(data): + """{name: desc} for a pacman sync database (ucrt64.db): each desc is {FIELD: [lines]}.""" + out = {} + with _open_tar(data) as t: + for m in t.getmembers(): + if not m.name.endswith('/desc'): + continue + desc, field = {}, None + for line in t.extractfile(m).read().decode('utf-8').splitlines(): + if line.startswith('%') and line.endswith('%') and len(line) > 2: + field = line[1:-1] + desc[field] = [] + elif line and field: + desc[field].append(line) + if desc.get('NAME'): + out[desc['NAME'][0]] = desc + return out + + +def _dep_name(dep): + return re.split(r'[<>=:]', dep, maxsplit=1)[0].strip() + + +def resolve(db, roots=ROOTS): + """Every package the roots need, roots first; names a package provides resolve to the package.""" + provides = {} + for name, d in db.items(): + for p in d.get('PROVIDES', []): + provides.setdefault(_dep_name(p), name) + order, seen = [], set() + todo = [r if r.startswith(PREFIX) else PREFIX + r for r in roots] + while todo: + n = _dep_name(todo.pop(0)) + n = n if n in db else provides.get(n, n) + if n in seen: + continue + if n not in db: + raise ToolchainError(f'the package database has no {n}') + seen.add(n) + order.append(n) + todo += db[n].get('DEPENDS', []) + pkgs = [] + for n in order: + d = db[n] + pkgs.append({'name': n, 'version': d['VERSION'][0], 'filename': d['FILENAME'][0], + 'sha256': d['SHA256SUM'][0], 'csize': int(d['CSIZE'][0]), 'isize': int(d.get('ISIZE', ['0'])[0])}) + return pkgs + + +def make_lock(repo=REPO, roots=ROOTS, version='dev'): + """The lock a release ships: the resolved package files of the repository's current database.""" + db = parse_db(net.get_bytes(repo + 'ucrt64.db', version=version)) + return {'format': LOCK_FORMAT, 'repo': repo, 'roots': list(roots), 'packages': resolve(db, roots), + 'zbic': {'name': ZBIC[0], 'version': ZBIC[1]}, 'made': time.strftime('%Y-%m-%d')} + + +def lock_digest(lock): + files = sorted(p['filename'] + ':' + p['sha256'] for p in lock['packages']) + return hashlib.sha256('\n'.join(files).encode()).hexdigest() + + +def load_lock(layout, version='dev'): + """The release's toolchain.lock.json, or a lock resolved from the live database in a checkout.""" + if layout.lock_file.is_file(): + lock = json.loads(layout.lock_file.read_text(encoding='utf-8')) + if lock.get('format') != LOCK_FORMAT: + raise ToolchainError(f'{layout.lock_file}: unknown format {lock.get("format")}') + return lock + return make_lock(version=version) + + +# ---------------------------------------------------------------- install + +def installed(layout): + p = layout.msys_root / 'installed.json' + try: + return json.loads(p.read_text(encoding='utf-8')) + except (OSError, ValueError): + return None + + +def missing_files(msys): + return [r for r in REQUIRED if not (Path(msys) / r).exists()] + + +def is_current(layout, lock): + inst = installed(layout) + return bool(inst) and inst.get('digest') == lock_digest(lock) and not missing_files(layout.msys) + + +def _extract_package(path, dest): + """Unpack one .pkg.tar.zst into dest (the msys64 folder): only its ucrt64/ files, without the documentation. + The archive is read as a stream, member by member, so a 350 MB package never sits in memory whole.""" + with open(path, 'rb') as raw: + zst = raw.read(4) == b'\x28\xb5\x2f\xfd' + raw.seek(0) + if zst: + try: + from compression import zstd # Python 3.14 + fileobj = zstd.ZstdFile(raw) + except ImportError: + import zstandard + fileobj = zstandard.ZstdDecompressor().stream_reader(raw) + else: + fileobj = raw + with tarfile.open(fileobj=fileobj, mode='r|*') as t: + for m in t: + name = m.name.lstrip('./') + if name.startswith('ucrt64/') and not name.startswith(SKIP_SHARE): + t.extract(m, dest, filter='data') + + +def install(layout, lock, progress=None, cancel=None, log=print, version='dev'): + """Download and unpack the lock's packages into toolchain/msys64 (through msys64.new, swapped in at the end). + progress(phase, done, total) with phase 'download' (bytes) or 'unpack' (packages).""" + pkgs = lock['packages'] + layout.pkgs.mkdir(parents=True, exist_ok=True) + need = [p for p in pkgs if not ((layout.pkgs / p['filename']).is_file() + and (layout.pkgs / p['filename']).stat().st_size == p['csize'])] + total = sum(p['csize'] for p in need) + log(f'toolchain: {len(pkgs)} packages, {len(need)} to download ({total / 1e6:.0f} MB)') + done_before = 0 + for p in need: + dest = layout.pkgs / p['filename'] + last_error = None + for mirror in [lock.get('repo', REPO)] + [m for m in MIRRORS if m != lock.get('repo', REPO)]: + try: + net.download(mirror + p['filename'], dest, sha256=p['sha256'], size=p['csize'], cancel=cancel, + progress=(lambda d, t, base=done_before: progress and progress('download', base + d, total)), + version=version) + last_error = None + break + except net.DownloadError as e: + last_error = e + log(f'toolchain: {e}') + if last_error: + raise ToolchainError(f'{p["filename"]} could not be downloaded: {last_error}') + done_before += p['csize'] + for p in pkgs: # files kept from an earlier download are checked too + if net.sha256_file(layout.pkgs / p['filename']) != p['sha256']: + (layout.pkgs / p['filename']).unlink() + raise ToolchainError(f'{p["filename"]} was damaged on disk; run the setup again to download it anew') + staging = layout.toolchain / 'msys64.new' + shutil.rmtree(staging, ignore_errors=True) + staging.mkdir(parents=True) + for i, p in enumerate(pkgs): + if cancel and cancel(): + raise net.Cancelled() + if progress: + progress('unpack', i, len(pkgs)) + _extract_package(layout.pkgs / p['filename'], staging) + if progress: + progress('unpack', len(pkgs), len(pkgs)) + missing = missing_files(staging / 'ucrt64') + if missing: + raise ToolchainError(f'the packages lack {", ".join(missing)}') + (staging / 'installed.json').write_text(json.dumps({ + 'digest': lock_digest(lock), 'packages': [f'{p["name"]} {p["version"]}' for p in pkgs], + 'installed': time.strftime('%Y-%m-%d %H:%M')}, indent=1), encoding='utf-8') + old = layout.toolchain / f'msys64.old-{int(time.time())}' + if layout.msys_root.exists(): + os.replace(layout.msys_root, old) + os.replace(staging, layout.msys_root) + shutil.rmtree(old, ignore_errors=True) + keep = {p['filename'] for p in pkgs} + for f in layout.pkgs.iterdir(): + if f.name not in keep: + f.unlink(missing_ok=True) + log(f'toolchain: installed in {layout.msys}') + + +# ---------------------------------------------------------------- zbic + +def wheel_tag(): + """The wheel platform tags this interpreter loads, best first.""" + v = f'cp{sys.version_info.major}{sys.version_info.minor}' + if os.name == 'nt': + return [f'{v}-{v}-win_amd64'] + return [f'{v}-{v}-manylinux', f'{v}-{v}-musllinux'] + + +def zbic_ok(layout): + """zbic loads from pydeps/ in the interpreter the build runs with (a fresh process: this one may have cached + that pydeps/ did not exist).""" + import subprocess + if not layout.pydeps.is_dir(): + return False + code = 'import sys; sys.path.insert(0, sys.argv[1]); import zbic; assert callable(zbic.decompress)' + flags = {'creationflags': 0x08000000} if os.name == 'nt' else {} # CREATE_NO_WINDOW + r = subprocess.run([layout.python_for_builds(), '-I', '-c', code, str(layout.pydeps)], capture_output=True, + **flags) + return r.returncode == 0 + + +def ensure_zbic(layout, progress=None, cancel=None, log=print, version='dev'): + if zbic_ok(layout): + return + name, ver = ZBIC + info = net.get_json(f'https://pypi.org/pypi/{name}/{ver}/json', version=version) + files = [u for u in info.get('urls', []) if u['filename'].endswith('.whl')] + pick = None + for tag in wheel_tag(): + for u in files: + if tag in u['filename'] and ('x86_64' in u['filename'] or 'amd64' in u['filename']): + pick = u + break + if pick: + break + if not pick: + raise ToolchainError(f'PyPI has no {name} {ver} wheel for Python {sys.version_info.major}.' + f'{sys.version_info.minor} on this system') + dest = layout.downloads / pick['filename'] + net.download(pick['url'], dest, sha256=pick['digests']['sha256'], size=pick.get('size'), cancel=cancel, + progress=(lambda d, t: progress and progress('download', d, t)), version=version) + shutil.rmtree(layout.pydeps, ignore_errors=True) + layout.pydeps.mkdir(parents=True) + with zipfile.ZipFile(dest) as z: + for m in z.infolist(): + target = (layout.pydeps / m.filename).resolve() + if not str(target).startswith(str(layout.pydeps.resolve())): + raise ToolchainError(f'{pick["filename"]}: unsafe path {m.filename}') + z.extractall(layout.pydeps) + dest.unlink(missing_ok=True) + if not zbic_ok(layout): + raise ToolchainError(f'{name} {ver} was unpacked into {layout.pydeps} but does not load') + log(f'zbic: {pick["filename"]}') diff --git a/launcher/ui/__init__.py b/launcher/ui/__init__.py new file mode 100644 index 00000000..66afff19 --- /dev/null +++ b/launcher/ui/__init__.py @@ -0,0 +1 @@ +"""The launcher's window: art.py draws the sprites, app.py lays out the screens and handles input.""" diff --git a/launcher/ui/app.py b/launcher/ui/app.py new file mode 100644 index 00000000..36778003 --- /dev/null +++ b/launcher/ui/app.py @@ -0,0 +1,1297 @@ +"""The launcher's window, laid out like the game's options menu (docs/launcher.md, "Look"): a wooden title band with +the Triforce and the name in the game's Cetra Display, a list of rows on the left (a bronze label cell and a dark value cell each), a white +glowing cursor on the selected row, filigree dividers between groups and a description box on the right that +explains the selected row. + +The window is one Tk canvas at 1000 x 640 logical pixels, scaled to the display. The pictures come from art.py; text, +flat cells and the orange choices are canvas items. Screens: + + first_run the dump, the install folder, three choices and Begin Installation + home Play, Updates, the game folder, Rebuild the Game, Settings + job the steps of an install, update or rebuild, with their progress, and Cancel; after a failure Try Again + and Copy Report + settings the launcher's choices, the folders, the logs and Uninstall + uninstall what to remove, then Uninstall or Cancel + +Mouse, keyboard (arrows, Enter or Space, Esc, F5) and an XInput gamepad (d-pad or stick, A, B, Y) all move the cursor. +""" +import argparse +import os +import shutil +import sys +import time +import tkinter as tk +import tkinter.font as tkfont +from pathlib import Path +from tkinter import filedialog + +from .. import NAME, build, feed, gamefiles, jobs, versions, winsys +from ..context import Context +from . import art + +W, H = 1000, 640 +BAND = (52, 948, 56) # x0, x1, height +PANEL = (112, 60, 580) # x0, top, x1 +ROWS = (140, 552, 96) # x0, x1, top +LABEL_W = 222 # 54 % of the row +GAP = 12 +DESC = (614, 336) # x, width +P = art.PALETTE + + +class Row: + def __init__(self, kind, label='', value='', style='normal', big=False, icon=None, badge=False, desc=None, + on=None, left=None, right=None, get=None, options=None, fraction=0.0, actions=None, check=False, + focus_value=False, height=None, enabled=True): + self.kind, self.label, self.value, self.style = kind, label, value, style + self.big, self.icon, self.badge, self.desc, self.check = big, icon, badge, desc, check + self.on, self.left, self.right, self.get, self.options = on, left, right, get, options + self.fraction, self.actions, self.focus_value, self.enabled = fraction, actions or [], focus_value, enabled + default_h = {'divider': 26, 'action': 54, 'pair': 44}.get(kind, 64 if big else 40) + self.h = height or default_h + self.rect = self.value_rect = None + self.parts = [] # pair: [(rect, label, callback)] + + @property + def focusable(self): + return self.kind != 'divider' + + +def fit(font, text, width): + """text shortened in the middle with an ellipsis so it fits width pixels.""" + if font.measure(text) <= width: + return text + lo, hi = 1, len(text) + while lo < hi: + mid = (lo + hi + 1) // 2 + keep = mid // 2 + cand = text[:keep] + '…' + text[len(text) - (mid - keep):] + if font.measure(cand) <= width: + lo = mid + else: + hi = mid - 1 + keep = lo // 2 + return text[:keep] + '…' + text[len(text) - (lo - keep):] + + +def free_gb(path): + p = Path(path) + while not p.exists() and p.parent != p: + p = p.parent + try: + return shutil.disk_usage(p).free / 1e9 + except OSError: + return 0.0 + + +def drive_of(path): + return Path(path).anchor.rstrip('\\/') or str(Path(path).parent) + + +class App: + def __init__(self, root, ctx, args): + self.root, self.ctx, self.args = root, ctx, args + self.layout, self.settings = ctx.layout, ctx.settings + self.s = self.compute_scale() + self.sprites = {} + self.photos = {} + self.rows, self.focus = [], 0 + self.screen = None + self.job = None + self.job_rev = -1 + self.job_return = 'home' + self.check = None + self.release = None + self.check_error = '' + self.checked_at = 0.0 + self.message = '' + self.dumps = None + self.dump_search = None + self.uninstall_game = False + self.game_proc = None + self.cursor_frames, self.cursor_k = [], 0 + self.last_second = 0 + self.gamepad = winsys.Gamepad() + + root.title(NAME) + root.resizable(False, False) + root.configure(background=P['window']) + self.canvas = tk.Canvas(root, width=self.X(W), height=self.X(H), highlightthickness=0, bd=0, + background=P['window'], cursor='arrow') + self.canvas.pack() + self.set_icon() + self.load_fonts() + self.draw_static() + self.bind_input() + self.start() + self.root.after(80, self.animate) + self.root.after(150, self.tick) + self.root.after(50, self.poll_gamepad) + self.center() + + # ---------------------------------------------------------------- scale, icon, fonts + + def compute_scale(self): + try: + s = self.root.winfo_fpixels('1i') / 96.0 + except tk.TclError: + s = 1.0 + sw, sh = self.root.winfo_screenwidth(), self.root.winfo_screenheight() + s = min(max(s, 1.0), sw * 0.94 / W, sh * 0.88 / H) + return max(0.75, round(s * 4) / 4) + + def X(self, v): + return int(round(v * self.s)) + + def center(self): + self.root.update_idletasks() + w, h = self.X(W), self.X(H) + x = max(0, (self.root.winfo_screenwidth() - w) // 2) + y = max(0, (self.root.winfo_screenheight() - h) // 3) + self.root.geometry(f'{w}x{h}+{x}+{y}') + + def set_icon(self): + try: + self.photos['icon'] = art.photo(self.root, art.triforce(64)) + self.root.iconphoto(True, self.photos['icon']) + if os.name == 'nt': + ico = self.layout.cache / 'nx2navi.ico' + if not ico.is_file(): + ico.write_bytes(art.triforce_ico()) + self.root.iconbitmap(default=str(ico)) + except (tk.TclError, OSError) as e: + self.ctx.log(f'icon: {e}') + + def load_fonts(self): + """The game's Cetra Display when this PC has decoded it from the player's game (data/cache/fonts), else the + system's interface font.""" + fams = {} + for role, (path, family) in gamefiles.cached_fonts(self.layout).items(): + if winsys.register_font(path): + fams[role] = family + available = set(tkfont.families(self.root)) + fallback = next((f for f in ('Segoe UI', 'Source Sans 3', 'Noto Sans', 'DejaVu Sans', 'Helvetica') + if f in available), 'TkDefaultFont') + # Every text is in the game's Cetra Display; only the build log keeps a monospaced face so its lines align. + text = title = fams.get('title') if fams.get('title') in available else fallback + mono = next((f for f in ('Consolas', 'Cascadia Mono', 'DejaVu Sans Mono', 'Courier New') if f in available), + 'TkFixedFont') + self.game_fonts = bool(fams.get('title') and fams['title'] in available) + self.ctx.log(f'fonts: {text}' + (' (the game\'s, from data/cache/fonts)' if self.game_fonts else + ' (the game\'s Cetra Display is not decoded yet)')) + + self.fspec = {} + + def F(family, px, weight='normal'): + font = tkfont.Font(root=self.root, family=family, size=-max(8, int(round(px * self.s))), weight=weight) + self.fspec[id(font)] = (family, max(8, int(round(px * self.s))), weight) + return font + self.fcache = {} + self.f = {'band': F(title, 25), 'big': F(text, 23), 'big_small': F(text, 21), 'label': F(text, 17), + 'value': F(text, 16), 'desc': F(text, 17), 'small': F(text, 15), 'heading': F(title, 21), + 'sub': F(title, 17), 'action': F(text, 18), 'status': F(text, 15), 'mono': F(mono, 11.5), + 'badge': F(text, 15, 'bold'), 'action_big': F(text, 21)} + + def fit_font(self, font, text, width, smallest=0.72): + """font, or the same face a few pixels smaller so text fits width (down to smallest of its size).""" + if font.measure(text) <= width: + return font + family, px, weight = self.fspec[id(font)] + for size in range(px - 1, max(7, int(px * smallest)) - 1, -1): + key = (family, size, weight) + if key not in self.fcache: + self.fcache[key] = tkfont.Font(root=self.root, family=family, size=-size, weight=weight) + if self.fcache[key].measure(text) <= width: + return self.fcache[key] + return self.fcache[(family, max(7, int(px * smallest)), weight)] + + def fonts_from_game(self, game_dir='', dump=''): + """Decode the game's fonts in the background and redraw with them.""" + if self.game_fonts: + return + + def work(): + try: + got = gamefiles.extract_fonts(self.layout, game_dir, dump) + except Exception as e: + self.ctx.log(f'fonts: {e}') + got = {} + if got: + self.root.after(0, self.reload_fonts) + import threading + threading.Thread(target=work, daemon=True).start() + + def reload_fonts(self): + self.load_fonts() + self.draw_static() + self.render() + + # ---------------------------------------------------------------- pictures + + def sprite(self, key, make): + if key not in self.sprites: + self.sprites[key] = art.photo(self.root, make()) + return self.sprites[key] + + def draw_static(self): + c, s = self.canvas, self.s + c.delete('static') + c.create_image(0, 0, anchor='nw', image=self.sprite('backdrop', lambda: art.backdrop(self.X(W), self.X(H), s)), + tags='static') + bx0, bx1, bh = BAND + c.create_image(self.X(bx0), 0, anchor='nw', tags='static', + image=self.sprite('band', lambda: art.band(self.X(bx1) - self.X(bx0), self.X(bh), s))) + px0, ptop, px1 = PANEL + c.create_image(self.X(px0), self.X(ptop), anchor='nw', tags='static', + image=self.sprite('panel', lambda: art.panel(self.X(px1) - self.X(px0), self.X(H - ptop), s))) + tri = self.sprite('triforce', lambda: art.triforce(self.X(44))) + title_w = self.f['band'].measure(NAME) + gap = self.X(12) + total = self.X(44) + gap + title_w + x = self.X(W / 2) - total // 2 + c.create_image(x, self.X(bh / 2), anchor='w', image=tri, tags='static') + tx = x + self.X(44) + gap + c.create_text(tx + 1, self.X(bh / 2) + self.X(2), text=NAME, anchor='w', font=self.f['band'], fill='#1a120a', + tags='static') + c.create_text(tx, self.X(bh / 2), text=NAME, anchor='w', font=self.f['band'], fill=P['band_text'], + tags='static') + c.tag_lower('static') + + # ---------------------------------------------------------------- input + + def bind_input(self): + r = self.root + for key, fn in (('', lambda e: self.move(-1)), ('', lambda e: self.move(1)), + ('', lambda e: self.side(-1)), ('', lambda e: self.side(1)), + ('', lambda e: self.activate()), ('', lambda e: self.activate()), + ('', lambda e: self.activate()), ('', lambda e: self.back()), + ('', lambda e: self.check_now(force=True))): + r.bind(key, fn) + self.canvas.bind('', self.on_motion) + self.canvas.bind('', self.on_click) + r.protocol('WM_DELETE_WINDOW', self.on_close) + + def poll_gamepad(self): + try: + for b in self.gamepad.poll(): + {'up': lambda: self.move(-1), 'down': lambda: self.move(1), 'left': lambda: self.side(-1), + 'right': lambda: self.side(1), 'a': self.activate, 'b': self.back, + 'y': lambda: self.check_now(force=True)}.get(b, lambda: None)() + except Exception as e: + self.ctx.log(f'gamepad: {e}') + self.gamepad.dll = None + self.root.after(50, self.poll_gamepad) + + def focusables(self): + return [i for i, r in enumerate(self.rows) if r.focusable] + + def move(self, d): + idx = self.focusables() + if not idx: + return + pos = idx.index(self.focus) if self.focus in idx else 0 + self.focus = idx[max(0, min(len(idx) - 1, pos + d))] + self.render() + + def focused(self): + return self.rows[self.focus] if 0 <= self.focus < len(self.rows) else None + + def side(self, d): + r = self.focused() + if not r: + return + if r.kind == 'toggle': + r.on(d < 0) # left = On, right = Off + elif r.kind == 'enum': + (r.left if d < 0 else r.right)() + elif r.kind == 'pair': + r.pick = max(0, min(len(r.actions) - 1, getattr(r, 'pick', 0) + d)) + else: + return + self.render() + + def activate(self): + r = self.focused() + if not r or not r.enabled: + return + if r.kind == 'toggle': + r.on(not r.get()) + elif r.kind == 'enum': + r.right() + elif r.kind == 'pair': + r.actions[getattr(r, 'pick', 0)][1]() + return + elif r.on: + r.on() + return + self.render() + + def back(self): + if self.screen == 'settings': + self.show('home') + elif self.screen == 'uninstall': + self.show('settings') + elif self.screen == 'job' and self.job and self.job.result is None: + self.cancel_job() + elif self.screen == 'job': + self.leave_job() + + def hit(self, x, y): + for i, r in enumerate(self.rows): + if r.rect and r.focusable: + x0, y0, x1, y1 = r.rect + if x0 <= x <= x1 and y0 - self.X(GAP / 2) <= y <= y1 + self.X(GAP / 2): + return i + return None + + def on_motion(self, e): + i = self.hit(e.x, e.y) + if i is not None and i != self.focus: + self.focus = i + r = self.rows[i] + if r.kind == 'pair': + r.pick = 0 if e.x < (r.rect[0] + r.rect[2]) / 2 else 1 + self.render() + elif i is not None and self.rows[i].kind == 'pair': + r = self.rows[i] + pick = 0 if e.x < (r.rect[0] + r.rect[2]) / 2 else 1 + if pick != getattr(r, 'pick', 0): + r.pick = pick + self.render() + + def on_click(self, e): + i = self.hit(e.x, e.y) + if i is None: + return + self.focus = i + r = self.rows[i] + if not r.enabled: + self.render() + return + if r.kind == 'toggle' and r.value_rect: + x0, _, x1, _ = r.value_rect + r.on(e.x < (x0 + x1) / 2) + self.render() + elif r.kind == 'enum' and r.value_rect: + x0, _, x1, _ = r.value_rect + (r.left if e.x < x0 + (x1 - x0) / 3 else r.right)() + self.render() + elif r.kind == 'pair': + r.pick = 0 if e.x < (r.rect[0] + r.rect[2]) / 2 else 1 + r.actions[r.pick][1]() + else: + self.activate() + + def on_close(self): + if self.job and self.job.result is None: + self.job.cancel() + self.root.destroy() + + # ---------------------------------------------------------------- screens + + def start(self): + a = self.args + gd = self.settings.get('game_dir') + if a.rolled_back: + self.ctx.log(f'the stub went back from {a.rolled_back} to {self.layout.version}') + for old in self.layout.root.glob('python.old*'): # the Python an update replaced (launcher/native/nx2navi.c) + shutil.rmtree(old, ignore_errors=True) + if gd or self.settings.get('dump'): + self.fonts_from_game(gd, self.settings.get('dump')) + if a.uninstall: + self.show('uninstall') + elif a.continue_update is not None: + self.run_job(jobs.ContinueUpdateJob(self.ctx, gd if gd and build.built_version(gd) else '', + a.continue_update, self.settings.get('dump')), 'home') + elif gd and build.built_version(gd): + self.show('home') + self.after_update_notice() + if self.settings.get('auto_check'): + self.check_now() + else: + self.show('first_run') + self.find_dumps() + + def after_update_notice(self): + if self.args.rolled_back: + self.message = (f'Version {self.args.rolled_back} did not start, so NX2navi went back to ' + f'{versions.display(self.layout.version)}.') + + def show(self, screen, focus=None): + self.screen = screen + self.rows = getattr(self, 'rows_' + screen)() + idx = self.focusables() + self.focus = focus if focus is not None and focus in idx else (idx[0] if idx else 0) + if screen == 'job' and focus is None: + self.focus = self.job_focus() + self.render() + + # ---- first run + + def find_dumps(self): + import threading + extra = [self.args.source] if self.args.source else [] + extra.append(str(self.layout.root.parent)) + + def work(): + try: + found = gamefiles.find_dumps(self.layout, extra) + except Exception as e: + self.ctx.log(f'dump search: {e}') + found = [] + self.root.after(0, lambda: self.dumps_found(found)) + self.dump_search = threading.Thread(target=work, daemon=True) + self.dump_search.start() + + def dumps_found(self, found): + self.dumps = found + if found and not self.settings.get('dump'): + self.settings['dump'] = str(found[0]) + self.settings.save() + self.fonts_from_game('', str(found[0])) + if self.screen == 'first_run': + self.show('first_run', self.focus) + + def game_dir(self): + return self.settings.get('game_dir') or str(self.layout.default_game_dir()) + + def rows_first_run(self): + dump = self.settings.get('dump') + if dump: + dump_value, dump_style, check = Path(dump).name, 'normal', True + where = str(Path(dump).parent) + dump_desc = (f'Your dump: {Path(dump).name}, in {where}. Select this row to choose another .nup file.',) + elif self.dumps is None: + dump_value, dump_style, check = 'Looking for your dump…', 'dim', False + dump_desc = ('Looking for your dump of Ocarina of Time in Downloads, Desktop, Documents and the folder ' + 'the setup ran from.',) + else: + dump_value, dump_style, check = 'Choose your .nup file', 'salmon', False + dump_desc = ('Choose your own dump of Ocarina of Time with the 1.1.1 update, as one .nup file. None was ' + 'found in Downloads, Desktop or Documents.',) + gd = self.game_dir() + need = 30 + free = free_gb(gd) + folder_desc = (f'The game is built into this folder. {free:.0f} GB free on {drive_of(gd)}; the game needs ' + f'about {need} GB. Select this row to choose another folder.',) + ready = bool(dump) + begin_desc = (f'Build Ocarina of Time on this PC from your own dump. It takes 10 to 25 minutes, once, and ' + f'downloads 170 MB of build tools. The game needs about {need} GB on {drive_of(gd)} ' + f'({free:.0f} GB free).',) if ready else \ + ('Choose your dump first: select Game Dump above.',) + return [ + Row('value', 'Game Dump', dump_value, dump_style, big=True, check=check, on=self.choose_dump, + desc=dump_desc), + Row('value', 'Install Folder', gd, big=True, on=self.choose_folder, desc=folder_desc), + Row('divider'), + self.toggle('Desktop Shortcut', 'desktop_shortcut', + 'Put an NX2navi shortcut on the desktop. The Start menu always has one.'), + self.toggle('Check for Updates at Start', 'auto_check', + 'Look for a new version of NX2navi each time the launcher opens. Nothing is installed ' + 'until you choose Update.'), + self.channel_row(), + Row('divider'), + Row('action', 'Begin Installation', on=self.begin_install, desc=begin_desc, enabled=True), + ] + + def toggle(self, label, key, desc): + def on(v): + self.settings[key] = bool(v) + self.settings.save() + if key == 'desktop_shortcut': + winsys.desktop_shortcut(self.layout, bool(v)) + return Row('toggle', label, on=on, get=lambda: bool(self.settings.get(key)), desc=(desc,)) + + def channel_row(self): + def flip(): + self.settings['channel'] = 'preview' if self.settings.get('channel') == 'stable' else 'stable' + self.settings.save() + self.release, self.checked_at = None, 0 + return Row('enum', 'Update Channel', options=('Stable', 'Preview'), + get=lambda: 'Preview' if self.settings.get('channel') == 'preview' else 'Stable', + left=flip, right=flip, + desc=('Stable gets each finished release. Preview also gets test builds before they are ' + 'released, which may have new problems.',)) + + def choose_dump(self): + start = str(Path(self.settings.get('dump')).parent) if self.settings.get('dump') else str(Path.home()) + path = filedialog.askopenfilename(parent=self.root, title='Choose your Ocarina of Time dump', + initialdir=start, filetypes=[('Game dump', '*.nup'), ('All files', '*.*')]) + if not path: + return + if not gamefiles.is_game_nup(self.layout, path): + self.message = f'{Path(path).name} is not a dump of Ocarina of Time.' + self.render() + return + self.settings['dump'] = path + self.settings.save() + self.message = '' + self.fonts_from_game('', path) + self.show(self.screen, self.focus) + + def choose_folder(self): + path = filedialog.askdirectory(parent=self.root, title='Choose the folder for the game', + initialdir=str(Path(self.game_dir()).parent)) + if not path: + return + p = Path(path) + if p.exists() and any(p.iterdir()) and not (p / 'build-cache').is_dir() and p.name.lower() != 'nx2navi': + p = p / 'NX2navi' + self.settings['game_dir'] = str(p) + self.settings.save() + self.show(self.screen, self.focus) + + def begin_install(self): + dump = self.settings.get('dump') + if not dump: + self.focus = 0 + self.choose_dump() + return + gd = self.game_dir() + if free_gb(gd) < 31 and not (Path(gd) / 'romfs' / 'romfs.bin').is_file(): + self.message = f'{drive_of(gd)} has {free_gb(gd):.0f} GB free; the game needs about 30 GB.' + self.render() + return + self.settings['game_dir'] = gd + self.settings.save() + if self.settings.get('desktop_shortcut'): + winsys.desktop_shortcut(self.layout, True) + self.run_job(jobs.InstallJob(self.ctx, dump, gd), 'home', back='first_run') + + # ---- home + + def rows_home(self): + gd = self.settings.get('game_dir') + built = build.built_version(gd) + mine = versions.display(self.layout.version) + play_value = 'Ocarina of Time' + if self.release and jobs.update_available(self.ctx, self.release): + nv = versions.display(self.release.version) + upd = Row('value', 'Updates', f'Version {nv}', 'orange_arrow', big=True, icon='refresh', badge=True, + on=self.start_update, focus_value=True, desc=self.update_desc) + elif self.check and self.check.result is None: + upd = Row('value', 'Updates', 'Checking…', 'dim', big=True, icon='refresh', on=lambda: None, + desc=('Looking for a new version of NX2navi.',)) + elif self.check_error: + upd = Row('value', 'Updates', 'Could not check', 'dim', big=True, icon='refresh', + on=lambda: self.check_now(force=True), + desc=(f'The update check failed: {self.check_error}', 'Select this row to try again.')) + elif self.checked_at: + upd = Row('value', 'Updates', f'Up to Date · {mine}', 'normal', big=True, icon='refresh', check=True, + on=lambda: self.check_now(force=True), + desc=(f'{mine} is the newest version. Checked {self.ago(self.checked_at)}.', + 'Select this row to check again.')) + else: + upd = Row('value', 'Updates', f'Version {mine}', 'normal', big=True, icon='refresh', + on=lambda: self.check_now(force=True), + desc=(f'You have NX2navi {mine}. Select this row to look for a new version.',)) + rebuild_value = f'Built with {versions.display(built)}' if built else 'Not built' + rebuild_style = 'normal' if versions.display(built) == mine else 'salmon' + play_desc = ('Start Ocarina of Time. The launcher steps aside while you play and comes back when you quit.',) + if built and versions.display(built) != mine: + play_desc += (f'The game was built with {versions.display(built)}. Rebuild the Game brings it to {mine}.',) + if self.message: + play_desc = (self.message,) + play_desc + return [ + Row('value', 'Play', play_value, big=True, icon='play', on=self.play, desc=play_desc), + upd, + Row('divider'), + Row('value', 'Game Folder', gd, on=lambda: self.open(gd), + desc=('Open the game folder. Your saves are in its profile folder, the game\'s settings in ' + 'nx2navi.ini and its logs in logs.',)), + Row('value', 'Rebuild the Game', rebuild_value, rebuild_style, on=self.start_rebuild, + desc=('Build the game again from the files of the last build. Only what changed is compiled. Use it ' + 'after changing build tools, or if the game stopped starting.',)), + Row('value', 'Settings', '', 'arrow', on=lambda: self.show('settings'), + desc=('Updates, the game folder, the logs and uninstalling.',)), + ] + + def update_desc(self): + r = self.release + nv, mine = versions.display(r.version), versions.display(self.layout.version) + notes = feed.plain_notes(r.notes, 6) + return {'title': f'Version {nv} Is Ready', + 'text': [f'You have {mine}. Updating downloads the new version and rebuilds the game in a few ' + f'minutes. Your saves and settings stay as they are.'], + 'divider': bool(notes), 'subtitle': "What's New" if notes else '', 'bullets': notes} + + def ago(self, t): + d = time.time() - t + if d < 90: + return 'just now' + if d < 3600: + return f'{int(d // 60)} minutes ago' + return time.strftime('at %H:%M', time.localtime(t)) + + def check_now(self, force=False): + if self.check and self.check.result is None: + return + if not force and time.time() - self.settings.get('last_check', 0) < 600 and self.release: + return + self.check_error = '' + self.check = jobs.CheckJob(self.ctx).start() + if self.screen == 'home': + self.show('home', self.focus) + + def check_done(self): + c = self.check + if c.result == 'ok': + self.release = c.release + self.checked_at = time.time() + self.settings['last_check'] = self.checked_at + if self.release: + self.settings['last_seen'] = self.release.version + self.settings.save() + self.ctx.log(f'update check: {self.release.version if self.release else "no release"}') + else: + self.check_error = c.error or 'cancelled' + self.ctx.log(f'update check: {self.check_error}') + if self.screen == 'home': + self.show('home', self.focus) + + def play(self): + gd = self.settings.get('game_dir') + exe = Path(gd) / 'ocarina.exe' + if not exe.is_file(): + self.message = 'The game is not built yet.' + self.render() + return + if self.game_proc and self.game_proc.poll() is None: + return + try: + self.game_proc = winsys.start_game(exe) + except OSError as e: + self.message = f'The game could not start: {e}' + self.render() + return + self.ctx.log(f'play: {exe}') + if self.settings.get('hide_while_playing'): + self.root.withdraw() + self.root.after(1000, self.watch_game) + + def watch_game(self): + if self.game_proc and self.game_proc.poll() is None: + self.root.after(1000, self.watch_game) + return + code = self.game_proc.returncode if self.game_proc else 0 + self.game_proc = None + self.root.deiconify() + self.root.lift() + if code not in (0, None): + self.message = f'The game ended with code {code}. Its log is logs\\nx2navi.log in the game folder.' + self.show('home', self.focus) + + def open(self, path): + try: + winsys.open_path(path) + except OSError as e: + self.message = str(e) + self.render() + + def start_update(self): + if not self.release: + return + gd = self.settings.get('game_dir') + if gd and build.game_running(gd): + self.message = 'Close the game first: the update rebuilds it.' + self.show('home', self.focus) + return + self.run_job(jobs.UpdateDownloadJob(self.ctx, self.release), 'home') + + def start_rebuild(self): + gd = self.settings.get('game_dir') + self.run_job(jobs.RebuildJob(self.ctx, gd, self.settings.get('dump')), 'home') + + # ---- settings + + def rows_settings(self): + gd = self.settings.get('game_dir') + dump = self.settings.get('dump') + return [ + self.toggle('Check for Updates at Start', 'auto_check', + 'Look for a new version of NX2navi each time the launcher opens. Nothing is installed ' + 'until you choose Update.'), + self.toggle('Hide Launcher While Playing', 'hide_while_playing', + 'Hide this window while the game runs; it comes back when you quit the game.'), + self.toggle('Keep Build Cache', 'keep_cache', + 'Keep the translated code and compiled files (about 5 GB) in the game folder, so an update ' + 'compiles only what changed. Off: they are deleted after each build.'), + self.channel_row(), + Row('divider'), + Row('value', 'Game Folder', gd or 'Not chosen', on=lambda: gd and self.open(gd), + desc=('Open the game folder: saves in profile, the game\'s settings in nx2navi.ini, its logs in ' + 'logs.',)), + Row('value', 'Game Dump', Path(dump).name if dump else 'Not chosen', on=self.choose_dump, + desc=(f'The dump the game was built from: {dump}.' if dump else 'No dump chosen yet.', + 'Select this row to choose another .nup file.')), + Row('divider'), + Row('value', 'Open Logs Folder', '', 'arrow', on=lambda: self.open(self.layout.logs), + desc=('The launcher\'s logs. Attach launcher.log when you report a problem with installing or ' + 'updating; never send game files.',)), + Row('value', 'Uninstall NX2navi', 'Keeps your saves', 'salmon', on=lambda: self.show('uninstall'), + desc=('Remove the launcher and its build tools. You choose on the next screen whether the built ' + 'game goes too.',)), + Row('action', 'Back', on=lambda: self.show('home'), height=40, desc=('Back to the main menu.',)), + ] + + # ---- uninstall + + def rows_uninstall(self): + def on(v): + self.uninstall_game = bool(v) + gd = self.settings.get('game_dir') + return [ + Row('toggle', 'Remove the Built Game', on=on, get=lambda: self.uninstall_game, + desc=(f'On: the game folder ({gd or "none"}) is deleted too, saves included.', + 'Off: the game folder and your saves stay where they are.')), + Row('divider'), + Row('pair', actions=[('Uninstall', self.do_uninstall), ('Cancel', lambda: self.show('settings'))], + desc=('Uninstall removes NX2navi, its build tools and its shortcuts. Your dump is never touched.',)), + ] + + def do_uninstall(self): + gd = self.settings.get('game_dir') if self.uninstall_game else '' + if gd and build.game_running(gd): + self.message = 'Close the game first.' + self.render() + return + self.ctx.log(f'uninstall (game folder too: {bool(gd)})') + winsys.uninstall(self.layout, gd, self.ctx.log) + self.root.destroy() + + # ---- jobs + + def run_job(self, job, return_to, back=None): + self.job, self.job_return, self.job_back = job, return_to, back or return_to + self.job_rev = -1 + self.message = '' + winsys.keep_awake(True) + job.start() + self.show('job') + + def rows_job(self): + j = self.job + rows = [] + for st in j.steps: + if st.state == jobs.RUNNING: + rows.append(Row('progress', st.label, fraction=st.fraction, value=f'{int(st.fraction * 100)}%', + focus_value=True, desc=None)) + else: + style = {jobs.DONE: 'normal', jobs.SKIPPED: 'dim', jobs.PENDING: 'dim', jobs.FAILED: 'salmon'}[st.state] + rows.append(Row('value', st.label, st.value, style, check=st.state == jobs.DONE, desc=None, + focus_value=st.state == jobs.FAILED)) + if st.id in ('romfs', 'verify') and st is not j.steps[-1]: + rows.append(Row('divider')) + if j.result is None: + label = {'install': 'Cancel Installation', 'update': 'Cancel Update'}.get(j.kind, 'Cancel') + rows.append(Row('action', label, on=self.cancel_job, height=44)) + elif j.result == 'failed': + rows.append(Row('pair', actions=[('Try Again', self.retry_job), ('Copy Report', self.copy_report)])) + for r in rows: + r.desc = self.job_desc + return rows + + def job_focus(self): + """The cursor sits on the running step, or on Try Again after a failure.""" + for i, r in enumerate(self.rows): + if self.job.result == 'failed' and r.kind == 'pair': + return i + if r.kind == 'progress': + return i + return self.focusables()[-1] if self.focusables() else 0 + + def job_desc(self): + j = self.job + cur = j.current() + lines = [ln for ln in j.lines if ln.strip() and not ln.startswith('@@')][-6:] + if j.result == 'failed': + return {'title': 'The Build Stopped' if j.kind != 'update' or cur.id not in ('download', 'verify') + else 'The Update Stopped', + 'text': [self.friendly(j.error), 'Nothing in your game folder was changed.' if cur.id not in + ('download', 'verify', 'tools', 'tools_update') else ''], + 'log': lines, 'note': 'Try again first. If it stops again, use Copy Report and send it with your ' + 'issue. Never send game files.'} + if j.result == 'cancelled': + return {'title': 'Cancelled', 'text': ['Nothing was changed that a new start cannot finish.']} + if j.result == 'ok': + return {'title': 'Done', 'text': [f'Finished in {jobs.mmss(j.elapsed())}.']} + text = [cur.desc + (f' {cur.detail}.' if cur.detail else '')] + eta = j.eta() + return {'title': self.step_title(cur), 'text': text, 'progress': j.overall(), + 'progress_label': ('Overall', self.eta_text(eta)), + 'log': lines, + 'note': 'Keep using your PC if you like. It stays awake until the build is done.' + if j.kind != 'update' or cur.id not in ('download', 'verify') else + 'Your saves and settings stay as they are.'} + + @staticmethod + def eta_text(eta): + if not eta: + return '' + minutes = max(1, int(eta // 60 + 1)) + return 'about 1 minute left' if minutes == 1 else f'about {minutes} minutes left' + + def step_title(self, st): + return {'tools': 'Gathering Build Tools', 'tools_update': 'Updating Build Tools', 'check': 'Inspecting Your Game', + 'exefs': 'Unpacking the Program', 'romfs': 'Packing the Game Data', 'lift': 'Translating the Code', + 'compile': 'Compiling', 'compile_changed': 'Compiling Changed Files', 'shaders': 'Preparing the Shaders', + 'ready': 'Ready to Play', + 'download': 'Downloading', 'verify': 'Verifying', 'swap': 'Swapping In the New Build'}.get(st.id, + st.label) + + @staticmethod + def friendly(err): + e = (err or 'unknown error').strip() + return e[0].upper() + e[1:] + ('' if e.endswith('.') else '.') + + def cancel_job(self): + if self.job and self.job.result is None: + self.job.cancel() + + def retry_job(self): + j = self.job + if isinstance(j, jobs.InstallJob): + self.run_job(jobs.InstallJob(self.ctx, j.dump, j.game_dir), self.job_return, self.job_back) + elif isinstance(j, jobs.ContinueUpdateJob): + self.run_job(jobs.ContinueUpdateJob(self.ctx, j.game_dir, j.old, j.dump), self.job_return) + elif isinstance(j, jobs.RebuildJob): + self.run_job(jobs.RebuildJob(self.ctx, j.game_dir, j.dump), self.job_return) + elif isinstance(j, jobs.UpdateDownloadJob): + self.run_job(jobs.UpdateDownloadJob(self.ctx, j.release), self.job_return) + + def copy_report(self): + j = self.job + report = [f'NX2navi {self.layout.version} ({"checkout" if self.layout.dev else "release"}), {sys.platform}', + f'Job: {j.kind}, step: {j.current().label}', f'Error: {j.error}', '', 'Last lines:'] + report += [ln for ln in list(j.lines)[-60:]] + self.root.clipboard_clear() + self.root.clipboard_append('\n'.join(report)) + self.message = 'Report copied.' + self.render() + + def leave_job(self): + j = self.job + if j.result == 'ok': + self.show(self.job_return) + elif j.result == 'cancelled': + self.show(self.job_back) + else: + self.show(self.job_back) + + def job_finished(self): + j = self.job + winsys.keep_awake(False) + self.ctx.log(f'job {j.kind}: {j.result} {j.error}') + if j.result == 'ok': + if j.after == 'restart': + if winsys.restart(self.layout, ['--continue-update', versions.display(self.layout.version)]): + self.root.after(200, self.root.destroy) + return + self.message = 'The update is ready: start NX2navi again to finish it.' + elif j.after == 'staged': + self.message = 'The update was downloaded; a checkout keeps running from the repository.' + if not self.settings.get('keep_cache') and j.kind in ('install', 'update', 'rebuild'): + self.drop_build_cache() + if j.kind == 'install': + self.message = 'The game is built. Select Play.' + elif j.kind in ('update', 'rebuild'): + self.message = 'The game is rebuilt and up to date.' if j.kind == 'rebuild' else \ + f'NX2navi is now {versions.display(self.layout.version)}.' + self.release = None + self.reload_fonts_if_new() + self.show(self.job_return) + if j.kind == 'install' and self.settings.get('auto_check'): + self.check_now() + elif j.result == 'cancelled': + self.show(self.job_back) + else: + self.show('job') + + def drop_build_cache(self): + """Keep Build Cache off: delete the lifted C and the objects (about 5 GB) but keep state.json, which an + update needs to rebuild without the dump; the next build then lifts and compiles everything again.""" + gd = self.settings.get('game_dir') + if not gd: + return + cache = Path(gd) / 'build-cache' + for name in ('lift', 'obj', 'gen'): + shutil.rmtree(cache / name, ignore_errors=True) + for name in ('link.rsp', 'link.key'): + (cache / name).unlink(missing_ok=True) + + def reload_fonts_if_new(self): + if not self.game_fonts and gamefiles.cached_fonts(self.layout): + self.reload_fonts() + + # ---------------------------------------------------------------- timers + + def tick(self): + try: + if self.check and self.check.result is not None and not getattr(self.check, 'seen', False): + self.check.seen = True + self.check_done() + if self.screen == 'job' and self.job: + if self.job.rev != self.job_rev: + self.job_rev = self.job.rev + if self.job.result is not None: + self.job_finished() + else: + self.rows = self.rows_job() + self.focus = self.job_focus() + self.render() + elif int(time.time()) != self.last_second and self.job.result is None: + self.render() + self.last_second = int(time.time()) + except Exception as e: # a drawing error must not stop the timer + self.ctx.log(f'tick: {e!r}') + self.root.after(150, self.tick) + + def animate(self): + if self.cursor_frames: + self.cursor_k = (self.cursor_k + 1) % len(self.cursor_frames) + self.canvas.itemconfigure('cursor', image=self.cursor_frames[self.cursor_k]) + self.root.after(250, self.animate) + + # ---------------------------------------------------------------- drawing + + def render(self): + c = self.canvas + c.delete('dyn') + if self.screen == 'job' and self.job: + if self.job.result is None or self.rows == []: + self.rows = self.rows_job() + if not (0 <= self.focus < len(self.rows)) or not self.rows[self.focus].focusable: + self.focus = self.job_focus() + x0, x1, y = ROWS + for i, r in enumerate(self.rows): + self.draw_row(r, self.X(x0), self.X(y), self.X(x1), self.X(y + r.h)) + y += r.h + GAP + self.draw_status() + self.draw_cursor() + self.draw_desc() + + def draw_status(self): + if self.screen != 'job' or not self.job: + return + j = self.job + steps = [s for s in j.steps] + cur = j.current() + n = steps.index(cur) + 1 if cur in steps else len(steps) + left = f'Stopped at step {n} of {len(steps)}' if j.result == 'failed' else f'Step {n} of {len(steps)}' + if self.message: + left = self.message + y = self.X(H - 24) + self.canvas.create_text(self.X(PANEL[0] + 30), y, text=left, anchor='sw', font=self.f['status'], + fill=P['status'], tags='dyn') + self.canvas.create_text(self.X(PANEL[2] - 30), y, text=f'Elapsed {jobs.mmss(j.elapsed())}', anchor='se', + font=self.f['status'], fill=P['status'], tags='dyn') + + def text(self, x, y, text, font, fill, anchor='w', shadow=True, width=0): + c = self.canvas + if shadow: + c.create_text(x + 1, y + 1, text=text, font=font, fill='#2a1e10', anchor=anchor, tags='dyn', + width=width) + return c.create_text(x, y, text=text, font=font, fill=fill, anchor=anchor, tags='dyn', width=width) + + def draw_row(self, r, x0, y0, x1, y1): + c, s, X = self.canvas, self.s, self.X + r.rect = (x0, y0, x1, y1) + if r.kind == 'divider': + img = self.sprite(('filigree', x1 - x0), lambda: art.filigree(x1 - x0, s)[0]) + c.create_image(x0, (y0 + y1) // 2, anchor='w', image=img, tags='dyn') + return + if r.kind in ('action', 'pair'): + parts = [(r.label, r.on)] if r.kind == 'action' else r.actions + gap = X(GAP) + w = (x1 - x0 - gap * (len(parts) - 1)) // len(parts) + r.parts = [] + for k, (label, cb) in enumerate(parts): + px0 = x0 + k * (w + gap) + px1 = px0 + w if k < len(parts) - 1 else x1 + img = self.sprite(('label', px1 - px0, y1 - y0), lambda a=px1 - px0, b=y1 - y0: art.label_cell(a, b, s)) + ring = max(1, int(round(s))) + c.create_image(px0 - ring, y0 - ring, anchor='nw', image=img, tags='dyn') + font = self.fit_font(self.f['action_big'] if r.h >= 50 else self.f['action'], label, + px1 - px0 - X(16)) + self.text((px0 + px1) // 2, (y0 + y1) // 2, label, font, P['text_bright'], anchor='center') + r.parts.append(((px0, y0, px1, y1), label, cb)) + r.value_rect = None + return + lw = X(LABEL_W) + lx1 = x0 + lw + img = self.sprite(('label', lw, y1 - y0), lambda: art.label_cell(lw, y1 - y0, s)) + ring = max(1, int(round(s))) + c.create_image(x0 - ring, y0 - ring, anchor='nw', image=img, tags='dyn') + tx = x0 + X(16) + if r.icon: + size = X(32) + ic = self.sprite(('icon', r.icon, size), lambda: art.framed_icon(size, s, r.icon)) + c.create_image(tx, (y0 + y1) // 2, anchor='w', image=ic, tags='dyn') + tx += size + X(14) + font = (self.f['big'] if r.icon else self.f['big_small']) if r.big else self.f['label'] + font = self.fit_font(font, r.label, lx1 - tx - X(10) - (X(32) if r.badge else 0)) + tid = self.text(tx, (y0 + y1) // 2, r.label, font, P['text_bright'] if r.big else P['text']) + if r.badge: + bx = self.canvas.bbox(tid)[2] + X(10) + size = X(22) + b = self.sprite(('badge', size), lambda: art.badge(size, s)) + c.create_image(bx, (y0 + y1) // 2, anchor='w', image=b, tags='dyn') + c.create_text(bx + (size + 4) // 2, (y0 + y1) // 2, text='!', font=self.f['badge'], fill='#3a2208', + tags='dyn') + vx0 = lx1 + r.value_rect = (vx0, y0, x1, y1) + c.create_rectangle(vx0, y0, x1 - 1, y1 - 1, fill=P['value'], outline=P['value_border'], width=1, tags='dyn') + pad = X(3) if not r.big else X(4) + ix0, iy0, ix1, iy1 = vx0 + pad, y0 + pad, x1 - pad, y1 - pad + cy = (y0 + y1) // 2 + cx = (vx0 + x1) // 2 + vfont = self.f['label'] if r.kind in ('toggle', 'enum', 'progress') or r.style == 'orange_arrow' \ + else self.f['value'] + if r.kind == 'toggle': + on = bool(r.get()) + mid = (ix0 + ix1) // 2 + for k, (label, sel) in enumerate((('On', on), ('Off', not on))): + sx0, sx1 = (ix0, mid) if k == 0 else (mid, ix1) + if sel: + self.orange(sx0, iy0, sx1, iy1) + self.canvas.create_text((sx0 + sx1) // 2, cy, text=label, font=vfont, + fill=P['on_text'] if sel else P['off'], tags='dyn') + elif r.kind == 'enum': + self.orange(ix0, iy0, ix1, iy1) + cw, ch = X(9), X(12) + for left in (True, False): + img = self.sprite(('chev', cw, ch, left, 'on'), lambda lf=left: art.chevron(cw, ch, P['on_text'], lf)) + c.create_image(ix0 + X(9) if left else ix1 - X(9), cy, anchor='w' if left else 'e', image=img, + tags='dyn') + self.canvas.create_text(cx, cy, text=r.get(), font=vfont, fill=P['on_text'], tags='dyn') + elif r.kind == 'progress': + fw = int((ix1 - ix0) * max(0.0, min(1.0, r.fraction))) + if fw > 2: + self.orange(ix0, iy0, ix0 + fw, iy1) + self.canvas.create_text(cx + 1, cy + 1, text=r.value, font=vfont, fill='#1a120a', tags='dyn') + self.canvas.create_text(cx, cy, text=r.value, font=vfont, fill=P['on_text'], tags='dyn') + elif r.style == 'orange_arrow': + self.orange(ix0, iy0, ix1, iy1) + cw, ch = X(9), X(12) + label_w = vfont.measure(r.value) + total = label_w + X(10) + cw + sx = cx - total // 2 + self.canvas.create_text(sx, cy, text=r.value, font=vfont, fill=P['on_text'], anchor='w', tags='dyn') + img = self.sprite(('chev', cw, ch, False, 'on'), lambda: art.chevron(cw, ch, P['on_text'])) + c.create_image(sx + label_w + X(10), cy, anchor='w', image=img, tags='dyn') + elif r.style == 'arrow': + cw, ch = X(9), X(12) + img = self.sprite(('chev', cw, ch, False, 'dim'), lambda: art.chevron(cw, ch, '#d9ccae')) + c.create_image(x1 - X(16), cy, anchor='e', image=img, tags='dyn') + else: + color = {'normal': P['value_text'], 'dim': P['dim'], 'salmon': P['salmon']}.get(r.style, P['value_text']) + avail = (x1 - vx0) - X(20) - (X(21) if r.check else 0) + vfont = self.fit_font(vfont, r.value, avail, 0.8) + label = fit(vfont, r.value, avail) + tw = vfont.measure(label) + sx = cx - (tw + (X(21) if r.check else 0)) // 2 + if r.check: + size = X(14) + ck = self.sprite(('check', size), lambda: art.check_mark(size, s)) + c.create_image(sx, cy, anchor='w', image=ck, tags='dyn') + sx += X(21) + self.canvas.create_text(sx, cy, text=label, font=vfont, fill=color, anchor='w', tags='dyn') + + def orange(self, x0, y0, x1, y1, tags=('dyn',)): + c = self.canvas + c.create_rectangle(x0, y0, x1 - 1, y1 - 1, fill=P['orange'], outline=P['orange_dark'], width=1, tags=tags) + if x1 - x0 > 4: + c.create_rectangle(x0 + 1, y0 + 1, x1 - 2, y1 - 2, outline=P['orange_light'], width=1, tags=tags) + + def draw_cursor(self): + r = self.focused() + self.cursor_frames = [] + if not r or not r.rect: + return + if r.kind == 'pair' and r.parts: + box = r.parts[getattr(r, 'pick', 0)][0] + elif (r.kind in ('toggle', 'enum', 'progress') or r.focus_value) and r.value_rect: + box = r.value_rect + else: + box = r.rect + x0, y0, x1, y1 = box + frames, mg = self.glow(x1 - x0, y1 - y0) + self.cursor_frames = frames + self.cursor_k %= len(frames) + self.canvas.create_image(x0 - mg, y0 - mg, anchor='nw', image=frames[self.cursor_k], tags=('dyn', 'cursor')) + + def glow(self, w, h): + key = ('glow', w, h) + if key not in self.sprites: + frames, mg = art.glow_frames(w, h, self.s) + self.sprites[key] = ([art.photo(self.root, f) for f in frames], mg) + return self.sprites[key] + + def draw_desc(self): + r = self.focused() + d = r.desc if r else None + if callable(d): + d = d() + if not d: + if self.message and self.screen != 'job': + d = (self.message,) + else: + return + if isinstance(d, tuple): + d = {'text': [t for t in d if t]} + elif self.message and self.screen in ('first_run', 'settings', 'uninstall'): + d = dict(d) + d['text'] = [self.message] + list(d.get('text', [])) + c, X = self.canvas, self.X + dx, dw = DESC + x0 = X(dx) + w = X(dw) + padx, pady = X(26), X(22) + inner = w - 2 * padx + items = [] + y = 0 + + def add(kind, *args): + items.append((kind, args)) + + if d.get('title'): + add('text', d['title'], self.f['heading'], P['text_bright'], X(14)) + for t in d.get('text', []): + if t: + add('text', t, self.f['desc'], P['desc_text'], X(12)) + if d.get('progress') is not None: + add('progress', d['progress'], d.get('progress_label', ('', ''))) + if d.get('divider'): + add('filigree') + if d.get('subtitle'): + add('text', d['subtitle'], self.f['sub'], P['gold'], X(8)) + for b in d.get('bullets', []): + add('bullet', b) + if d.get('log'): + add('log', d['log']) + if d.get('note'): + add('text', d['note'], self.f['small'], P['muted'], X(4)) + # measure: draw at a provisional top, then move everything to the final top + top0 = X(112) + y = top0 + pady + made = [] + for kind, args in items: + if kind == 'text': + text, font, fill, after = args + tid = c.create_text(x0 + padx, y, text=text, font=font, fill=fill, anchor='nw', width=inner, + tags=('dyn', 'desc')) + bb = c.bbox(tid) + y = bb[3] + after + made.append(tid) + elif kind == 'progress': + frac, (left, right) = args + c.create_text(x0 + padx, y, text=left, font=self.f['small'], fill=P['muted'], anchor='nw', + tags=('dyn', 'desc')) + c.create_text(x0 + w - padx, y, text=right, font=self.f['small'], fill=P['muted'], anchor='ne', + tags=('dyn', 'desc')) + y += self.f['small'].metrics('linespace') + X(6) + h = X(22) + c.create_rectangle(x0 + padx, y, x0 + w - padx - 1, y + h - 1, fill=P['value'], + outline=P['value_border'], tags=('dyn', 'desc')) + fw = int((inner - X(6)) * frac) + if fw > 2: + self.orange(x0 + padx + X(3), y + X(3), x0 + padx + X(3) + fw, y + h - X(3), ('dyn', 'desc')) + y += h + X(14) + elif kind == 'filigree': + img = self.sprite(('filigree', inner), lambda: art.filigree(inner, self.s)[0]) + c.create_image(x0 + padx, y + X(9), anchor='w', image=img, tags=('dyn', 'desc')) + y += X(22) + elif kind == 'bullet': + (text,) = args + c.create_rectangle(x0 + padx + X(1), y + X(8), x0 + padx + X(6), y + X(13), fill='#c99a45', + outline='', tags=('dyn', 'desc')) + tid = c.create_text(x0 + padx + X(16), y, text=text, font=self.f['value'], fill=P['desc_text'], + anchor='nw', width=inner - X(16), tags=('dyn', 'desc')) + y = c.bbox(tid)[3] + X(8) + elif kind == 'log': + (lines,) = args + lh = self.f['mono'].metrics('linespace') + shown = [fit(self.f['mono'], ln, inner - X(24)) for ln in lines[-6:]] + h = lh * max(len(shown), 1) + X(20) + c.create_rectangle(x0 + padx, y, x0 + w - padx - 1, y + h - 1, fill=P['value'], + outline=P['value_border'], tags=('dyn', 'desc')) + ty = y + X(10) + for ln in shown: + color = P['salmon'] if ln.lower().startswith(('error', 'stopped')) else P['log_text'] + c.create_text(x0 + padx + X(12), ty, text=ln, font=self.f['mono'], fill=color, anchor='nw', + tags=('dyn', 'desc')) + ty += lh + y += h + X(14) + height = y - top0 + pady - X(4) + top = X(236) if height < X(200) else X(112) + top = min(top, X(H - 30) - height) if top + height > X(H - 30) else top + dy = top - top0 + if dy: + c.move('desc', 0, dy) + hb = max(X(60), height) + key = ('desc', w, hb // 4) + if key not in self.sprites: + img, pad = art.desc_box(w, hb, self.s) + self.sprites[key] = (art.photo(self.root, img), pad) + pic, pad = self.sprites[key] + bid = c.create_image(x0 - pad, top - pad, anchor='nw', image=pic, tags='dyn') + c.tag_lower(bid, 'desc') + + +def parse_args(argv): + ap = argparse.ArgumentParser(prog='NX2navi') + ap.add_argument('--from', dest='source', help='a folder to look in for the dump first (the setup\'s folder)') + ap.add_argument('--continue-update', metavar='OLD', help='finish an update after the restart (from OLD)') + ap.add_argument('--rolled-back', metavar='VERSION', help='the stub went back from VERSION') + ap.add_argument('--uninstall', action='store_true') + ap.add_argument('--root', help='the launcher folder (tests)') + ap.add_argument('--selftest', action='store_true', help='draw every screen and close') + ap.add_argument('--shots', help='with --selftest: save a PNG of each screen into this folder') + a, _unknown = ap.parse_known_args(argv) + return a + + +def single_instance(layout, wait): + """An exclusive lock on data/launcher.lock for this process's life, or None when another launcher holds it after + wait seconds (the one an update restart replaces may still be closing).""" + path = layout.data / 'launcher.lock' + deadline = time.time() + wait + while True: + try: + f = open(path, 'a+') + try: + if os.name == 'nt': + import msvcrt + msvcrt.locking(f.fileno(), msvcrt.LK_NBLCK, 1) + else: + import fcntl + fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB) + return f + except OSError: + f.close() + raise + except OSError: + if time.time() >= deadline: + return None + time.sleep(0.25) + + +def main(argv=None): + a = parse_args(sys.argv[1:] if argv is None else argv) + winsys.dpi_aware() + winsys.app_id() + from ..paths import Layout + ctx = Context(Layout(root=a.root) if a.root else None) + lock = None + if not a.selftest: + lock = single_instance(ctx.layout, 20 if (a.continue_update or a.rolled_back) else 2) + if lock is None: + ctx.log('another launcher is running') + return 0 + root = tk.Tk(className='NX2navi') + app = App(root, ctx, a) + if a.selftest: + from .selftest import run + root.after(300, lambda: run(app, a.shots)) + root.mainloop() + return 0 diff --git a/launcher/ui/art.py b/launcher/ui/art.py new file mode 100644 index 00000000..9a7ca8c8 --- /dev/null +++ b/launcher/ui/art.py @@ -0,0 +1,538 @@ +"""The window's pictures, drawn with numpy into RGBA arrays and handed to Tk as PNG data. + +Everything is drawn here from shapes and colours: the misty green backdrop, the wooden title band, the bronze label +cells with their mottled texture, the description box, the white glowing cursor, the filigree divider, the Triforce +and the small icons. Tk's canvas draws lines and polygons without antialiasing on Windows, so every curved or soft +shape is a picture; flat rectangles and text stay canvas items. + +Colours follow the game's options menu (docs/launcher.md, "Look"). Sizes are in logical pixels times the display +scale s. +""" +import base64 +import re +import struct +import zlib + +import numpy as np + +# ---------------------------------------------------------------- colours + +PALETTE = { + 'window': '#06180d', + 'panel': '#3a2b22', + 'label_top': '#7a5c32', 'label_mid': '#775930', 'label_bottom': '#72552d', 'label_border': '#6a4f29', + 'ring': '#2a2019', + 'value': '#2f2d22', 'value_border': '#45402f', + 'orange': '#d69127', 'orange_light': '#e3a234', 'orange_dark': '#8f5f17', + 'desc': '#403024', 'desc_border': '#4f3f30', + 'text': '#e9dcc0', 'text_bright': '#f1e6cc', 'value_text': '#e3d5b6', 'desc_text': '#e9dfc8', + 'muted': '#d4c6a6', 'dim': '#7d786a', 'off': '#5e5b51', 'on_text': '#fdf4de', 'salmon': '#e88f6c', + 'ok': '#a7d68c', 'gold': '#e6b65a', 'band_text': '#f3ecdc', 'status': '#b9a988', 'log_text': '#cfc4ab', + 'line': '#54442f', 'dot': '#5d4b36', 'filigree': '#7a6443', 'glow': '#e9ebee', + 'tri_top': '#d49f36', 'tri_bottom': '#fcbf44', +} + + +def rgb(hexcolor): + h = hexcolor.lstrip('#') + return np.array([int(h[i:i + 2], 16) / 255.0 for i in (0, 2, 4)], dtype=np.float32) + + +# ---------------------------------------------------------------- PNG and Tk + +def png_bytes(img): + """PNG of an HxWx4 float array in 0..1 (straight alpha) or uint8 array.""" + a = img if img.dtype == np.uint8 else (np.clip(img, 0, 1) * 255 + 0.5).astype(np.uint8) + h, w = a.shape[:2] + raw = b''.join(b'\0' + a[y].tobytes() for y in range(h)) + + def chunk(tag, data): + return struct.pack('>I', len(data)) + tag + data + struct.pack('>I', zlib.crc32(tag + data) & 0xFFFFFFFF) + return (b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', w, h, 8, 6, 0, 0, 0)) + + chunk(b'IDAT', zlib.compress(raw, 3)) + chunk(b'IEND', b'')) + + +def photo(master, img): + import tkinter as tk + return tk.PhotoImage(master=master, data=base64.b64encode(png_bytes(img)).decode('ascii'), format='png') + + +def ico_bytes(images): + """A Windows .ico holding PNG-compressed images (one HxWx4 array per size, square).""" + pngs = [png_bytes(im) for im in images] + head = struct.pack('= x1 or y0 >= y1: + return dst + s = src[y0 - y:y1 - y, x0 - x:x1 - x] + d = dst[y0:y1, x0:x1] + sa, da = s[..., 3:4], d[..., 3:4] + oa = sa + da * (1 - sa) + oc = np.where(oa > 0, (s[..., :3] * sa + d[..., :3] * da * (1 - sa)) / np.maximum(oa, 1e-6), 0) + d[..., :3], d[..., 3:4] = oc, oa + return dst + + +def paint(img, mask, color, alpha=1.0): + """Paint a colour through a coverage mask (HxW, 0..1) onto img.""" + layer = new(*mask.shape, color) + layer[..., 3] = np.clip(mask, 0, 1) * alpha + return over(img, layer) + + +def vgradient(h, w, stops): + """HxWx3 colours down the height through (position 0..1, colour) stops.""" + pos = np.array([p for p, _ in stops], dtype=np.float32) + cols = np.stack([rgb(c) for _, c in stops]) + t = (np.arange(h, dtype=np.float32) + 0.5) / max(h, 1) + col = np.stack([np.interp(t, pos, cols[:, k]) for k in range(3)], axis=1) + return np.broadcast_to(col[:, None, :], (h, w, 3)).copy() + + +def hgradient_alpha(h, w, stops): + """HxW alphas across the width through (position, alpha) stops.""" + pos = np.array([p for p, _ in stops], dtype=np.float32) + al = np.array([a for _, a in stops], dtype=np.float32) + t = (np.arange(w, dtype=np.float32) + 0.5) / max(w, 1) + return np.broadcast_to(np.interp(t, pos, al)[None, :], (h, w)).copy() + + +def hgradient_rgba(h, w, stops): + """An RGBA layer whose colour and alpha run across the width through (position, colour, alpha) stops.""" + pos = np.array([p for p, _, _ in stops], dtype=np.float32) + cols = np.stack([rgb(c) for _, c, _ in stops]) + al = np.array([a for _, _, a in stops], dtype=np.float32) + t = (np.arange(w, dtype=np.float32) + 0.5) / max(w, 1) + layer = new(h, w) + for k in range(3): + layer[..., k] = np.interp(t, pos, cols[:, k])[None, :] + layer[..., 3] = np.interp(t, pos, al)[None, :] + return layer + + +def value_noise(h, w, period_x, period_y=None, seed=0): + """Smooth noise in 0..1 with features about period pixels apart.""" + period_y = period_y or period_x + rng = np.random.default_rng(seed) + gh, gw = int(h / period_y) + 3, int(w / period_x) + 3 + g = rng.random((gh, gw), dtype=np.float32) + ys = np.arange(h, dtype=np.float32) / period_y + xs = np.arange(w, dtype=np.float32) / period_x + y0, x0 = ys.astype(int), xs.astype(int) + fy, fx = ys - y0, xs - x0 + fy, fx = fy * fy * (3 - 2 * fy), fx * fx * (3 - 2 * fx) + a, b = g[y0][:, x0], g[y0][:, x0 + 1] + c, d = g[y0 + 1][:, x0], g[y0 + 1][:, x0 + 1] + top = a + (b - a) * fx[None, :] + bot = c + (d - c) * fx[None, :] + return top + (bot - top) * fy[:, None] + + +def fractal(h, w, period_x, period_y=None, octaves=3, seed=0, contrast=0.4): + """Fractal noise around mid grey, about contrast wide (the range of an SVG feTurbulence turned grey).""" + return 0.5 + (_fractal01(h, w, period_x, period_y, octaves, seed) - 0.5) * contrast + + +def _fractal01(h, w, period_x, period_y=None, octaves=3, seed=0): + period_y = period_y or period_x + out = np.zeros((h, w), dtype=np.float32) + amp, total = 1.0, 0.0 + for o in range(octaves): + out += amp * value_noise(h, w, max(period_x / 2 ** o, 1.5), max(period_y / 2 ** o, 1.5), seed + o * 101) + total += amp + amp *= 0.5 + out /= total + lo, hi = np.percentile(out, 1), np.percentile(out, 99) + return np.clip((out - lo) / max(hi - lo, 1e-6), 0, 1) + + +def overlay(rgb_arr, n, opacity): + """CSS mix-blend-mode overlay of a grey noise n (HxW) over colours, at opacity.""" + n = n[..., None] + blended = np.where(rgb_arr < 0.5, 2 * rgb_arr * n, 1 - 2 * (1 - rgb_arr) * (1 - n)) + return rgb_arr + (blended - rgb_arr) * opacity + + +def box_blur(a, r): + """Three box blurs of radius r over a 2-D array (close to a gaussian).""" + r = int(round(r)) + if r < 1: + return a + out = a.astype(np.float32) + for _ in range(3): + for axis in (0, 1): + pad = [(0, 0), (0, 0)] + pad[axis] = (r + 1, r) + p = np.pad(out, pad, mode='edge') + c = np.cumsum(p, axis=axis, dtype=np.float64) + if axis == 0: + out = ((c[2 * r + 1:] - c[:-2 * r - 1]) / (2 * r + 1)).astype(np.float32) + else: + out = ((c[:, 2 * r + 1:] - c[:, :-2 * r - 1]) / (2 * r + 1)).astype(np.float32) + return out + + +def rect_mask(h, w, x0, y0, x1, y1): + """Coverage of the rectangle [x0, x1) x [y0, y1) with fractional edges.""" + xs = np.arange(w, dtype=np.float32) + ys = np.arange(h, dtype=np.float32) + cx = np.clip(np.minimum(xs + 1, x1) - np.maximum(xs, x0), 0, 1) + cy = np.clip(np.minimum(ys + 1, y1) - np.maximum(ys, y0), 0, 1) + return cy[:, None] * cx[None, :] + + +def ring_mask(h, w, x0, y0, x1, y1, width): + return np.clip(rect_mask(h, w, x0, y0, x1, y1) - rect_mask(h, w, x0 + width, y0 + width, x1 - width, y1 - width), + 0, 1) + + +def polygon_mask(h, w, pts, ss=4): + """Antialiased coverage of a convex polygon (pixel coordinates), supersampled ss x ss.""" + ys = (np.arange(h * ss, dtype=np.float32) + 0.5) / ss + xs = (np.arange(w * ss, dtype=np.float32) + 0.5) / ss + X, Y = np.meshgrid(xs, ys) + inside = np.ones_like(X, dtype=bool) + n = len(pts) + area = sum(pts[i][0] * pts[(i + 1) % n][1] - pts[(i + 1) % n][0] * pts[i][1] for i in range(n)) + sign = 1 if area > 0 else -1 + for i in range(n): + (ax, ay), (bx, by) = pts[i], pts[(i + 1) % n] + inside &= sign * ((bx - ax) * (Y - ay) - (by - ay) * (X - ax)) >= 0 + return inside.reshape(h, ss, w, ss).mean(axis=(1, 3)).astype(np.float32) + + +def stroke_mask(h, w, polylines, width): + """Antialiased coverage of polylines (lists of (x, y)) stroked with round joins and caps.""" + ys = np.arange(h, dtype=np.float32) + 0.5 + xs = np.arange(w, dtype=np.float32) + 0.5 + X, Y = np.meshgrid(xs, ys) + best = np.full((h, w), 1e9, dtype=np.float32) + for line in polylines: + P = np.asarray(line, dtype=np.float32) + if len(P) == 1: + P = np.vstack([P, P]) + for (ax, ay), (bx, by) in zip(P[:-1], P[1:]): + dx, dy = bx - ax, by - ay + ll = dx * dx + dy * dy + if ll < 1e-9: + d = np.hypot(X - ax, Y - ay) + else: + t = np.clip(((X - ax) * dx + (Y - ay) * dy) / ll, 0, 1) + d = np.hypot(X - (ax + t * dx), Y - (ay + t * dy)) + np.minimum(best, d, out=best) + return np.clip(width / 2 + 0.5 - best, 0, 1) + + +def shadow(h, w, mask, color, alpha, blur, dx=0, dy=0): + """A soft shadow of a mask: blurred, offset, as an RGBA layer of the same size.""" + m = box_blur(np.roll(np.roll(mask, int(dy), 0), int(dx), 1), blur / 2) + layer = new(h, w, color) + layer[..., 3] = np.clip(m, 0, 1) * alpha + return layer + + +# ---------------------------------------------------------------- SVG path data (the filigree) + +def svg_polylines(d, scale=1.0, ox=0.0, oy=0.0, steps=10): + """Points along an SVG path made of M, L, C and Z commands.""" + toks = re.findall(r'[MLCZmlcz]|-?\d*\.?\d+', d) + lines, cur, pos, i, cmd = [], [], (0.0, 0.0), 0, None + start = (0.0, 0.0) + + def num(): + nonlocal i + v = float(toks[i]) + i += 1 + return v + + while i < len(toks): + if toks[i] in 'MLCZmlcz': + cmd = toks[i] + i += 1 + if cmd in 'Zz': + cur.append(start) + continue + if cmd in 'Mm': + if cur: + lines.append(cur) + pos = (num(), num()) + start = pos + cur = [pos] + cmd = 'L' + elif cmd in 'Ll': + pos = (num(), num()) + cur.append(pos) + elif cmd in 'Cc': + p0 = pos + c1, c2, p1 = (num(), num()), (num(), num()), (num(), num()) + for k in range(1, steps + 1): + t = k / steps + u = 1 - t + cur.append((u ** 3 * p0[0] + 3 * u * u * t * c1[0] + 3 * u * t * t * c2[0] + t ** 3 * p1[0], + u ** 3 * p0[1] + 3 * u * u * t * c1[1] + 3 * u * t * t * c2[1] + t ** 3 * p1[1])) + pos = p1 + if cur: + lines.append(cur) + return [[((x - ox) * scale, (y - oy) * scale) for x, y in line] for line in lines] + + +# The game's divider ornament, traced from the options menu: two pointed lens ends, S-waves into spiral hooks and a +# diamond knot under the middle, in a 120 x 38 box (centre x 60). +FILIGREE_PATHS = ( + 'M20 17.4 C20.4 17.3 21.6 16.8 22.2 16.9 C22.8 17 23.5 17.7 23.5 18.2 C23.5 18.7 23.1 19.4 22.4 19.8 C21.6 20.2 ' + '20.3 20.4 19 20.4 C17.7 20.4 15.9 20.2 14.5 19.9 C13.1 19.6 10.6 19.4 10.5 18.6 C10.4 17.9 12.6 16.3 14 15.4 ' + 'C15.4 14.5 17.2 13.6 19 13.2 C20.8 12.8 23.2 12.8 25 13.1 C26.8 13.4 28.1 13.9 29.5 14.8 C30.9 15.7 32.2 17.4 ' + '33.5 18.6 C34.8 19.8 36 21.2 37.5 22 C39 22.8 40.8 23.4 42.5 23.6 C44.2 23.8 46 23.5 47.5 23.1 C49 22.7 50.4 ' + '21.9 51.2 21.2 C52 20.5 52.5 19.5 52.4 18.8 C52.3 18.1 51.5 17.3 50.6 16.9 C49.7 16.5 48.2 16.2 47 16.3 C45.8 ' + '16.4 44.2 17.1 43.6 17.2', + 'M100 17.4 C99.6 17.3 98.4 16.8 97.8 16.9 C97.2 17 96.5 17.7 96.5 18.2 C96.5 18.7 96.8 19.4 97.6 19.8 C98.3 20.2 ' + '99.7 20.4 101 20.4 C102.3 20.4 104.1 20.2 105.5 19.9 C106.9 19.6 109.4 19.4 109.5 18.6 C109.6 17.9 107.4 16.3 ' + '106 15.4 C104.6 14.5 102.8 13.6 101 13.2 C99.2 12.8 96.8 12.8 95 13.1 C93.2 13.4 91.9 13.9 90.5 14.8 C89.1 15.7 ' + '87.8 17.4 86.5 18.6 C85.2 19.8 84 21.2 82.5 22 C81 22.8 79.2 23.4 77.5 23.6 C75.8 23.8 74 23.5 72.5 23.1 C71 ' + '22.7 69.6 21.9 68.8 21.2 C68 20.5 67.5 19.5 67.6 18.8 C67.7 18.1 68.5 17.3 69.4 16.9 C70.3 16.5 71.8 16.2 73 ' + '16.3 C74.2 16.4 75.8 17.1 76.4 17.2', + 'M40.6 17.6 C40.7 17 40.7 14.9 41.2 14 C41.7 13.1 42.6 12.4 43.8 12 C45 11.6 46.7 11.4 48.2 11.5 C49.7 11.6 51.3 ' + '11.8 52.6 12.3 C53.9 12.8 55 13.6 56 14.6 C57 15.6 58.2 17.6 58.6 18.2 L64.6 26 L60 31.6', + 'M79.4 17.6 C79.3 17 79.3 14.9 78.8 14 C78.3 13.1 77.4 12.4 76.2 12 C75 11.6 73.3 11.4 71.8 11.5 C70.3 11.6 68.7 ' + '11.8 67.4 12.3 C66.1 12.8 65 13.6 64 14.6 C63 15.6 61.8 17.6 61.4 18.2 L55.4 26 L60 31.6', +) + + +# ---------------------------------------------------------------- the window's pieces + +def backdrop(W, H, s): + """The whole window behind the panels: a dim misty field under a see-through dark green, darker at the sides.""" + img = new(H, W, PALETTE['window'], 1.0) + m = int(30 * s) + scene = new(H, W - 2 * m) + scene[..., :3] = vgradient(H, W - 2 * m, [(0, '#394244'), (0.36, '#414b4c'), (0.60, '#474234'), (1, '#3e3523')]) + scene[..., 3] = 1 + over(img, scene, m, 0) + band = np.zeros((H, W), dtype=np.float32) + y0, y1 = int(250 * s), int(330 * s) + t = (np.arange(y1 - y0) + 0.5) / max(y1 - y0, 1) + band[y0:y1, m:W - m] = (np.interp(t, [0, 0.55, 1], [0, 0.85, 0]))[:, None] + paint(img, box_blur(band, 10 * s), '#262a22') + over(img, hgradient_rgba(H, W, [(0, '#05180d', 0.96), (0.14, '#092615', 0.82), (0.5, '#0c2e1b', 0.72), + (0.86, '#092615', 0.82), (1, '#05180d', 0.96)])) + inner = rect_mask(H, W, 0, 0, W, H) + vig = 1 - box_blur(np.pad(inner, int(45 * s), constant_values=0), 45 * s)[int(45 * s):-int(45 * s), + int(45 * s):-int(45 * s)] + paint(img, np.clip(vig * 1.6, 0, 1), '#020e07', 0.6) + return img + + +def band(w, h, s): + """The wooden title band, its two-line bottom edge and the shadow under it; (image, extra height below).""" + pad = int(14 * s) + img = new(h + pad, w) + wood = vgradient(h, w, [(0, '#4c3b26'), (0.30, '#5a4529'), (0.65, '#6b5232'), (1, '#806239')]) + wood = overlay(wood, fractal(h, w, 300 * s, 11 * s, 3, seed=9), 0.32) + wood = overlay(wood, fractal(h, w, 33 * s, None, 3, seed=4), 0.2) + shade = rect_mask(h + pad, w, 0, 0, w, h + 4 * s) + over(img, shadow(h + pad, w, shade, '#000000', 0.6, 16 * s, 0, 5 * s)) + top = new(h, w) + top[..., :3], top[..., 3] = wood, 1 + over(img, top) + paint(img, rect_mask(h + pad, w, 0, h, w, h + 2 * s), '#2c2217') + paint(img, rect_mask(h + pad, w, 0, h + 2 * s, w, h + 4 * s), '#a9874f') + return img + + +def panel(w, h, s): + """The left list panel with the shadow on its right.""" + sh = int(26 * s) + img = new(h, w + sh) + body = rect_mask(h, w + sh, 0, 0, w, h) + over(img, shadow(h, w + sh, body, '#000000', 0.35, 18 * s, 6 * s, 0)) + col = np.broadcast_to(rgb(PALETTE['panel']), (h, w, 3)).copy() + col = overlay(col, fractal(h, w, 33 * s, None, 3, seed=4), 0.18) + layer = new(h, w) + layer[..., :3], layer[..., 3] = col, 1 + over(img, layer) + return img + + +def label_cell(w, h, s, bright=False): + """A bronze label cell: gradient, mottled texture, 1 px border and the 1 px dark ring around the row.""" + r = max(1, int(round(s))) + img = new(h + 2 * r, w + 2 * r, PALETTE['ring'], 1.0) + col = vgradient(h, w, [(0, PALETTE['label_top']), (0.5, PALETTE['label_mid']), (1, PALETTE['label_bottom'])]) + col = overlay(col, fractal(h, w, 33 * s, None, 3, seed=(w * 7 + h) % 997), 0.22) + if bright: + col = np.clip(col * 1.08, 0, 1) + cell = new(h, w) + cell[..., :3], cell[..., 3] = col, 1 + over(img, cell, r, r) + paint(img, ring_mask(h + 2 * r, w + 2 * r, r, r, w + r, h + r, r), PALETTE['label_border']) + return img + + +def desc_box(w, h, s): + pad = int(20 * s) + img = new(h + 2 * pad, w + 2 * pad) + body = rect_mask(h + 2 * pad, w + 2 * pad, pad, pad, pad + w, pad + h) + over(img, shadow(h + 2 * pad, w + 2 * pad, body, '#000000', 0.35, 18 * s, 0, 6 * s)) + col = np.broadcast_to(rgb(PALETTE['desc']), (h, w, 3)).copy() + col = overlay(col, fractal(h, w, 33 * s, None, 3, seed=11), 0.15) + layer = new(h, w) + layer[..., :3], layer[..., 3] = col, 1 + over(img, layer, pad, pad) + paint(img, ring_mask(h + 2 * pad, w + 2 * pad, pad, pad, pad + w, pad + h, max(1, round(s))), + PALETTE['desc_border']) + return img, pad + + +def glow_frames(w, h, s, n=8): + """The selection cursor around a w x h box: a 3 px white frame 6 px outside it with a soft blue glow inside and + out, n frames of one breath. Returns (frames, margin) where the box sits at (margin, margin).""" + gap, thick, spread = 6 * s, 3 * s, 16 * s + mg = int(gap + spread) + H, W = h + 2 * mg, w + 2 * mg + ring = ring_mask(H, W, mg - gap, mg - gap, mg + w + gap, mg + h + gap, thick) + soft_out = box_blur(ring, 6 * s) + soft_far = box_blur(ring, 12 * s) + frames = [] + for k in range(n): + t = 0.5 - 0.5 * np.cos(2 * np.pi * k / n) + img = new(H, W) + paint(img, np.clip(soft_far * 2.2, 0, 1), '#96b4e6', 0.35 + 0.25 * t) + paint(img, np.clip(soft_out * 2.8, 0, 1), '#afc5e8', 0.7 + 0.3 * t) + paint(img, ring, PALETTE['glow']) + frames.append(img) + return frames, mg + + +def filigree(w, s): + """The divider: a thin line with a dot at each end and the ornament in the middle; (image, height).""" + h = int(round(26 * s)) + img = new(h, w) + mid = h / 2 + orn_w = 78 * s + scale = orn_w / 108.0 + ox = (w - orn_w) / 2 + dot_r = 1.5 * s + gap = 5 * s + left_end, right_start = ox - gap, ox + orn_w + gap + xs0, xs1 = 4 * s + 2 * dot_r + gap, w - 4 * s - 2 * dot_r - gap + paint(img, rect_mask(h, w, xs0, mid - 0.5 * s, left_end, mid + 0.5 * s), PALETTE['line']) + paint(img, rect_mask(h, w, right_start, mid - 0.5 * s, xs1, mid + 0.5 * s), PALETTE['line']) + dots = stroke_mask(h, w, [[(4 * s + dot_r, mid)], [(w - 4 * s - dot_r, mid)]], 2 * dot_r) + paint(img, dots, PALETTE['dot']) + lines = [] + for d in FILIGREE_PATHS: + for line in svg_polylines(d, scale, 6, 8): + lines.append([(x + ox, y + mid - 13 * scale) for x, y in line]) + paint(img, stroke_mask(h, w, lines, 2.2 * scale), PALETTE['filigree']) + return img, h + + +def triforce(w): + """The Triforce, w pixels wide: three triangles, each shading from top to bottom, on a transparent ground.""" + h = int(round(w * 217 / 250)) + k = w / 250.0 + tris = [((125, 1), (186.2, 107), (63.8, 107)), ((62.2, 109), (123.4, 215), (1, 215)), + ((187.8, 109), (249, 215), (126.6, 215))] + img = new(h, w) + top, bottom = rgb(PALETTE['tri_top']), rgb(PALETTE['tri_bottom']) + for tri in tris: + pts = [(x * k, y * k) for x, y in tri] + mask = polygon_mask(h, w, pts) + y0, y1 = min(p[1] for p in pts), max(p[1] for p in pts) + t = np.clip((np.arange(h, dtype=np.float32) + 0.5 - y0) / max(y1 - y0, 1e-6), 0, 1)[:, None, None] + layer = new(h, w) + layer[..., :3] = top + (bottom - top) * t + layer[..., 3] = mask + over(img, layer) + return img + + +def triforce_ico(): + """The launcher's icon at the sizes Windows asks for.""" + imgs = [] + for n in (16, 24, 32, 48, 64, 128, 256): + t = triforce(n) + sq = new(n, n) + over(sq, t, 0, (n - t.shape[0]) // 2) + imgs.append(sq) + return ico_bytes(imgs) + + +def check_mark(size, s, color=None): + img = new(size, size) + k = size / 24.0 + paint(img, stroke_mask(size, size, [[(5 * k, 12.5 * k), (9.2 * k, 16.7 * k), (19 * k, 7 * k)]], 3 * k), + color or PALETTE['ok']) + return img + + +def chevron(w, h, color, left=False): + img = new(h, w) + pts = [(w - 0.5, 0), (0.5, h / 2), (w - 0.5, h)] if left else [(0.5, 0), (w - 0.5, h / 2), (0.5, h)] + paint(img, polygon_mask(h, w, pts), color) + return img + + +def framed_icon(size, s, kind): + """The square-framed icon of a big row (play or refresh), like the icon frames of the game's menu rows.""" + img = new(size, size) + t = 1.5 * s + paint(img, ring_mask(size, size, 0.5, 0.5, size - 0.5, size - 0.5, t), '#e9dcc0') + c = size / 2 + if kind == 'play': + k = size / 32 + paint(img, polygon_mask(size, size, [(c - 5 * k, c - 7 * k), (c + 7 * k, c), (c - 5 * k, c + 7 * k)]), + '#efe4ca') + elif kind == 'refresh': + import math + k = size / 32 + r = 7.5 * k + arcs = [] + for a0, a1 in ((200, 340), (20, 160)): + arcs.append([(c + r * math.cos(math.radians(a)), c + r * math.sin(math.radians(a))) + for a in range(a0, a1 + 1, 6)]) + heads = [[(c + r * math.cos(math.radians(340)) - 3.5 * k, c + r * math.sin(math.radians(340)) - 1 * k), + (c + r * math.cos(math.radians(340)), c + r * math.sin(math.radians(340))), + (c + r * math.cos(math.radians(340)) + 0.5 * k, c + r * math.sin(math.radians(340)) - 4 * k)], + [(c + r * math.cos(math.radians(160)) + 3.5 * k, c + r * math.sin(math.radians(160)) + 1 * k), + (c + r * math.cos(math.radians(160)), c + r * math.sin(math.radians(160))), + (c + r * math.cos(math.radians(160)) - 0.5 * k, c + r * math.sin(math.radians(160)) + 4 * k)]] + paint(img, stroke_mask(size, size, arcs + heads, 2.2 * k), '#efe4ca') + return img + + +def badge(size, s): + """The orange "!" disc beside Updates when a new version is ready (the "!" itself is text).""" + img = new(size + 4, size + 4) + ring = stroke_mask(size + 4, size + 4, [[((size + 4) / 2, (size + 4) / 2)]], size + 1.5 * s) + over(img, shadow(size + 4, size + 4, ring, '#fbbd45', 0.6, 6 * s)) + paint(img, ring, '#3a2a14') + disc = stroke_mask(size + 4, size + 4, [[((size + 4) / 2, (size + 4) / 2)]], size) + layer = new(size + 4, size + 4) + layer[..., :3] = vgradient(size + 4, size + 4, [(0, '#fbbd45'), (1, '#e2942a')]) + layer[..., 3] = disc + over(img, layer) + return img diff --git a/launcher/ui/selftest.py b/launcher/ui/selftest.py new file mode 100644 index 00000000..f3d410d2 --- /dev/null +++ b/launcher/ui/selftest.py @@ -0,0 +1,173 @@ +"""--selftest [--shots DIR]: draw every screen with made-up states, optionally save a PNG of each, and close. +Nothing runs: the jobs are built and their steps set by hand.""" +import os +import subprocess +import sys +import time +from pathlib import Path + +from .. import feed, jobs + + +def grab_gdi(x, y, w, h, path): + """Windows: copy the screen rectangle with GDI and write it as PNG (art.png_bytes); no extra packages.""" + import ctypes + from ctypes import wintypes + import numpy as np + from . import art + user32, gdi32 = ctypes.windll.user32, ctypes.windll.gdi32 + user32.GetDC.restype = wintypes.HDC + gdi32.CreateCompatibleDC.restype = wintypes.HDC + gdi32.CreateCompatibleBitmap.restype = wintypes.HBITMAP + gdi32.SelectObject.restype = wintypes.HGDIOBJ + screen = user32.GetDC(None) + mem = gdi32.CreateCompatibleDC(screen) + bmp = gdi32.CreateCompatibleBitmap(screen, w, h) + old = gdi32.SelectObject(mem, bmp) + gdi32.BitBlt(mem, 0, 0, w, h, screen, x, y, 0x00CC0020) # SRCCOPY + + class BITMAPINFOHEADER(ctypes.Structure): + _fields_ = [('biSize', wintypes.DWORD), ('biWidth', wintypes.LONG), ('biHeight', wintypes.LONG), + ('biPlanes', wintypes.WORD), ('biBitCount', wintypes.WORD), ('biCompression', wintypes.DWORD), + ('biSizeImage', wintypes.DWORD), ('biXPelsPerMeter', wintypes.LONG), + ('biYPelsPerMeter', wintypes.LONG), ('biClrUsed', wintypes.DWORD), + ('biClrImportant', wintypes.DWORD)] + bi = BITMAPINFOHEADER(ctypes.sizeof(BITMAPINFOHEADER), w, -h, 1, 32, 0, 0, 0, 0, 0, 0) + buf = (ctypes.c_ubyte * (w * h * 4))() + gdi32.GetDIBits(mem, bmp, 0, h, buf, ctypes.byref(bi), 0) + gdi32.SelectObject(mem, old) + gdi32.DeleteObject(bmp) + gdi32.DeleteDC(mem) + user32.ReleaseDC(None, screen) + a = np.frombuffer(buf, dtype=np.uint8).reshape(h, w, 4)[..., [2, 1, 0, 3]].copy() + a[..., 3] = 255 + Path(path).write_bytes(art.png_bytes(a)) + return True + + +def grab(app, path): + """A PNG of the window: GDI on Windows, else PIL's ImageGrab, else ImageMagick's import on X11.""" + app.root.update() + x, y = app.root.winfo_rootx(), app.root.winfo_rooty() + w, h = app.root.winfo_width(), app.root.winfo_height() + if os.name == 'nt': + try: + return grab_gdi(x, y, w, h, path) + except Exception as e: + app.ctx.log(f'selftest: screen capture failed: {e!r}') + try: + from PIL import ImageGrab + ImageGrab.grab(bbox=(x, y, x + w, y + h), xdisplay=os.environ.get('DISPLAY') if os.name != 'nt' else None + ).save(path) + return True + except Exception: + pass + try: + subprocess.run(['import', '-window', str(app.root.winfo_id()), str(path)], check=True, capture_output=True) + return True + except (OSError, subprocess.CalledProcessError): + return False + + +def fake_release(version='0.3.11'): + m = {'format': 1, 'version': version, 'app': {'name': f'NX2navi-app-{version}.zip', 'size': 1, 'sha256': '0'}} + notes = ('- Each program module comes from the .nup package that holds it whole.\n' + '- The PC stays awake while the game builds.\n- This launcher: one-click install, updates and play.') + return feed.Release(m, notes) + + +def states(app): + ctx = app.ctx + s = app.settings + s['dump'] = s.get('dump') or str(Path.home() / 'Downloads' / 'Ocarina of Time.nup') + s['game_dir'] = s.get('game_dir') or ('D:\\Games\\NX2navi' if os.name == 'nt' else '/games/NX2navi') + app.dumps = [Path(s['dump'])] + + def first_run(): + app.show('first_run', len(app.rows_first_run()) - 1) + + def installing(): + j = jobs.InstallJob(ctx, s['dump'], s['game_dir']) + for sid, val in (('tools', 'Done · 0:48'), ('check', 'Version 1.1.1'), ('exefs', 'Done · 0:04'), + ('romfs', 'Done · 0:31')): + j.step(sid).state, j.step(sid).value, j.step(sid).fraction = jobs.DONE, val, 1.0 + j.begin('lift') + j.progress('lift', 0.34, '412 of 1,204 chunks') + j.started = time.time() - 192 + for ln in ('check: Ocarina of Time 1.1.1 (build e0f3ccfa)', 'romfs: 26.0 of 26.0 GB, 31 s', 'lift:', + ' emitting with 16 workers', ' emitted 412/1204 chunks, 18s'): + j.lines.append(ln) + app.job, app.job_return, app.job_back = j, 'home', 'first_run' + app.show('job') + + def home(): + app.release, app.checked_at, app.check, app.check_error, app.message = None, time.time() - 5, None, '', '' + app.show('home', 0) + + def update_ready(): + app.release = fake_release() + app.show('home', 1) + + def updating(): + j = jobs.UpdateDownloadJob(ctx, fake_release()) + for sid, val in (('download', 'Done · 0:06'), ('verify', 'Checksum OK')): + j.step(sid).state, j.step(sid).value, j.step(sid).fraction = jobs.DONE, val, 1.0 + j.skip('tools_update', 'Unchanged') + j.step('lift').state, j.step('lift').value, j.step('lift').fraction = jobs.DONE, 'Done · 0:41', 1.0 + j.begin('compile_changed') + j.progress('compile_changed', 0.61, '238 of 391 changed files') + j.started = time.time() - 104 + app.job, app.job_return = j, 'home' + app.show('job') + + def settings(): + app.show('settings', 0) + + def failed(): + j = jobs.InstallJob(ctx, s['dump'], s['game_dir']) + for sid, val in (('tools', 'Done · 0:48'), ('check', 'Version 1.1.1'), ('exefs', 'Done · 0:04'), + ('romfs', 'Done · 0:31'), ('lift', 'Done · 0:48')): + j.step(sid).state, j.step(sid).value, j.step(sid).fraction = jobs.DONE, val, 1.0 + j.step('compile').state, j.step('compile').value = jobs.FAILED, 'Stopped' + j.result, j.error = 'failed', '3 files did not compile' + j.started, j.finished = time.time() - 362, time.time() + for ln in ('compile:', ' compiled 1200 of 1212, 236 s', 'error: 3 files did not compile'): + j.lines.append(ln) + app.job, app.job_return, app.job_back = j, 'home', 'first_run' + app.show('job') + + def uninstall(): + app.show('uninstall', 0) + + return [('1-first-run', first_run), ('2-installing', installing), ('3-home', home), + ('4-update-ready', update_ready), ('5-updating', updating), ('6-settings', settings), + ('7-build-failed', failed), ('8-uninstall', uninstall)] + + +def run(app, shots): + out = Path(shots) if shots else None + if out: + out.mkdir(parents=True, exist_ok=True) + seq = states(app) + ok = True + + def step(i=0): + nonlocal ok + if i >= len(seq): + app.ctx.log(f'selftest: {len(seq)} screens drawn') + print(f'selftest: {len(seq)} screens drawn' + (f', shots in {out}' if out else ''), flush=True) + app.root.after(100, app.root.destroy) + return + name, fn = seq[i] + try: + fn() + app.root.update() + except Exception as e: + ok = False + print(f'selftest: {name}: {e!r}', file=sys.stderr, flush=True) + raise + if out: + app.root.after(350, lambda: (grab(app, out / f'{name}.png'), step(i + 1))) + else: + app.root.after(50, lambda: step(i + 1)) + step() diff --git a/launcher/updater.py b/launcher/updater.py new file mode 100644 index 00000000..a5261d62 --- /dev/null +++ b/launcher/updater.py @@ -0,0 +1,158 @@ +"""Updating the launcher's own release folder (launcher/paths.py, "app/"). + +An update is staged beside the running version and switched to with two small files, so a failed or interrupted +update never leaves a half-written release in use: + +1. The app archive goes to data/downloads, is checked against the manifest's size and sha256 and unpacks into + app/.part, which is renamed to app/ once it is complete and holds the expected VERSION. +2. A new Python (the manifest's runtime id differs from python/runtime-id.txt) unpacks into python.next; the stub + swaps it in at its next start, while no Python runs. +3. app/previous.txt gets the running version and app/current.txt the new one; the launcher starts the stub again, + which runs the new version. The stub goes back to previous.txt when the new version fails to start. +4. The new version rebuilds the game (launcher/jobs.py), then prune() removes every version but these two. +""" +import os +import shutil +import zipfile +from pathlib import Path + +from . import net, versions + + +class UpdateError(Exception): + pass + + +def read_text(path, default=''): + try: + return Path(path).read_text(encoding='utf-8').strip() + except OSError: + return default + + +def write_text(path, text): + path = Path(path) + tmp = path.with_name(path.name + '.tmp') + tmp.write_text(text + '\n', encoding='utf-8') + os.replace(tmp, path) + + +def current(layout): + return read_text(layout.apps / 'current.txt') or layout.version + + +def previous(layout): + return read_text(layout.apps / 'previous.txt') + + +def runtime_id(layout): + return read_text(layout.python / 'runtime-id.txt') + + +def safe_extract(zip_path, dest): + """Unpack a zip into dest, refusing absolute names and names that climb out of it.""" + dest = Path(dest) + root = dest.resolve() + with zipfile.ZipFile(zip_path) as z: + for info in z.infolist(): + name = info.filename.replace('\\', '/') + if name.startswith('/') or ':' in name.split('/')[0] or '..' in name.split('/'): + raise UpdateError(f'{Path(zip_path).name}: unsafe path {info.filename}') + target = (dest / name).resolve() + if target != root and root not in target.parents: + raise UpdateError(f'{Path(zip_path).name}: unsafe path {info.filename}') + z.extractall(dest) + + +def _fetch(release, key, layout, progress, cancel, version): + url, headers, size, sha = release.asset(key) + entry = release.manifest[key] + dest = layout.downloads / entry['name'] + net.download(url, dest, sha256=sha, size=size, headers=headers, progress=progress, cancel=cancel, + version=version) + return dest + + +def stage_app(layout, release, progress=None, cancel=None, version='dev'): + """Download, check and unpack the release's app archive into app/; returns that folder.""" + zip_path = _fetch(release, 'app', layout, progress, cancel, version) + ver = versions.display(release.version) + final = layout.apps / ver + part = layout.apps / f'{ver}.part' + shutil.rmtree(part, ignore_errors=True) + part.mkdir(parents=True) + try: + safe_extract(zip_path, part) + got = read_text(part / 'VERSION') + if versions.display(got) != ver or not (part / 'launcher' / 'start.pyw').is_file(): + raise UpdateError(f'{zip_path.name} holds version {got or "?"} without a launcher, not {ver}') + if final.exists(): + if final.resolve() == layout.app.resolve(): + raise UpdateError(f'{ver} is the version that runs now') + shutil.rmtree(final) + os.replace(part, final) + except BaseException: + shutil.rmtree(part, ignore_errors=True) + raise + zip_path.unlink(missing_ok=True) + return final + + +def runtime_needed(layout, release): + rt = release.manifest.get('runtime') + return bool(rt) and not layout.dev and rt.get('id') != runtime_id(layout) + + +def stage_runtime(layout, release, progress=None, cancel=None, version='dev'): + """Unpack a new Python into python.next for the stub to swap in; False when the running one is current.""" + if not runtime_needed(layout, release): + return False + zip_path = _fetch(release, 'runtime', layout, progress, cancel, version) + nxt = layout.root / 'python.next' + shutil.rmtree(nxt, ignore_errors=True) + part = layout.root / 'python.next.part' + shutil.rmtree(part, ignore_errors=True) + part.mkdir() + safe_extract(zip_path, part) + if not (part / 'python.exe').is_file() or read_text(part / 'runtime-id.txt') != release.manifest['runtime']['id']: + shutil.rmtree(part, ignore_errors=True) + raise UpdateError(f'{zip_path.name} is not the runtime {release.manifest["runtime"]["id"]}') + os.replace(part, nxt) + zip_path.unlink(missing_ok=True) + return True + + +def switch_to(layout, ver): + """Make ver the version the stub starts, keeping the running one as previous.""" + if not (layout.apps / ver / 'launcher' / 'start.pyw').is_file(): + raise UpdateError(f'app/{ver} is not a complete release') + now = current(layout) + if now != ver: + write_text(layout.apps / 'previous.txt', now) + write_text(layout.apps / 'current.txt', ver) + + +def rollback(layout): + """Back to previous.txt's version; False when there is none to go back to.""" + prev = previous(layout) + if not prev or not (layout.apps / prev / 'launcher' / 'start.pyw').is_file(): + return False + write_text(layout.apps / 'current.txt', prev) + (layout.apps / 'previous.txt').unlink(missing_ok=True) + return True + + +def prune(layout): + """Remove every app version but the current and previous one, an old Python the stub set aside, and stale + downloads.""" + keep = {current(layout), previous(layout)} + if layout.apps.is_dir(): + for d in layout.apps.iterdir(): + if d.is_dir() and d.name not in keep: + shutil.rmtree(d, ignore_errors=True) + for d in layout.root.glob('python.old*'): + shutil.rmtree(d, ignore_errors=True) + if layout.downloads.is_dir(): + for f in layout.downloads.iterdir(): + if f.suffix in ('.part', '.zip'): + f.unlink(missing_ok=True) diff --git a/launcher/versions.py b/launcher/versions.py new file mode 100644 index 00000000..265204d6 --- /dev/null +++ b/launcher/versions.py @@ -0,0 +1,25 @@ +"""Release versions: "0.3.11", optionally with build metadata ("0.3.11+abc1234") that never orders releases.""" +import re + +_NUM = re.compile(r'^\s*v?(\d+(?:\.\d+)*)(?:-([0-9A-Za-z.-]+))?(?:\+([0-9A-Za-z.-]+))?\s*$') + + +def parse(text): + """(numbers, pre-release) for comparing; a pre-release sorts before its release. ValueError when malformed.""" + m = _NUM.match(text or '') + if not m: + raise ValueError(f'not a version: {text!r}') + nums = tuple(int(x) for x in m.group(1).split('.')) + nums = nums + (0,) * (4 - len(nums)) if len(nums) < 4 else nums + pre = m.group(2) + return nums, (1,) if pre is None else (0, pre) + + +def newer(a, b): + """True when version a is newer than version b.""" + return parse(a) > parse(b) + + +def display(text): + """The version without build metadata.""" + return (text or '').split('+', 1)[0].strip() diff --git a/launcher/winsys.py b/launcher/winsys.py new file mode 100644 index 00000000..1370e80e --- /dev/null +++ b/launcher/winsys.py @@ -0,0 +1,192 @@ +"""What the launcher needs from the operating system: crisp drawing on scaled displays, the game's fonts for this +process only, starting the game and the stub, shortcuts, the uninstall entry and a gamepad. Windows first; the +other branches exist so the window can be developed and tested on Linux.""" +import ctypes +import os +import subprocess +import sys +from pathlib import Path + +CREATE_NO_WINDOW = 0x08000000 +DETACHED_PROCESS = 0x00000008 +CREATE_NEW_PROCESS_GROUP = 0x00000200 +UNINSTALL_KEY = r'Software\Microsoft\Windows\CurrentVersion\Uninstall\NX2navi' + + +def dpi_aware(): + """Draw at the display's real resolution instead of letting Windows stretch a 96-dpi window (blurry).""" + if os.name != 'nt': + return + try: + ctypes.windll.shcore.SetProcessDpiAwareness(1) + except (AttributeError, OSError): + try: + ctypes.windll.user32.SetProcessDPIAware() + except (AttributeError, OSError): + pass + + +def app_id(): + """Group the window under the launcher's own taskbar icon rather than python's.""" + if os.name == 'nt': + try: + ctypes.windll.shell32.SetCurrentProcessExplicitAppUserModelID('NX2navi.Launcher') + except (AttributeError, OSError): + pass + + +def register_font(path): + """Make a font file usable by this process only (Tk then finds it by family name). True on success.""" + path = str(path) + if os.name == 'nt': + try: + return ctypes.windll.gdi32.AddFontResourceExW(ctypes.c_wchar_p(path), 0x10, None) > 0 # FR_PRIVATE + except (AttributeError, OSError): + return False + try: + fc = ctypes.CDLL('libfontconfig.so.1') + fc.FcConfigAppFontAddFile.argtypes = [ctypes.c_void_p, ctypes.c_char_p] + return bool(fc.FcConfigAppFontAddFile(None, path.encode())) + except OSError: + return False + + +def open_path(path): + path = str(path) + if os.name == 'nt': + os.startfile(path) # noqa: S606 — opens a folder in Explorer + else: + subprocess.Popen(['xdg-open', path], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + +def start_game(exe): + """Start the game from its own folder; returns the process to watch.""" + exe = Path(exe) + flags = {'creationflags': CREATE_NEW_PROCESS_GROUP} if os.name == 'nt' else {'start_new_session': True} + return subprocess.Popen([str(exe)], cwd=str(exe.parent), stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, close_fds=True, **flags) + + +def start_detached(cmd, cwd=None): + flags = {'creationflags': DETACHED_PROCESS | CREATE_NEW_PROCESS_GROUP} if os.name == 'nt' \ + else {'start_new_session': True} + subprocess.Popen(cmd, cwd=cwd, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + close_fds=True, **flags) + + +def restart(layout, args=()): + """Start the launcher again through its stub (which swaps in a staged Python and starts app/current.txt's + version). False in a checkout, which has no stub.""" + if not layout.stub.is_file(): + return False + start_detached([str(layout.stub), *args], cwd=str(layout.root)) + return True + + +def stub_run(layout, args, timeout=30): + """Run the stub for one of its own commands (shortcuts); True when it succeeded.""" + if not layout.stub.is_file(): + return False + flags = {'creationflags': CREATE_NO_WINDOW} if os.name == 'nt' else {} + try: + return subprocess.run([str(layout.stub), *args], timeout=timeout, capture_output=True, **flags).returncode == 0 + except (OSError, subprocess.TimeoutExpired): + return False + + +def desktop_shortcut(layout, enable): + return stub_run(layout, ['--desktop-shortcut', 'on' if enable else 'off']) + + +def keep_awake(on): + if os.name != 'nt': + return + try: + ES_CONTINUOUS, ES_SYSTEM_REQUIRED = 0x80000000, 0x00000001 + ctypes.windll.kernel32.SetThreadExecutionState(ES_CONTINUOUS | (ES_SYSTEM_REQUIRED if on else 0)) + except (AttributeError, OSError): + pass + + +def uninstall(layout, remove_game='', log=print): + """Remove the shortcuts, the uninstall entry and, once this process has exited, the launcher's folder (and the + game folder when remove_game names it). Saves inside a kept game folder stay.""" + stub_run(layout, ['--shortcuts', 'off']) + if os.name == 'nt': + try: + import winreg + winreg.DeleteKey(winreg.HKEY_CURRENT_USER, UNINSTALL_KEY) + except OSError: + pass + targets = [str(layout.root)] + ([str(remove_game)] if remove_game else []) + if layout.toolchain.parent != layout.root: + targets.append(str(layout.toolchain.parent)) + script = 'ping -n 3 127.0.0.1 >nul & ' + ' & '.join(f'rmdir /s /q "{t}"' for t in targets) + start_detached(['cmd.exe', '/c', script], cwd=os.environ.get('TEMP', 'C:\\')) + else: + log('uninstall: remove these folders yourself: ' + ', '.join([str(layout.root), str(remove_game)])) + + +# ---------------------------------------------------------------- gamepad (XInput) + +class _XInputGamepad(ctypes.Structure): + _fields_ = [('wButtons', ctypes.c_ushort), ('bLeftTrigger', ctypes.c_ubyte), ('bRightTrigger', ctypes.c_ubyte), + ('sThumbLX', ctypes.c_short), ('sThumbLY', ctypes.c_short), ('sThumbRX', ctypes.c_short), + ('sThumbRY', ctypes.c_short)] + + +class _XInputState(ctypes.Structure): + _fields_ = [('dwPacketNumber', ctypes.c_uint), ('Gamepad', _XInputGamepad)] + + +BUTTONS = {0x0001: 'up', 0x0002: 'down', 0x0004: 'left', 0x0008: 'right', 0x1000: 'a', 0x2000: 'b', 0x8000: 'y', + 0x4000: 'x', 0x0010: 'start'} + + +class Gamepad: + """Polls the first XInput pad; poll() returns the buttons pressed since the last call (edges), the left stick + counted as the d-pad.""" + + def __init__(self): + self.dll = None + self.held = set() + if os.name == 'nt': + for name in ('xinput1_4', 'xinput1_3', 'xinput9_1_0'): + try: + self.dll = getattr(ctypes.windll, name) + break + except (AttributeError, OSError): + continue + + def poll(self): + if not self.dll: + return set() + st = _XInputState() + down = set() + for pad in range(4): + try: + if self.dll.XInputGetState(pad, ctypes.byref(st)) == 0: + g = st.Gamepad + down = {name for bit, name in BUTTONS.items() if g.wButtons & bit} + if g.sThumbLY > 16000: + down.add('up') + elif g.sThumbLY < -16000: + down.add('down') + if g.sThumbLX > 16000: + down.add('right') + elif g.sThumbLX < -16000: + down.add('left') + break + except OSError: + return set() + pressed = down - self.held + self.held = down + return pressed + + +def is_windows(): + return os.name == 'nt' + + +def frozen_python(): + return Path(sys.executable).parent diff --git a/tests/launcher_test.py b/tests/launcher_test.py new file mode 100644 index 00000000..3ed1ca1f --- /dev/null +++ b/tests/launcher_test.py @@ -0,0 +1,678 @@ +"""launcher/ and tools/make_game.py's bookkeeping, without game files or Windows. + +Run: python tests/launcher_test.py (Python 3.14 for the zstd tests; the window test needs tkinter, numpy and a display) + +- versions, settings +- toolchain: the package database, dependency resolution and an install from a local fake repository +- net: downloads that resume, check their sha256 and stop on cancel +- feed: GitHub releases (stable, preview, private with a token) and a plain manifest URL, served locally +- updater: staging a release, switching, rollback, pruning, refusing unsafe archives +- gamefiles: the .bfotf font wrapping and zstd dictionaries, on a font of this system made into a fake RomFS +- make_game.py: a synthetic .nup through the check, exefs and romfs steps with --events, skipped steps on a second + run, --rebuild without the game files, and the object cache of the compile step with the system's C compiler +- jobs: an install driven by a fake make_game.py +- the window: --selftest draws every screen (skipped without a display) +""" +import hashlib +import http.server +import io +import json +import os +import shutil +import struct +import subprocess +import sys +import tarfile +import tempfile +import threading +import unittest +import zipfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(ROOT / 'tools')) + +from launcher import build, feed, gamefiles, jobs, net, settings, toolchain, updater, versions # noqa: E402 +from launcher.context import Context # noqa: E402 +from launcher.paths import Layout # noqa: E402 + +try: + from compression import zstd as ZSTD +except ImportError: + ZSTD = None + + +# ---------------------------------------------------------------- helpers + +class Server: + """A local HTTP server for a folder, with extra routes (path -> (status, headers, body)) and Range support.""" + + def __init__(self, folder, routes=None): + folder, routes = Path(folder), routes if routes is not None else {} + self.hits = [] + server = self + + class H(http.server.BaseHTTPRequestHandler): + def log_message(self, *a): + pass + + def do_GET(self): + server.hits.append((self.path, dict(self.headers))) + if self.path in routes: + status, headers, body = routes[self.path] + self.send_response(status) + for k, v in headers.items(): + self.send_header(k, v) + self.send_header('Content-Length', str(len(body))) + self.end_headers() + self.wfile.write(body) + return + p = folder / self.path.lstrip('/').split('?')[0] + if not p.is_file(): + self.send_error(404) + return + data = p.read_bytes() + rng = self.headers.get('Range') + if rng and rng.startswith('bytes='): + start = int(rng[6:].split('-')[0]) + if start >= len(data): + self.send_error(416) + return + self.send_response(206) + self.send_header('Content-Range', f'bytes {start}-{len(data) - 1}/{len(data)}') + data = data[start:] + else: + self.send_response(200) + self.send_header('Content-Length', str(len(data))) + self.end_headers() + try: + self.wfile.write(data) + except ConnectionError: # the client stopped reading (the cancel test) + pass + + self.httpd = http.server.ThreadingHTTPServer(('127.0.0.1', 0), H) + self.url = f'http://127.0.0.1:{self.httpd.server_address[1]}/' + threading.Thread(target=self.httpd.serve_forever, daemon=True).start() + + def close(self): + self.httpd.shutdown() + + +def sha(data): + return hashlib.sha256(data).hexdigest() + + +def tar_bytes(files, compress=True): + """A tar (zstd-compressed when the stdlib has zstd, else gzip) of {name: bytes}.""" + buf = io.BytesIO() + mode = 'w:gz' if not (compress and ZSTD) else 'w' + with tarfile.open(fileobj=buf, mode=mode) as t: + for name, data in files.items(): + info = tarfile.TarInfo(name) + info.size = len(data) + t.addfile(info, io.BytesIO(data)) + data = buf.getvalue() + return ZSTD.compress(data) if compress and ZSTD else data + + +def desc(name, version, filename, data, depends=(), provides=()): + lines = ['%FILENAME%', filename, '', '%NAME%', name, '', '%VERSION%', version, '', '%CSIZE%', str(len(data)), '', + '%ISIZE%', str(len(data) * 3), '', '%SHA256SUM%', sha(data), ''] + if depends: + lines += ['%DEPENDS%', *depends, ''] + if provides: + lines += ['%PROVIDES%', *provides, ''] + return '\n'.join(lines).encode() + + +def fake_repo(folder, damage=None): + """A pacman repository with the ten root packages (each holding one of make_game.py's required files), one + dependency and one name provided by another package.""" + P = toolchain.PREFIX + files = {'clang': {'ucrt64/bin/clang.exe': b'clang', 'ucrt64/bin/clang++.exe': b'clang++'}, + 'sdl3': {'ucrt64/bin/SDL3.dll': b'sdl', 'ucrt64/lib/libSDL3.dll.a': b'implib'}, + 'vulkan-memory-allocator': {'ucrt64/include/vma/vk_mem_alloc.h': b'// vma'}, + 'vulkan-loader': {'ucrt64/lib/libvulkan-1.dll.a': b'implib'}, + 'llvm-libs': {'ucrt64/bin/libLLVM.dll': b'llvm', 'ucrt64/share/doc/llvm/README': b'skipped'}} + deps = {'clang': [P + 'llvm-libs=23.1', P + 'gcc'], 'sdl3': [P + 'vulkan']} + entries = {} + for root in list(toolchain.ROOTS) + ['llvm-libs']: + name = P + root + fn = f'{name}-1.0-1-any.pkg.tar.zst' + data = tar_bytes({'.PKGINFO': b'pkg', **files.get(root, {f'ucrt64/share/{root}/x': b'x'})}) + if damage == root: + (folder / fn).write_bytes(data[:-3] + b'bad') + else: + (folder / fn).write_bytes(data) + entries[f'{name}-1.0-1/desc'] = desc(name, '1.0-1', fn, data, deps.get(root, ()), + [P + 'vulkan'] if root == 'vulkan-loader' else ()) + (folder / 'ucrt64.db').write_bytes(tar_bytes(entries)) + + +class Tmp(unittest.TestCase): + def setUp(self): + self.tmp = Path(tempfile.mkdtemp(prefix='nx2navi-test-')) + self.addCleanup(shutil.rmtree, self.tmp, True) + self.layout = Layout(root=self.tmp / 'launcher', app=ROOT).ensure() + + +# ---------------------------------------------------------------- versions, settings + +class VersionTest(unittest.TestCase): + def test_order(self): + self.assertTrue(versions.newer('0.3.11', '0.3.10')) + self.assertTrue(versions.newer('0.4', '0.3.99')) + self.assertFalse(versions.newer('0.3.10+abc', '0.3.10')) + self.assertTrue(versions.newer('1.0.0', '1.0.0-rc1')) + self.assertEqual(versions.display('0.3.11+abc1234'), '0.3.11') + with self.assertRaises(ValueError): + versions.parse('banana') + + +class SettingsTest(Tmp): + def test_roundtrip_and_reset(self): + s = settings.Settings(self.layout.settings) + self.assertTrue(s['auto_check']) + s['auto_check'], s['game_dir'], s['channel'] = False, 'D:/g', 'preview' + s.save() + t = settings.Settings(self.layout.settings) + self.assertEqual((t['auto_check'], t['game_dir'], t['channel']), (False, 'D:/g', 'preview')) + t.reset() + self.assertEqual((t['auto_check'], t['game_dir'], t['channel']), (True, 'D:/g', 'stable')) + + def test_bad_types_ignored(self): + self.layout.settings.write_text(json.dumps({'auto_check': 'yes', 'channel': 'preview', 'junk': 1})) + s = settings.Settings(self.layout.settings) + self.assertEqual((s['auto_check'], s['channel']), (True, 'preview')) + self.assertNotIn('junk', s) + + +# ---------------------------------------------------------------- toolchain + +class ToolchainTest(Tmp): + def test_resolve_and_install(self): + repo = self.tmp / 'repo' + repo.mkdir() + fake_repo(repo) + srv = Server(repo) + self.addCleanup(srv.close) + lock = toolchain.make_lock(srv.url) + names = [p['name'] for p in lock['packages']] + self.assertEqual(names[0], toolchain.PREFIX + 'clang') + self.assertIn(toolchain.PREFIX + 'llvm-libs', names) # a dependency with a version constraint + self.assertEqual(names.count(toolchain.PREFIX + 'vulkan-loader'), 1) # 'vulkan' resolves to its provider + self.assertFalse(toolchain.is_current(self.layout, lock)) + phases = [] + toolchain.install(self.layout, lock, progress=lambda ph, d, t: phases.append(ph), log=lambda m: None) + self.assertTrue(toolchain.is_current(self.layout, lock)) + self.assertTrue((self.layout.msys / 'bin' / 'clang.exe').is_file()) + self.assertFalse((self.layout.msys / 'share' / 'doc').exists()) # documentation left out + self.assertIn('download', phases) + self.assertIn('unpack', phases) + hits = len(srv.hits) + toolchain.install(self.layout, lock, log=lambda m: None) # cached package files: no download + self.assertEqual(len(srv.hits), hits) + self.assertEqual(toolchain.lock_digest(lock), toolchain.installed(self.layout)['digest']) + + def test_damaged_package_refused(self): + repo = self.tmp / 'repo' + repo.mkdir() + fake_repo(repo, damage='opus') + srv = Server(repo) + self.addCleanup(srv.close) + lock = toolchain.make_lock(srv.url) + with self.assertRaises(toolchain.ToolchainError): + toolchain.install(self.layout, lock, log=lambda m: None) + self.assertFalse(toolchain.is_current(self.layout, lock)) + + def test_wheel_tag(self): + tag = toolchain.wheel_tag()[0] + self.assertTrue(tag.startswith(f'cp{sys.version_info.major}{sys.version_info.minor}-')) + + +# ---------------------------------------------------------------- net + +class NetTest(Tmp): + def test_download_resume_and_checks(self): + data = os.urandom(3 * net.CHUNK + 123) + (self.tmp / 'f.bin').write_bytes(data) + srv = Server(self.tmp) + self.addCleanup(srv.close) + dest = self.tmp / 'out' / 'f.bin' + dest.parent.mkdir() + (dest.parent / 'f.bin.part').write_bytes(data[:1000]) # a part file from a cut connection + net.download(srv.url + 'f.bin', dest, sha256=sha(data), size=len(data)) + self.assertEqual(dest.read_bytes(), data) + self.assertEqual(srv.hits[-1][1].get('Range'), 'bytes=1000-') + with self.assertRaises(net.DownloadError): + net.download(srv.url + 'f.bin', self.tmp / 'out' / 'g.bin', sha256='0' * 64) + self.assertFalse((self.tmp / 'out' / 'g.bin').exists()) + with self.assertRaises(net.Cancelled): + net.download(srv.url + 'f.bin', self.tmp / 'out' / 'h.bin', cancel=lambda: True) + with self.assertRaises(net.DownloadError): + net.download(srv.url + 'missing.bin', self.tmp / 'out' / 'm.bin') + + +# ---------------------------------------------------------------- feed and updater + +def make_app_zip(folder, ver, extra=None): + app = folder / f'app-{ver}' + (app / 'launcher').mkdir(parents=True) + (app / 'launcher' / 'start.pyw').write_text('# test\n') + (app / 'VERSION').write_text(ver + '\n') + z = folder / f'NX2navi-app-{ver}.zip' + with zipfile.ZipFile(z, 'w') as zf: + for p in app.rglob('*'): + if p.is_file(): + zf.write(p, p.relative_to(app).as_posix()) + for name, data in (extra or {}).items(): + zf.writestr(name, data) + return z + + +def manifest_for(z, ver): + return {'format': 1, 'version': ver, 'notes': '- One\n- Two', + 'app': {'name': z.name, 'size': z.stat().st_size, 'sha256': sha(z.read_bytes())}} + + +class FeedTest(Tmp): + def serve_github(self): + self.routes = {} + srv = Server(self.tmp, self.routes) + self.addCleanup(srv.close) + return srv + + def publish(self, releases): + self.routes['/repos/o/r/releases?per_page=20'] = (200, {'Content-Type': 'application/json'}, + json.dumps(releases).encode()) + + def release(self, srv, tag, ver, pre=False, manifest=True): + z = make_app_zip(self.tmp, ver) + m = self.tmp / f'm-{ver}.json' + m.write_text(json.dumps(manifest_for(z, ver))) + assets = [{'name': z.name, 'browser_download_url': srv.url + z.name, 'url': srv.url + z.name}] + if manifest: + assets.append({'name': feed.MANIFEST, 'browser_download_url': srv.url + m.name, 'url': srv.url + m.name}) + return {'tag_name': tag, 'prerelease': pre, 'draft': False, 'body': 'notes', 'html_url': 'x', 'assets': assets} + + def test_github_channels(self): + srv = self.serve_github() + self.publish([self.release(srv, 'v0.4.0-rc1', '0.4.0-rc1', pre=True), + self.release(srv, 'v0.3.12', '0.3.12', manifest=False), self.release(srv, 'v0.3.11', '0.3.11')]) + f = {'type': 'github', 'repo': 'o/r', 'api': srv.url.rstrip('/')} + self.assertEqual(feed.latest(f, 'stable').version, '0.3.11') # 0.3.12 lacks a manifest + self.assertEqual(feed.latest(f, 'preview').version, '0.4.0-rc1') + r = feed.latest(f, 'stable', token='secret') + self.assertEqual(r.headers.get('Authorization'), 'Bearer secret') + self.assertTrue(any(h.get('Authorization') == 'Bearer secret' for _, h in srv.hits)) + self.assertEqual(feed.plain_notes('## Head\n- **One**\n* `Two`\n'), ['Head', 'One', 'Two']) + + def test_url_feed_and_update(self): + z = make_app_zip(self.tmp, '0.9.0') + (self.tmp / feed.MANIFEST).write_text(json.dumps(manifest_for(z, '0.9.0'))) + srv = Server(self.tmp) + self.addCleanup(srv.close) + rel = feed.latest({'type': 'url', 'url': srv.url + feed.MANIFEST}) + self.assertEqual(rel.version, '0.9.0') + self.layout.apps.mkdir(parents=True) + (self.layout.apps / 'current.txt').write_text('0.8.0\n') + old = self.layout.apps / '0.8.0' / 'launcher' + old.mkdir(parents=True) + (old / 'start.pyw').write_text('') + for stale in ('0.7.0', '0.6.0'): + (self.layout.apps / stale).mkdir() + folder = updater.stage_app(self.layout, rel) + self.assertTrue((folder / 'launcher' / 'start.pyw').is_file()) + updater.switch_to(self.layout, '0.9.0') + self.assertEqual((updater.current(self.layout), updater.previous(self.layout)), ('0.9.0', '0.8.0')) + updater.prune(self.layout) + self.assertEqual(sorted(p.name for p in self.layout.apps.iterdir() if p.is_dir()), ['0.8.0', '0.9.0']) + self.assertTrue(updater.rollback(self.layout)) + self.assertEqual(updater.current(self.layout), '0.8.0') + self.assertFalse(updater.rollback(self.layout)) # nothing further back + + def test_unsafe_or_wrong_archives(self): + z = make_app_zip(self.tmp, '1.0.0', extra={'../evil.txt': 'x'}) + with self.assertRaises(updater.UpdateError): + updater.safe_extract(z, self.tmp / 'x') + self.assertFalse((self.tmp / 'evil.txt').exists()) + z2 = make_app_zip(self.tmp, '1.0.1') + m = manifest_for(z2, '1.0.2') # the archive says 1.0.1 + (self.tmp / feed.MANIFEST).write_text(json.dumps(m)) + srv = Server(self.tmp) + self.addCleanup(srv.close) + rel = feed.Release(m, assets={z2.name: srv.url + z2.name}) + with self.assertRaises(updater.UpdateError): + updater.stage_app(self.layout, rel) + self.assertFalse((self.layout.apps / '1.0.2').exists()) + + +# ---------------------------------------------------------------- gamefiles + +def sarc(files): + names = b'' + offs = [] + for n in files: + offs.append(len(names)) + b = n.encode() + b'\0' + names += b + b'\0' * (-len(b) % 4) + data, ents = b'', b'' + for (n, d), no in zip(files.items(), offs): + start = len(data) + data += d + ents += struct.pack('{len(body) // 4}I', body) + return struct.pack('>II', 0x7F9A0218 ^ key, len(otf) ^ key) + struct.pack(f'>{len(words)}I', + *(w ^ key for w in words)) + + +@unittest.skipUnless(ZSTD, 'needs compression.zstd (Python 3.14)') +class FontTest(Tmp): + FONT = Path('/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf') + + @unittest.skipUnless(FONT.is_file(), 'needs DejaVu Sans') + def test_font_from_romfs_image(self): + otf = self.FONT.read_bytes() + self.assertEqual(gamefiles.bfotf_to_otf(bfotf(otf)), otf) + self.assertIsNone(gamefiles.bfotf_to_otf(b'\0' * 64)) + samples = [otf[i:i + 512] for i in range(0, min(len(otf), 400000), 997)] + d = ZSTD.train_dict(samples, 16384) + packed = ZSTD.compress(bfotf(otf), zstd_dict=d) + self.assertNotEqual(ZSTD.get_frame_info(packed).dictionary_id, 0) + files = {'Pack/ZsDic.pack.zs': ZSTD.compress(sarc({'zs.zsdic': d.dict_content})), + gamefiles.FONTS['title']: packed} + romfs = self.tmp / 'game' / 'romfs' + romfs.mkdir(parents=True) + entries, blob = [], b'' + for path, data in sorted(files.items()): + blob += b'\0' * (-len(blob) % 16) + entries.append({'path': path, 'offset': len(blob), 'size': len(data)}) + blob += data + (romfs / 'romfs.bin').write_bytes(blob) + (romfs / 'index.json').write_text(json.dumps({'files': entries})) + got = gamefiles.extract_fonts(self.layout, self.tmp / 'game') + self.assertEqual(got['title'][1], 'DejaVu Sans') + self.assertEqual(Path(got['title'][0]).read_bytes(), otf) + self.assertEqual(gamefiles.cached_fonts(self.layout)['title'][1], 'DejaVu Sans') + + +# ---------------------------------------------------------------- make_game.py + +def nso(build_id=b'\0' * 16, size=0x300): + head = bytearray(0x100) + head[0:4] = b'NSO0' + segs = [(0x100 + i * size, i * 0x10000, size) for i in range(3)] + for i, (fo, mo, sz) in enumerate(segs): + struct.pack_into('/build-cache/state.json): +Steps, each skipped when a run before finished it for the same inputs (state in /build-cache/state.json): 1. check the title (0400486024DEC000) and main's build id (game 1.1.1) 2. exefs the five ExeFS files, every module's segments unpacked (ZBIC through the player's own `zbic` package) and stored plainly, as the runtime's loader reads them -> /exefs @@ -16,15 +17,28 @@ Steps, each skipped when a run before finished it for the same input (state in < 6. shaders every shader in the game files (tools/prepare_shaders.py, default bind state) translated, and each one's GPU driver build done once, so play does not stop to compile them (`ocarina.exe --prepare-shaders`) +The exefs and romfs steps depend only on the game files; lift and compile depend on this kit's code too. A newer kit +therefore lifts and compiles again but keeps the unpacked game files, and --rebuild needs no game files at all. The +compile step keeps every object with the dependency list clang wrote for it (build-cache/obj/objects.json) and +compiles a file again only when it, a header it includes, the flags or the compiler changed. The new ocarina.exe +replaces the old one only once it links, so a failed build leaves the last good one in place. + Nothing is written outside --out. The game folder then holds only what this PC built from the player's files: never share it. Needs Python 3.10+ with numpy, lz4, zstandard and zbic (pip), and MSYS2 UCRT64's clang and libraries -(README-BuildKit.md lists the commands). Delete /build-cache afterwards to free about 5 GB. +(README-BuildKit.md lists the commands). Delete /build-cache afterwards to free about 5 GB; the next build then +compiles everything again. + +--events adds one machine-readable line per step and progress change to stdout, each starting with "@@" and followed +by a JSON object (launcher/build.py reads them): {"ev": "step", "step": ..., "state": "start" | "done" | "skip"}, +{"ev": "progress", "step": ..., "done": n, "total": n, "unit": ...}, {"ev": "phase", ...}, {"ev": "error", "message": +...} and {"ev": "end", ...}. """ import argparse import concurrent.futures import hashlib import json import os +import re import shutil import struct import subprocess @@ -41,7 +55,8 @@ MAIN_BUILD_ID = 'e0f3ccfaf50065ae4ea01eb5aeb13b1a' # main of game 1.1.1 EXEFS_FILES = ('main', 'main.npdm', 'sdk', 'subsdk0', 'rtld') MODULES = ('main', 'sdk', 'subsdk0', 'rtld') ALIGN = 16 -STATE_VERSION = 1 +STATE_VERSION = 2 +STEPS = ('exefs', 'romfs', 'lift', 'compile', 'shaders') # Bring-up fixes in the lifted SDK, each anchored on exact generated text that must occur once. LIFT_FIXES = [ @@ -61,17 +76,62 @@ LIFT_FIXES = [ LINK_LIBS = ['-lvulkan-1', '-lSDL3', '-lopus', '-lvpx', '-l:libdav1d.a', '-lzstd', '-l:libatomic.a', '-lbcrypt', '-limm32', '-luuid', '-lole32', '-luser32', '-lshell32', '-lgdi32', '-ladvapi32', '-lwinmm'] SYSTEM_DLLS_SKIP = {'vulkan-1.dll'} # the graphics driver's loader, never a copy +EXE_NAME = 'ocarina.exe' + +# Runs a script the way `python script.py` does, with the script's folder first on sys.path. The launcher's +# embeddable Python leaves the script folder off sys.path (its ._pth file), so every script this build starts goes +# through this. +RUN_SCRIPT = ('import os, runpy, sys; s = sys.argv[1]; sys.argv = sys.argv[1:]; ' + 'sys.path.insert(0, os.path.dirname(os.path.abspath(s))); runpy.run_path(s, run_name="__main__")') + +EVENTS = False def say(msg): print(msg, flush=True) +def emit(ev, **fields): + """One "@@{json}" line for the launcher (--events); nothing otherwise.""" + if EVENTS: + print('@@' + json.dumps({'ev': ev, **fields}, ensure_ascii=False), flush=True) + + def fail(msg): + emit('error', message=msg) print(f'\nerror: {msg}', file=sys.stderr, flush=True) sys.exit(1) +def py_cmd(script, *args): + """The command line that runs a Python script of this kit with this interpreter (see RUN_SCRIPT).""" + flags = ['-I'] if sys.flags.isolated else [] + return [sys.executable, *flags, '-c', RUN_SCRIPT, str(script), *map(str, args)] + + +def sha256_bytes(data): + return hashlib.sha256(data).hexdigest() + + +def sha256_file(path): + h = hashlib.sha256() + with open(path, 'rb') as f: + while b := f.read(8 << 20): + h.update(b) + return h.hexdigest() + + +def tree_digest(root, patterns): + """sha256 over the relative names and contents of the files under root matching the glob patterns.""" + h = hashlib.sha256() + files = sorted({p for pat in patterns for p in Path(root).glob(pat) if p.is_file() and '__pycache__' not in p.parts}) + for p in files: + h.update(p.relative_to(root).as_posix().encode() + b'\0') + h.update(p.read_bytes()) + h.update(b'\0') + return h.hexdigest() + + # ---------------------------------------------------------------- input class Source: @@ -137,6 +197,27 @@ class Source: yield b +class BuiltSource: + """--rebuild: the ExeFS and RomFS an earlier run unpacked into the game folder, under that run's input identity. + The ExeFS modules are stored plainly there but keep their headers, so check() reads the same build id.""" + + def __init__(self, out, state): + self.out = out + if state.get('version') != STATE_VERSION or not state.get('input'): + fail(f'{out} holds no finished build of this kit to rebuild from: give --nup, or --exefs and --romfs') + for step in ('exefs', 'romfs'): + if step not in state.get('steps', {}): + fail(f'the last build in {out} did not finish its {step} step: give the game files again') + missing = [n for n in EXEFS_FILES if not (out / 'exefs' / n).is_file()] + missing += [f'romfs/{n}' for n in ('romfs.bin', 'index.json') if not (out / 'romfs' / n).is_file()] + if missing: + fail(f'{out} lacks {", ".join(missing)}: give the game files again') + self.identity = state['input'] + + def read_exefs(self, name): + return (self.out / 'exefs' / name).read_bytes() + + def check(src): main = src.read_exefs('main') if main[:4] != b'NSO0': @@ -182,11 +263,16 @@ def flat_nso(raw, name): def step_exefs(src, out): dst = out / 'exefs' dst.mkdir(parents=True, exist_ok=True) - for name in EXEFS_FILES: + for i, name in enumerate(EXEFS_FILES): raw = src.read_exefs(name) data = flat_nso(raw, name) if name in MODULES else raw (dst / name).write_bytes(data) say(f' {name:9} {len(raw) / 1e6:6.1f} MB -> {len(data) / 1e6:6.1f} MB') + emit('progress', step='exefs', done=i + 1, total=len(EXEFS_FILES), unit='files') + + +def exefs_digest(out): + return tree_digest(out / 'exefs', EXEFS_FILES) # ---------------------------------------------------------------- romfs @@ -202,6 +288,7 @@ def step_romfs(src, out, jobs): say(f' {len(files)} files, {total / 1e9:.1f} GB') entries, h, pos, done, t0 = [], hashlib.sha256(), 0, 0, time.time() part = dst / 'romfs.bin.part' + emit('progress', step='romfs', done=0, total=total, unit='bytes') def load(e): # small files are read whole on a worker; big ones are streamed by the writer rel, ref = e @@ -224,6 +311,7 @@ def step_romfs(src, out, jobs): h.update(chunk) pos += size done += size + emit('progress', step='romfs', done=done, total=total, unit='bytes') if start // batch % 40 == 0: say(f' {done / 1e9:5.1f} of {total / 1e9:.1f} GB, {time.time() - t0:.0f} s') part.replace(dst / 'romfs.bin') @@ -234,13 +322,31 @@ def step_romfs(src, out, jobs): # ---------------------------------------------------------------- lift +EMITTED = re.compile(r'emitted (\d+)/(\d+) chunks') + + +def lift_digest(): + """The lifter's code, the patch table and the bring-up fixes: what the lifted C depends on besides the ExeFS.""" + h = hashlib.sha256(tree_digest(KIT / 'recomp', ('*.py', 'patches-v111.txt')).encode()) + h.update(repr(LIFT_FIXES).encode()) + return h.hexdigest() + + def step_lift(out, cache): + """gen.py into build-cache/lift. The folder is kept: gen.py rewrites only the chunks whose text changed and + removes the ones it no longer writes.""" lift = cache / 'lift' - if lift.exists(): - shutil.rmtree(lift) - cmd = [sys.executable, str(KIT / 'recomp' / 'gen.py'), '--exefs', str(out / 'exefs'), '--out', str(lift), - '--patches', str(KIT / 'recomp' / 'patches-v111.txt')] - if subprocess.run(cmd).returncode: + cmd = py_cmd(KIT / 'recomp' / 'gen.py', '--exefs', out / 'exefs', '--out', lift, + '--patches', KIT / 'recomp' / 'patches-v111.txt') + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding='utf-8', + errors='replace', bufsize=1) + for line in p.stdout: + line = line.rstrip('\n') + say(f' {line}') + m = EMITTED.search(line) + if m: + emit('progress', step='lift', done=int(m.group(1)), total=int(m.group(2)), unit='chunks') + if p.wait(): fail('the code generator failed (see above)') for mod, old, new in LIFT_FIXES: hits = [p for p in sorted((lift / mod).glob('*.c')) if old in p.read_text(encoding='utf-8')] @@ -252,14 +358,6 @@ def step_lift(out, cache): # ---------------------------------------------------------------- compile -def sha256_file(path): - h = hashlib.sha256() - with open(path, 'rb') as f: - while b := f.read(8 << 20): - h.update(b) - return h.hexdigest() - - def modules_c(lift, out, gen): layout = json.loads((lift / 'layout.json').read_text(encoding='utf-8')) L = ['#include "dispatch.h"', '#include "host.h"', '#include "layout.h"', ''] @@ -286,12 +384,17 @@ def modules_c(lift, out, gen): 'const int rt_nvn_api_version[2] = {231, 40}; /* NVN2: main asks for major 231, minor 40 or later */', f'const char rt_port_version[] = {json.dumps(kit_version())};'] p = gen / 'modules.c' - p.write_text('\n'.join(L) + '\n', encoding='utf-8') - (gen / 'shim.c').write_text('/* Defined by the SDL window path only; this build uses the Win32 one. */\n' - 'void menu_sdl_event(void *e) { (void)e; }\n', encoding='utf-8') + write_if_changed(p, '\n'.join(L) + '\n') + write_if_changed(gen / 'shim.c', '/* Defined by the SDL window path only; this build uses the Win32 one. */\n' + 'void menu_sdl_event(void *e) { (void)e; }\n') return p +def write_if_changed(path, text): + if not path.is_file() or path.read_text(encoding='utf-8') != text: + path.write_text(text, encoding='utf-8') + + def kit_version(): v = KIT / 'VERSION' return v.read_text(encoding='utf-8').strip() if v.is_file() else '0.0.1-ocarina' @@ -323,6 +426,23 @@ def pe_imports(path): return names +def copy_if_changed(src, dst): + """Copy src over dst unless dst already has its size and modification time. A file the running game holds open + cannot be replaced on Windows: that stops the build with a message instead of a traceback.""" + s = os.stat(src) + try: + d = os.stat(dst) + if d.st_size == s.st_size and int(d.st_mtime) == int(s.st_mtime): + return False + except FileNotFoundError: + pass + try: + shutil.copy2(src, dst) + except PermissionError: + fail(f'{dst} is in use: close the game and try again') + return True + + def copy_dlls(exe, msys_bin, out): seen, todo, copied = set(), [exe], [] while todo: @@ -331,21 +451,136 @@ def copy_dlls(exe, msys_bin, out): if name in seen or name in SYSTEM_DLLS_SKIP or not src.is_file(): continue seen.add(name) - shutil.copy2(src, out / src.name) + copy_if_changed(src, out / src.name) copied.append(src.name) todo.append(src) return copied -def step_compile(out, cache, msys, jobs): - lift, gen, obj = cache / 'lift', cache / 'gen', cache / 'obj' +def parse_deps(text): + """The prerequisites of a make-style dependency file as clang -MMD writes it (the target dropped).""" + text = text.replace('\\\r\n', ' ').replace('\\\n', ' ') + tokens, cur, i = [], '', 0 + while i < len(text): + c = text[i] + if c == '\\' and i + 1 < len(text) and text[i + 1] in ' #': + cur += text[i + 1] + i += 2 + continue + if c == '$' and text[i + 1:i + 2] == '$': + cur += '$' + i += 2 + continue + if c.isspace(): + if cur: + tokens.append(cur) + cur = '' + else: + cur += c + i += 1 + if cur: + tokens.append(cur) + for k, t in enumerate(tokens): # the target ends with ':' (a drive letter's colon is followed by a separator) + if t.endswith(':'): + return tokens[k + 1:] + return tokens + + +class ObjectCache: + """Which objects are current: per object the key of its command and source, and the sha256 of every file its + dependency list names (build-cache/obj/objects.json).""" + + def __init__(self, path): + self.path = path + try: + self.entries = json.loads(path.read_text(encoding='utf-8')) if path.is_file() else {} + except (OSError, ValueError): + self.entries = {} + self._sha = {} + + def file_sha(self, p): + p = os.path.normcase(os.path.abspath(p)) + if p not in self._sha: + try: + self._sha[p] = sha256_file(p) + except OSError: + self._sha[p] = None + return self._sha[p] + + def current(self, obj, key): + e = self.entries.get(obj.name) + return (e is not None and e.get('key') == key and obj.is_file() + and all(self.file_sha(p) == h for p, h in e.get('deps', {}).items())) + + def record(self, obj, key, dep_file, src): + try: + deps = parse_deps(Path(dep_file).read_text(encoding='utf-8', errors='replace')) + except OSError: + deps = [] + deps = deps or [str(src)] + self.entries[obj.name] = {'key': key, 'deps': {p: self.file_sha(p) for p in deps}} + + def forget(self, obj): + self.entries.pop(obj.name, None) + + def save(self): + tmp = self.path.with_suffix('.tmp') + tmp.write_text(json.dumps(self.entries), encoding='utf-8') + tmp.replace(self.path) + + +def compiler_id(clang, env): + p = subprocess.run([str(clang), '--version'], capture_output=True, text=True, env=env) + st = os.stat(clang) + return sha256_bytes(f'{p.stdout}\n{st.st_size}\n{int(st.st_mtime)}'.encode()) + + +def compile_all(srcs, cmd_for, cache, jobs, env, progress=None, salt=''): + """Compile each (source, object) whose object is not current in cache. cmd_for(src, dst) -> argv without the + dependency-file options; salt identifies the compiler. Returns (compiled, reused, failures) with failures as + (src, stderr tail).""" + todo, keys = [], {} + for src, dst in srcs: + cmd = cmd_for(src, dst) + flags = [c for c in cmd if c not in (str(src), str(dst))] + keys[dst] = sha256_bytes(json.dumps([salt, flags, cache.file_sha(src)]).encode()) + if not cache.current(dst, keys[dst]): + todo.append((src, dst)) + reused = len(srcs) - len(todo) + + def cc(job): + src, dst = job + dep = dst.with_suffix('.d') + cmd = cmd_for(src, dst) + cmd[1:1] = ['-MMD', '-MF', str(dep)] + p = subprocess.run(cmd, capture_output=True, text=True, env=env, errors='replace') + return src, dst, dep, p.returncode, p.stderr[-1500:] + + done, bad = reused, [] + if progress: + progress(done, len(srcs)) + with concurrent.futures.ThreadPoolExecutor(max(1, jobs)) as ex: + for src, dst, dep, rc, err in ex.map(cc, todo): + done += 1 + if rc: + bad.append((src, err)) + cache.forget(dst) + else: + cache.record(dst, keys[dst], dep, src) + if progress: + progress(done, len(srcs)) + cache.save() + return len(todo), reused, bad + + +def step_compile(out, cache_dir, msys, jobs): + lift, gen, obj = cache_dir / 'lift', cache_dir / 'gen', cache_dir / 'obj' gen.mkdir(parents=True, exist_ok=True) obj.mkdir(parents=True, exist_ok=True) rt = KIT / 'runtime' clang, clangxx = msys / 'bin' / 'clang.exe', msys / 'bin' / 'clang++.exe' env = dict(os.environ, PATH=str(msys / 'bin') + os.pathsep + os.environ.get('PATH', '')) - if subprocess.run([sys.executable, str(KIT / 'tools' / 'romfs_index.py'), '--romfs', str(out / 'romfs'), - '--out', str(gen)]).returncode: + if subprocess.run(py_cmd(KIT / 'tools' / 'romfs_index.py', '--romfs', out / 'romfs', '--out', gen)).returncode: fail('the RomFS index could not be written') srcs = [(p, obj / f'{mod}_{p.stem}.o') for mod in ('main', 'sdk', 'subsdk0') for p in sorted((lift / mod).glob('*.c'))] for pat in ('*.c', '*.cpp', 'platform/*.c', 'ui/*.cpp', 'third_party/imgui/*.cpp', 'third_party/imgui/backends/*.cpp'): @@ -356,53 +591,74 @@ def step_compile(out, cache, msys, jobs): srcs += [(modules_c(lift, out, gen), obj / 'modules.o'), (gen / 'shim.c', obj / 'shim.o')] inc = ['-I', str(rt), '-I', str(lift), '-I', str(gen), '-I', str(rt / 'third_party' / 'imgui'), '-I', str(msys / 'include' / 'vma')] + cid = compiler_id(clang, env) - def cc(job): - src, dst = job + def cmd_for(src, dst): cpp = src.suffix in ('.cpp', '.cxx') # the lifted code at -O1 (its size makes -O2 slow to build), the runtime at -O2; neither may assume strict aliasing: # the lifted code reaches guest memory through casts of one address to several types (tools/build_game.py's flags) guest = dst.name.startswith(('main_', 'sdk_', 'subsdk0_')) - cmd = [str(clangxx if cpp else clang), '-c', '-std=gnu++17' if cpp else '-std=gnu11', '-O1' if guest else '-O2', - '-fno-strict-aliasing', '-w', *inc, '-o', str(dst), str(src)] - p = subprocess.run(cmd, capture_output=True, text=True, env=env) - return src, p.returncode, p.stderr[-1500:] + return [str(clangxx if cpp else clang), '-c', '-std=gnu++17' if cpp else '-std=gnu11', '-O1' if guest else '-O2', + '-fno-strict-aliasing', '-w', *inc, '-o', str(dst), str(src)] + + t0 = time.time() + last = [0.0] + + def progress(done, total): + now = time.time() + if done == total or now - last[0] >= 0.25: + last[0] = now + emit('progress', step='compile', done=done, total=total, unit='files') + if done % 50 == 0 or done == total: + say(f' compiled {done} of {total}, {now - t0:.0f} s') say(f' {len(srcs)} files on {jobs} threads') - t0, done, bad = time.time(), 0, [] - with concurrent.futures.ThreadPoolExecutor(jobs) as ex: - for src, rc, err in ex.map(cc, srcs): - done += 1 - if rc: - bad.append((src, err)) - if done % 50 == 0 or done == len(srcs): - say(f' compiled {done} of {len(srcs)}, {time.time() - t0:.0f} s') + cache = ObjectCache(obj / 'objects.json') + compiled, reused, bad = compile_all(srcs, cmd_for, cache, jobs, env, progress, salt=cid) + say(f' {compiled} compiled, {reused} unchanged since the last build') if bad: for src, err in bad[:3]: print(f'--- {src}\n{err}', file=sys.stderr) fail(f'{len(bad)} files did not compile') + # the manifest (per-monitor DPI awareness: the picture at the monitor's real size, not one Windows stretches) and the # icon, runtime/nx2navi.rc through MSYS2's windres rco = obj / 'nx2navi_rc.o' r = subprocess.run([str(msys / 'bin' / 'windres.exe'), '-I', str(rt), '-O', 'coff', '-i', str(rt / 'nx2navi.rc'), - '-o', str(rco)], capture_output=True, text=True, env=env) + '-o', str(rco)], capture_output=True, text=True, env=env, errors='replace') if r.returncode: fail(f'windres failed on runtime/nx2navi.rc: {r.stderr.strip()[:400]}') - rsp = cache / 'link.rsp' - rsp.write_text('\n'.join('"%s"' % o.as_posix() for o in [o for _, o in srcs] + [rco]), encoding='utf-8') - exe = out / 'ocarina.exe' - say(' linking') - p = subprocess.run([str(clangxx), '-o', str(exe), '@' + str(rsp), '-L', str(msys / 'lib'), *LINK_LIBS], - capture_output=True, text=True, env=env) - if p.returncode: - print(p.stderr[-4000:], file=sys.stderr) - fail('the link failed') - shutil.copy2(gen / 'romfs-index.bin', out / 'romfs-index.bin') + objs = [o for _, o in srcs] + [rco] + exe = out / EXE_NAME + rsp = cache_dir / 'link.rsp' + rsp.write_text('\n'.join('"%s"' % o.as_posix() for o in objs), encoding='utf-8') + link_key = sha256_bytes(json.dumps([LINK_LIBS, cid] + [[o.name, o.stat().st_size, o.stat().st_mtime_ns] + for o in objs]).encode()) + state_link = cache_dir / 'link.key' + if exe.is_file() and state_link.is_file() and state_link.read_text(encoding='utf-8') == link_key: + say(' ocarina.exe is current, not linked again') + else: + emit('phase', step='compile', phase='link') + say(' linking') + tmp = out / (EXE_NAME + '.new') + p = subprocess.run([str(clangxx), '-o', str(tmp), '@' + str(rsp), '-L', str(msys / 'lib'), *LINK_LIBS], + capture_output=True, text=True, env=env, errors='replace') + if p.returncode: + print(p.stderr[-4000:], file=sys.stderr) + tmp.unlink(missing_ok=True) + fail('the link failed') + try: + os.replace(tmp, exe) + except PermissionError: + tmp.unlink(missing_ok=True) + fail(f'{exe} is in use: close the game and try again') + state_link.write_text(link_key, encoding='utf-8') + copy_if_changed(gen / 'romfs-index.bin', out / 'romfs-index.bin') for name in ('nx2navi_xlate.exe', 'nx2navi_xlate.id'): - shutil.copy2(KIT / 'bin' / name, out / name) + copy_if_changed(KIT / 'bin' / name, out / name) if (KIT / 'bin' / 'amd_fidelityfx_vk.dll').is_file(): # FSR 3.1 (runtime/vk_ffx.cpp) - shutil.copy2(KIT / 'bin' / 'amd_fidelityfx_vk.dll', out / 'amd_fidelityfx_vk.dll') - say(f' ocarina.exe {exe.stat().st_size / 1e6:.0f} MB; beside it: {", ".join(copy_dlls(exe, msys / "bin", out))}') + copy_if_changed(KIT / 'bin' / 'amd_fidelityfx_vk.dll', out / 'amd_fidelityfx_vk.dll') + say(f' {EXE_NAME} {exe.stat().st_size / 1e6:.0f} MB; beside it: {", ".join(copy_dlls(exe, msys / "bin", out))}') (out / 'profile').mkdir(exist_ok=True) @@ -432,56 +688,111 @@ def keep_awake(): pass +def load_state(path): + try: + state = json.loads(path.read_text(encoding='utf-8')) if path.is_file() else {} + except (OSError, ValueError): + state = {} + if state.get('version') == 1: # the first kit: a list of finished steps, all tied to one kit version + done = state.get('done', []) + state = {'version': STATE_VERSION, 'input': state.get('input'), 'steps': {}} + for step in ('exefs', 'romfs'): + if step in done: + state['steps'][step] = step_key(step, state['input']) + return state + + +def step_key(step, identity, out=None): + """What a step's output depends on. exefs and romfs: the game files only; lift: the unpacked ExeFS and the + lifter; compile: the lift, the runtime sources and this script.""" + if step in ('exefs', 'romfs'): + return f'{step}-1|{identity}' + lift = f'lift|{exefs_digest(out)}|{lift_digest()}' + if step == 'lift': + return lift + compile_key = f'compile|{lift}|{tree_digest(KIT / "runtime", ("**/*",))}|{sha256_file(Path(__file__))}|{kit_version()}' + if step == 'compile': + return compile_key + # shaders: the built game and the translator that prepared them + return f'shaders-1|{compile_key}|{sha256_file(KIT / "tools" / "prepare_shaders.py")}|{sha256_file(KIT / "bin" / "nx2navi_xlate.id")}' + + def main(): + global EVENTS ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) ap.add_argument('--nup', help='the game as one .nup file') ap.add_argument('--exefs', help='the game\'s exefs folder (with --romfs, instead of --nup)') ap.add_argument('--romfs', help='the game\'s romfs folder') + ap.add_argument('--rebuild', action='store_true', + help='build again from the game files the last run unpacked into --out (after a kit update)') ap.add_argument('--out', help='the folder for the finished game') ap.add_argument('--check', action='store_true', help='only check the game\'s version; builds nothing') ap.add_argument('--msys', default=r'C:\msys64\ucrt64', help='MSYS2 UCRT64 folder (default C:\\msys64\\ucrt64)') ap.add_argument('--jobs', type=int, default=os.cpu_count() or 4, help='parallel compiles') + ap.add_argument('--events', action='store_true', help='machine-readable progress lines on stdout ("@@" + JSON)') + ap.add_argument('--until', choices=STEPS, help=argparse.SUPPRESS) # tests: stop after this step a = ap.parse_args() - if bool(a.nup) == bool(a.exefs or a.romfs) or (not a.nup and not (a.exefs and a.romfs)): - ap.error('give --nup, or both --exefs and --romfs') - src = Source(a) + EVENTS = a.events + given = bool(a.nup) + bool(a.exefs or a.romfs) + bool(a.rebuild) + if given != 1 or (not a.nup and not a.rebuild and not (a.exefs and a.romfs)): + ap.error('give --nup, both --exefs and --romfs, or --rebuild') + if a.rebuild and not a.out: + ap.error('--rebuild needs --out') + out = Path(a.out).resolve() if a.out else None + cache = out / 'build-cache' if out else None + state_path = cache / 'state.json' if cache else None + state = load_state(state_path) if state_path else {} + + emit('step', step='check', state='start') + src = BuiltSource(out, state) if a.rebuild else Source(a) check(src) say('check: Ocarina of Time 1.1.1 (build e0f3ccfa)') + emit('check', version='1.1.1', build=MAIN_BUILD_ID[:8]) + emit('step', step='check', state='done') if a.check: return - if not a.out: + if not out: ap.error('--out is needed to build') msys = Path(a.msys) for need in ('bin/clang.exe', 'bin/clang++.exe', 'bin/SDL3.dll', 'lib/libSDL3.dll.a', 'include/vma'): if not (msys / need).exists(): fail(f'{msys / need} is missing: install MSYS2 and the packages README-BuildKit.md lists') - for mod in ('numpy', 'lz4', 'zbic'): + for mod in ('numpy', 'lz4') + (() if a.rebuild else ('zbic',)): try: __import__(mod) except ImportError: fail(f'the Python package {mod} is missing: pip install numpy lz4 zstandard zbic') - out = Path(a.out).resolve() - cache = out / 'build-cache' cache.mkdir(parents=True, exist_ok=True) - state_path = cache / 'state.json' - state = json.loads(state_path.read_text(encoding='utf-8')) if state_path.is_file() else {} - if state.get('input') != src.identity or state.get('version') != STATE_VERSION or state.get('kit') != kit_version(): - state = {'input': src.identity, 'version': STATE_VERSION, 'kit': kit_version(), 'done': []} + if state.get('version') != STATE_VERSION or state.get('input') != src.identity: + state = {'version': STATE_VERSION, 'input': src.identity, 'steps': {}} + state['kit'] = kit_version() keep_awake() t0 = time.time() - for name, run in (('exefs', lambda: step_exefs(src, out)), ('romfs', lambda: step_romfs(src, out, min(a.jobs, 8))), - ('lift', lambda: step_lift(out, cache)), ('compile', lambda: step_compile(out, cache, msys, a.jobs)), - ('shaders', lambda: step_shaders(out))): - if name in state['done']: + runs = {'exefs': lambda: step_exefs(src, out), 'romfs': lambda: step_romfs(src, out, min(a.jobs, 8)), + 'lift': lambda: step_lift(out, cache), 'compile': lambda: step_compile(out, cache, msys, a.jobs), + 'shaders': lambda: step_shaders(out)} + for name in STEPS: + key = step_key(name, src.identity, out) + if state['steps'].get(name) == key: say(f'{name}: done before') - continue - say(f'{name}:') - t = time.time() - run() - state['done'].append(name) - state_path.write_text(json.dumps(state, indent=1), encoding='utf-8') - say(f'{name}: {time.time() - t:.0f} s') - say(f'\nfinished in {(time.time() - t0) / 60:.1f} min: {out / "ocarina.exe"}') + emit('step', step=name, state='skip') + else: + # a step that runs makes every step after it run too: their keys hash its output, not its key + for later in STEPS[STEPS.index(name):]: + state['steps'].pop(later, None) + say(f'{name}:') + emit('step', step=name, state='start') + t = time.time() + runs[name]() + state['steps'][name] = step_key(name, src.identity, out) + state_path.write_text(json.dumps(state, indent=1), encoding='utf-8') + say(f'{name}: {time.time() - t:.0f} s') + emit('step', step=name, state='done', seconds=round(time.time() - t, 1)) + if name == a.until: + break + state_path.write_text(json.dumps(state, indent=1), encoding='utf-8') + say(f'\nfinished in {(time.time() - t0) / 60:.1f} min: {out / EXE_NAME}') + emit('end', exe=str(out / EXE_NAME), seconds=round(time.time() - t0, 1), kit=kit_version()) if __name__ == '__main__': diff --git a/tools/make_launcher.py b/tools/make_launcher.py new file mode 100644 index 00000000..317c9fe5 --- /dev/null +++ b/tools/make_launcher.py @@ -0,0 +1,301 @@ +"""Build a release of the NX2navi launcher (docs/launcher.md, "Releases"). + + python tools/make_launcher.py --xlate build/xlate [--ffx amd_fidelityfx_vk.dll] [--out build/launcher-release] + [--feed owner/name] + [--notes notes.md] [--publish] + +Needs Python 3.14 (tarfile and zstd for the MSYS2 packages), uv on PATH, network access on the first run, and +either Windows (msiexec, and the C compiler of MSYS2 it downloads itself) or Linux with clang, lld and llvm-windres +(it cross-compiles against the same MSYS2 packages) plus msitools. Downloads are cached in toolchain/launcher-dl/ and +checked against the sha256 in PINS. + +The release folder (--out) holds: + + NX2navi/ the launcher folder exactly as NX2navi-Setup.exe installs it: + NX2navi.exe launcher/native/nx2navi.c with the Triforce icon (launcher/ui/art.py) + python/ python.org's embeddable Python 3.14.9, tkinter and Tcl/Tk 9.0 from its tcltk.msi, + the packages of tools/requirements-launcher.txt, runtime-id.txt + app/current.txt, app// launcher/ (without native/), runtime/, recomp/, the tools make_game.py uses, + bin/nx2navi_xlate.exe and .id (and amd_fidelityfx_vk.dll with --ffx), toolchain.lock.json (the MSYS2 packages of today, + pinned), feed.json, VERSION, the licence files + NX2navi-Setup-.exe launcher/native/setup.c with the folder above appended as a zip: the one file + players download + NX2navi-app-.zip app/ alone: what the launcher downloads to update itself + NX2navi-runtime-.zip python/ alone: downloaded only when a release changes the Python + NX2navi-update.json the manifest the launcher reads (launcher/feed.py) + +--publish uploads the four files as GitHub release v of the --feed repository with the gh CLI (a draft +first is wiser: add --draft). check_content runs over everything before it is packed: the release holds no game data. +""" +import argparse +import hashlib +import json +import os +import shutil +import subprocess +import sys +import time +import zipfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(ROOT / 'tools')) +import check_content # noqa: E402 +from launcher import toolchain # noqa: E402 +from launcher.paths import Layout # noqa: E402 + +MAIN_ROOT = (ROOT / 'build').resolve().parent # build/ and toolchain/ are junctions in a worktree +DL = MAIN_ROOT / 'toolchain' / 'launcher-dl' +WORK = MAIN_ROOT / 'toolchain' / 'launcher-work' +PY_VERSION = '3.14.9' +PINS = { + 'python-embed': (f'https://www.python.org/ftp/python/{PY_VERSION}/python-{PY_VERSION}-embed-amd64.zip', + '3c01d7cfc666f746a6f064f60fb2156ab0ec464d3d738ab796371a46a9bb24a2'), + # the installer's own Tcl/Tk and tkinter component, published beside it + 'tcltk': (f'https://www.python.org/ftp/python/{PY_VERSION}/amd64/tcltk.msi', + 'ac87609db3f715404dd41a989c2f1706ea1512f3d86217f938d11c63af8eb505'), +} +TOOLS = ('make_game.py', 'nup.py', 'romfs_index.py', 'check_content.py', 'hostmem.py', 'prepare_shaders.py', + 'shader_census.py', 'shader_list.py', 'zstd_compat.py') +TOP = ('LICENSE', 'LICENSE-ADDITIONS', 'THIRD_PARTY_NOTICES.md', 'nx2navi.ini.example') +SKIP_DIRS = {'__pycache__', '.pytest_cache', 'native'} +SKIP_SUFFIXES = {'.o', '.obj', '.pyc', '.exe', '.dll', '.a', '.lib'} +TCLTK_FILES = ('_tkinter.pyd', 'tcl90.dll', 'tcl9tk90.dll', 'zlib1.dll', 'libtommath.dll') +DEFAULT_FEED = 'tangerineturtle1/NX2navi' + + +def say(msg): + print(msg, flush=True) + + +def sha256_file(path): + h = hashlib.sha256() + with open(path, 'rb') as f: + while b := f.read(1 << 20): + h.update(b) + return h.hexdigest() + + +def fetch(name): + url, want = PINS[name] + dest = DL / url.rsplit('/', 1)[1] + if dest.is_file() and sha256_file(dest) == want: + return dest + from launcher import net + say(f'download {url}') + net.download(url, dest, sha256=want) + return dest + + +def run(cmd, **kw): + say(' ' + ' '.join(str(c) for c in cmd)) + subprocess.run([str(c) for c in cmd], check=True, **kw) + + +def copy_tree(src, dst, skip_dirs=SKIP_DIRS): + for p in sorted(src.rglob('*')): + rel = p.relative_to(src) + if any(part in skip_dirs for part in rel.parts) or p.is_dir() or p.suffix.lower() in SKIP_SUFFIXES: + continue + (dst / rel).parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(p, dst / rel) + + +def version(): + v = (ROOT / 'VERSION').read_text(encoding='utf-8').strip() + r = subprocess.run(['git', '-C', str(ROOT), 'rev-parse', '--short', 'HEAD'], capture_output=True, text=True) + return v, f'{v}+{r.stdout.strip()}' if r.returncode == 0 and r.stdout.strip() else v + + +# ---------------------------------------------------------------- python/ + +def tcltk_files(): + dest = WORK / 'tcltk' + if (dest / 'DLLs' / '_tkinter.pyd').is_file(): + return dest + msi = fetch('tcltk') + shutil.rmtree(dest, ignore_errors=True) + dest.mkdir(parents=True) + if os.name == 'nt': # an administrative image: copies the files, installs nothing + run(['msiexec', '/a', msi, '/qn', f'TARGETDIR={dest}']) + else: + run(['msiextract', '-C', dest, msi], stdout=subprocess.DEVNULL) + if not (dest / 'DLLs' / '_tkinter.pyd').is_file(): + sys.exit(f'{dest}: no _tkinter.pyd in {msi.name}') + return dest + + +def runtime_id(): + req = (ROOT / 'tools' / 'requirements-launcher.txt').read_bytes() + return f'py{PY_VERSION}-{hashlib.sha256(req + PINS["tcltk"][1].encode()).hexdigest()[:8]}' + + +def build_python(py): + with zipfile.ZipFile(fetch('python-embed')) as z: + z.extractall(py) + # The embeddable Python's sys.path is exactly this list; ../pydeps holds zbic (launcher/toolchain.py). + (py / 'python314._pth').write_text('python314.zip\n.\nLib/site-packages\n../pydeps\n', encoding='utf-8') + env = dict(os.environ, UV_CACHE_DIR=str(MAIN_ROOT / 'toolchain' / 'uv-cache')) + run(['uv', 'pip', 'install', '--target', py / 'Lib' / 'site-packages', '--python', sys.executable, + '--python-platform', 'windows', '--python-version', '3.14', '--only-binary', ':all:', '-q', + '-r', ROOT / 'tools' / 'requirements-launcher.txt'], env=env) + shutil.rmtree(py / 'Lib' / 'site-packages' / 'bin', ignore_errors=True) + tk = tcltk_files() + for dll in TCLTK_FILES: + if (tk / 'DLLs' / dll).is_file(): + shutil.copy2(tk / 'DLLs' / dll, py / dll) + copy_tree(tk / 'Lib' / 'tkinter', py / 'tkinter', skip_dirs={'__pycache__', 'test'}) + copy_tree(tk / 'tcl', py / 'tcl', skip_dirs={'__pycache__'}) + (py / 'runtime-id.txt').write_text(runtime_id() + '\n', encoding='utf-8') + + +# ---------------------------------------------------------------- app// + +def build_app(app, ver, xlate, feed_repo, lock, ffx=None): + copy_tree(ROOT / 'launcher', app / 'launcher') + for d in ('runtime', 'recomp'): + copy_tree(ROOT / d, app / d) + (app / 'tools').mkdir(parents=True) + for t in TOOLS: + shutil.copy2(ROOT / 'tools' / t, app / 'tools' / t) + for t in TOP: + if (ROOT / t).is_file(): + shutil.copy2(ROOT / t, app / t) + (app / 'VERSION').write_text(ver + '\n', encoding='utf-8') + (app / 'bin').mkdir() + for n in ('nx2navi_xlate.exe', 'nx2navi_xlate.id'): + if not (xlate / n).is_file(): + sys.exit(f'{xlate / n} is missing: build the shader translator first (tools/build_xlate.py)') + shutil.copy2(xlate / n, app / 'bin' / n) + if ffx: # FSR 3.1 loads it from beside the game (runtime/vk_ffx.cpp); make_game.py copies it there + shutil.copy2(ffx, app / 'bin' / 'amd_fidelityfx_vk.dll') + (app / 'toolchain.lock.json').write_text(json.dumps(lock, indent=1), encoding='utf-8') + (app / 'feed.json').write_text(json.dumps({'type': 'github', 'repo': feed_repo}, indent=1), encoding='utf-8') + + +# ---------------------------------------------------------------- the native programs + +def native_compiler(lock): + """(cc argv prefix, windres argv prefix) for x86_64 Windows: MSYS2's own clang on Windows, else the host's clang + against the same MSYS2 packages as a sysroot.""" + lay = Layout(root=MAIN_ROOT / 'toolchain' / 'launcher-msys') + if not toolchain.is_current(lay, lock): + say('MSYS2 packages for the native programs') + toolchain.install(lay, lock, log=say) + sysroot = lay.msys + if os.name == 'nt': + return [str(sysroot / 'bin' / 'clang.exe')], [str(sysroot / 'bin' / 'windres.exe')] + for tool in ('clang', 'ld.lld'): + if not shutil.which(tool): + sys.exit(f'{tool} is needed to cross-compile the launcher programs') + windres = shutil.which('llvm-windres') or shutil.which('llvm-windres-18') or '/usr/lib/llvm-18/bin/llvm-windres' + if not Path(windres).exists(): + sys.exit('llvm-windres is needed to compile the launcher programs\' resources') + return (['clang', '--target=x86_64-w64-mingw32', f'--sysroot={sysroot}', '-fuse-ld=lld'], + [windres, '--target=x86_64-w64-mingw32']) + + +def build_native(work, cc, windres): + from launcher.ui import art + work.mkdir(parents=True, exist_ok=True) + (work / 'nx2navi.ico').write_bytes(art.triforce_ico()) + shutil.copy2(ROOT / 'launcher' / 'native' / 'app.manifest', work / 'app.manifest') + res = work / 'res.o' + run([*windres, '-I', work, ROOT / 'launcher' / 'native' / 'res.rc', '-O', 'coff', '-o', res]) + flags = ['-municode', '-mwindows', '-O2', '-Wall', '-static', '-s'] + stub = work / 'NX2navi.exe' + setup = work / 'setup-stub.exe' + run([*cc, *flags, ROOT / 'launcher' / 'native' / 'nx2navi.c', res, '-o', stub, '-lshell32', '-lole32', '-luuid']) + run([*cc, *flags, ROOT / 'launcher' / 'native' / 'setup.c', res, '-o', setup, '-l:libz.a', '-lshell32', + '-lole32', '-luuid', '-ladvapi32']) + return stub, setup + + +# ---------------------------------------------------------------- packing + +def zip_dir(src, dest, prefix=''): + with zipfile.ZipFile(dest, 'w', zipfile.ZIP_DEFLATED, compresslevel=9) as z: + for p in sorted(src.rglob('*')): + if p.is_file(): + z.write(p, (Path(prefix) / p.relative_to(src)).as_posix()) + return dest + + +def make_setup(stub, payload, dest, ver): + """setup-stub.exe + the payload zip + the 48-byte trailer launcher/native/setup.c reads.""" + data = stub.read_bytes() + off = len(data) + z = payload.read_bytes() + trailer = b'NX2NSET1' + off.to_bytes(8, 'little') + len(z).to_bytes(8, 'little') + \ + ver.encode('utf-8')[:24].ljust(24, b'\0') + dest.write_bytes(data + z + trailer) + return dest + + +def check_folder(folder): + bad = [f'{p.relative_to(folder)}: {m}' for p in sorted(folder.rglob('*')) if p.is_file() + for m in check_content.problems_for(p, allow_large=True)] + if bad: + sys.exit('check_content refuses these release files:\n ' + '\n '.join(bad[:20])) + + +def entry(path): + return {'name': path.name, 'size': path.stat().st_size, 'sha256': sha256_file(path)} + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument('--xlate', type=Path, required=True, help='folder with nx2navi_xlate.exe and nx2navi_xlate.id') + ap.add_argument('--ffx', type=Path, help="AMD's signed amd_fidelityfx_vk.dll (FidelityFX SDK 1.1.4, MIT) for FSR 3.1") + ap.add_argument('--out', type=Path, default=MAIN_ROOT / 'build' / 'launcher-release') + ap.add_argument('--feed', default=DEFAULT_FEED, help='the GitHub repository the launcher looks for releases in') + ap.add_argument('--notes', type=Path, help='release notes (markdown) for the manifest and the GitHub release') + ap.add_argument('--lock', type=Path, help='a toolchain.lock.json to reuse instead of resolving today\'s packages') + ap.add_argument('--version', help='the release version instead of VERSION\'s (test releases)') + ap.add_argument('--publish', action='store_true', help='upload as GitHub release v with gh') + ap.add_argument('--draft', action='store_true', help='with --publish: as a draft') + a = ap.parse_args() + if sys.version_info < (3, 14): + sys.exit('Python 3.14 or later is needed (compression.zstd for the MSYS2 packages)') + ver, build = version() + if a.version: + ver, build = a.version, a.version + build[len(ver):] + out = a.out + shutil.rmtree(out, ignore_errors=True) + folder = out / 'NX2navi' + t0 = time.time() + lock = json.loads(a.lock.read_text(encoding='utf-8')) if a.lock else toolchain.make_lock() + say(f'release {build}: toolchain lock with {len(lock["packages"])} packages') + build_app(folder / 'app' / ver, ver, a.xlate, a.feed, lock, a.ffx) + (folder / 'app' / 'current.txt').write_text(ver + '\n', encoding='utf-8') + build_python(folder / 'python') + cc, windres = native_compiler(lock) + stub, setup_stub = build_native(out / 'native', cc, windres) + shutil.copy2(stub, folder / 'NX2navi.exe') + check_folder(folder) + app_zip = zip_dir(folder / 'app' / ver, out / f'NX2navi-app-{ver}.zip') + rt_zip = zip_dir(folder / 'python', out / f'NX2navi-runtime-{runtime_id()}.zip') + payload = zip_dir(folder, out / 'native' / 'payload.zip') + setup = make_setup(setup_stub, payload, out / f'NX2navi-Setup-{ver}.exe', ver) + notes = a.notes.read_text(encoding='utf-8') if a.notes else '' + manifest = {'format': 1, 'version': ver, 'build': build, 'notes': notes, + 'app': entry(app_zip), 'runtime': {'id': runtime_id(), **entry(rt_zip)}, + 'toolchain': toolchain.lock_digest(lock), 'setup': entry(setup), + 'made': time.strftime('%Y-%m-%d %H:%M')} + man = out / 'NX2navi-update.json' + man.write_text(json.dumps(manifest, indent=1), encoding='utf-8') + for p in (setup, app_zip, rt_zip, man): + say(f'{p.name}: {p.stat().st_size / 1e6:.1f} MB') + say(f'release folder {out} in {time.time() - t0:.0f} s') + if a.publish: + cmd = ['gh', 'release', 'create', f'v{ver}', setup, app_zip, rt_zip, man, '--repo', a.feed, + '--title', f'NX2navi {ver}'] + cmd += ['--notes-file', a.notes] if a.notes else ['--notes', f'NX2navi {ver}'] + if a.draft: + cmd.append('--draft') + run(cmd) + + +if __name__ == '__main__': + main() diff --git a/tools/requirements-launcher.txt b/tools/requirements-launcher.txt new file mode 100644 index 00000000..5241f71d --- /dev/null +++ b/tools/requirements-launcher.txt @@ -0,0 +1,6 @@ +# Packages of the launcher release's bundled Python 3.14.9 (tools/make_launcher.py): what the launcher and +# tools/make_game.py import. numpy: recomp/gen.py and the window's drawing (launcher/ui/art.py). lz4: NSO segments. +# zstandard: tools/nup.py. zbic is not bundled: the launcher downloads it from PyPI on the player's PC. +numpy==2.5.4 +lz4==4.4.5 +zstandard==0.25.0