From 89c453d7bb374975234e200f0f0a135e960fb6a8 Mon Sep 17 00:00:00 2001 From: epicmeme546 <178368684+epicmeme546@users.noreply.github.com> Date: Thu, 12 Nov 2026 08:11:57 -0500 Subject: [PATCH] Add a Mods screen to the launcher and build the mods' code patches into the launcher's game. Mods, on the home screen, opens the game's mods folder, adds a downloaded mod from its .zip, .7z or .rar file or its folder, switches each mod on or off, and sets the load order (mods_disabled and mods_order in nx2navi.ini, written as the game writes them). Each mod's description says what it changes, which other mod changes the same files and which of the two wins, and why the game leaves it off when it does. Before the game starts, Play puts the mods folder in order (launcher/mods.py prepare): an archive in it is unpacked into a folder of its name and kept, a mod whose files sit a folder or more down has them moved up, and a romfs, exefs or cheats folder put straight into the mods folder becomes a mod of its own. An archive that would write outside its folder is refused, and nothing is deleted. The launcher's build (tools/make_game.py) left the mods' code cheats and ExeFS patch files out, so those mods stayed off. The lift now turns them into patch sets first, as the setup does, with the files in the lift's key and build-cache/mod-code.json recording what it took up. Play builds the game again before it starts when an enabled mod brings code no build has seen. The shaders' key names the lift and the compile without the mods' code, so such a build compiles only what the patches change and leaves the shaders prepared. tools/mod_files.py is the one place the tools, the build and the launcher find a mod's parts. Tested: a zipped mod holding its own folder, put into a mods folder, was unpacked and moved up by the launcher's code, and the game served its file in place of the game's. The lifter took a code cheat and a pchtxt patch made for the game's main as patch sets. The launcher's and the setup's tests pass. Co-Authored-By: Claude Opus 5.5 --- docs/launcher.md | 36 ++- docs/releases/v0.9.0.md | 7 + launcher/jobs.py | 11 + launcher/mods.py | 505 ++++++++++++++++++++++++++++++++++++++++ launcher/ui/app.py | 277 +++++++++++++++++++++- launcher/ui/selftest.py | 30 ++- nx2navi.ini.example | 8 +- tests/launcher_test.py | 152 +++++++++++- tools/cheats.py | 23 +- tools/exefs_patches.py | 25 +- tools/make_game.py | 134 +++++++++-- tools/make_kit.py | 8 +- tools/make_launcher.py | 2 +- tools/make_release.py | 2 +- tools/mod_files.py | 99 ++++++++ 15 files changed, 1234 insertions(+), 85 deletions(-) create mode 100644 launcher/mods.py create mode 100644 tools/mod_files.py diff --git a/docs/launcher.md b/docs/launcher.md index e41048a8..66232499 100644 --- a/docs/launcher.md +++ b/docs/launcher.md @@ -88,7 +88,9 @@ identity (`NX2navi.Launcher`), so the taskbar groups and pins it under NX2navi r Screens (`launcher/ui/app.py`): first run (dump, install folder, Desktop Shortcut, Check for Updates at Start, Update Channel, Begin Installation); the steps of an install, update or rebuild with their progress, the build log's last lines and Cancel, and after a failure Try Again and Copy Report; home (Play, Updates, Game Folder, Saves, -Rebuild the Game, Customize Launcher, Settings); customize (Color Scheme, with Custom Background Color, Panel Color and +Mods, Rebuild the Game, Customize Launcher, Settings); mods (Open Mods Folder, Add a Mod, Add a Mod Folder, one On/Off +row for each mod, More Mods when they fill more than a page, Load Order, Back); load order (each mod with its place, +left and right to move it); customize (Color Scheme, with Custom Background Color, Panel Color and Button Color, Background, with a picture set Background Dimming and Remove Background, Animations, Sounds, Add to Steam on Windows, Back); the colour picker (Hue, Saturation, Brightness, Hex Code, Reset, Done); saves (Export Save, Add Log and Settings, Export To, Import a Save, Import a Save Folder, Open Saves Folder, Back); settings (update check, Install Updates Automatically, Hide Launcher While Playing, Keep @@ -338,6 +340,38 @@ copies the current save to `saves/backup ` and makes the import the s (`runtime/fs_host.c` `import_pending`), and the menu says where the old save went. An import in the launcher replaces one still waiting from the menu. +## Mods + +Mods (home, under Saves; `launcher/mods.py`) is where the player puts the mods they download. The game reads the game +folder's `mods` folder (`runtime/mods.c`; `mods_dir` in nx2navi.ini names another): one folder per mod, holding a +`romfs` folder with the game files it replaces, an `exefs` folder with code patches (`.ips`, `.pchtxt`) or a `cheats` +folder, either at its top or in the Atmosphere layout (`atmosphere/contents/0400486024DEC000/romfs` and the others +`tools/mod_files.py` lists). + +A mod goes in as it came. Play first puts the folder in order (`mods.prepare`): an archive (`.zip`, and `.7z` or +`.rar` through Windows' own `tar`, which reads them on Windows 10 and 11) is unpacked into a folder of its name unless +that folder is there already, and the archive stays; a mod folder whose files sit one or more folders down (an archive +that held the mod's own folder) gets them moved up; a `romfs`, `romfslite`, `exefs` or `cheats` folder put straight into +the mods folder becomes a mod of its own, *Unnamed mod*. An archive that would write outside its folder is refused. +Nothing is deleted. Add a Mod and Add a Mod Folder do the same for an archive or folder the player picks, which is +unpacked or copied into the mods folder under a free name; Open Mods Folder opens it. + +Each mod has an On/Off row; its description says what it changes (how many game files, code patches, cheats), which +other mod changes the same files and which of the two wins, and why the game leaves it off when it does: a code module +(`subsdk*`), a download with options to pick from (several folders that each hold a mod), a mod for another game or +another version, or nothing the game reads. Load Order sets which mod wins a file two of them change: the higher one. +The switches and the order are the game's own settings, `mods_disabled` and `mods_order` in nx2navi.ini, written as the +game writes them; they apply at the game's next start, so they wait while it runs. + +**Mods that change the game's code.** The game is lifted from the console's code ahead of time, so code cheats and +ExeFS patches cannot be applied while it runs: the build turns them into patch sets of the lifted code +(`tools/cheats.py`, `tools/exefs_patches.py`) that the game switches on with their mod, and the Cheats tab of the +game's menu switches each cheat. The lift step's key names those files, so a build lifts and compiles again when they +change (only the changed parts compile, and the shaders stay prepared), and `build-cache/mod-code.json` records the +files it took up. When an enabled mod brings code no build has seen, Play builds the game again first (a few minutes; +the home's Mods row says "Build Waiting") and starts it after. A Play from Steam then opens the window for that build. +Code mods are built from the game folder's own `mods` folder only. + ## Importing a save folder Import a Save Folder (Saves) makes another save the game's own. A save put into the game folder's `saves` folder (its diff --git a/docs/releases/v0.9.0.md b/docs/releases/v0.9.0.md index fa955e49..41f72df1 100644 --- a/docs/releases/v0.9.0.md +++ b/docs/releases/v0.9.0.md @@ -55,6 +55,13 @@ for the launcher; a GPU with Vulkan 1.3 and `VK_EXT_external_memory_host` (curre ## New +- **Mods in the launcher.** Mods, on the launcher's home screen, opens the mods folder and adds a mod you downloaded, + as its .zip, .7z or .rar file or its folder; one put into the mods folder by hand works as well. Before the game + starts, the launcher unpacks archives and moves a mod's files to where the game looks for them when they sit a folder + or two down. Each mod has an On/Off switch and says what it changes, which other mod changes the same files and + which of them wins, and why the game cannot use it when it cannot; Load Order sets which mod wins. A mod with code + patches or code cheats is built into the game before it starts, which takes a few minutes the first time: the + launcher's own build left those out until now. - **One Anti-aliasing setting**: the game's own SMAA, off, FXAA, SMAA at its Ultra preset, or TAA. - **Direct mouse camera on game 1.1.1**: the camera takes the mouse's own movement after the stick's dead zone. Contributed by Gaetarra (pull request #462). diff --git a/launcher/jobs.py b/launcher/jobs.py index a411236f..64dd095d 100644 --- a/launcher/jobs.py +++ b/launcher/jobs.py @@ -414,6 +414,17 @@ class RebuildJob(Job): self.note_scan(self.game_dir) +class ModsRebuildJob(RebuildJob): + """Play with a mod whose code cheats or ExeFS patches the game was not built with (launcher/mods.py + code_waiting): the game is built again with them, compiling only what they change, and then starts (App.play).""" + title = 'Building Your Mods In' + retune = False + + def __init__(self, ctx, game_dir): + super().__init__(ctx, game_dir) + self.after = 'play' + + class UpdateDownloadJob(Job): """Before the restart: download, check and stage the release, then switch app/current.txt to it.""" kind = 'update' diff --git a/launcher/mods.py b/launcher/mods.py new file mode 100644 index 00000000..296ff7b7 --- /dev/null +++ b/launcher/mods.py @@ -0,0 +1,505 @@ +"""The game's mods folder, for the launcher's Mods screen and Play. + +The game reads /mods (nx2navi.ini mods_dir names another folder; a relative one is under the game folder): one +folder per mod, each part of it where tools/mod_files.py says (runtime/mods.c). The load order and the switches are +the game's own settings, mods_order and mods_disabled in nx2navi.ini, which this module writes as runtime/cfg.c does: +the top mod wins a file two mods replace, and a mod nobody switched off is on. + +A mod as players download it often does not sit that way, so prepare() puts the mods folder in order before the game +starts, and add() does the same for an archive or a folder the player picks: +- an archive in the mods folder (.zip; .7z and .rar through the system's tar, which reads them on Windows 10 and 11) + is unpacked into a folder of its own name, unless that folder is there already. The archive stays: removing the + folder and the archive removes the mod, and switching it off keeps both; +- a mod folder with nothing of a mod at its top and one folder inside, down to where the mod's parts are, gets that + folder's contents (an archive unpacked into a folder of its own name, holding the mod's folder); +- a romfs, romfslite, exefs or cheats folder put straight into the mods folder becomes a mod of its own (the + Atmosphere layouts, atmosphere/, contents/ and the title's folder, already are one as they are). +Nothing is deleted: files only move inside the mods folder, and a mod that holds several folders with mod files (a +download with options to pick from) stays as it is, with a note. + +Mods that change the game's code (code cheats, ExeFS patches) need the game built with them (tools/make_game.py lift). +code_waiting() names the enabled ones whose files the last build did not take in (build-cache/mod-code.json), and +Play rebuilds the game first then. The build reads them from /mods only. +""" +import json +import os +import re +import shutil +import subprocess +import sys +import zipfile +from pathlib import Path + +from . import gameini + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tools')) +import mod_files # noqa: E402 + +ARCHIVES = ('.zip', '.7z', '.rar') +LOOSE_PARTS = ('romfs', 'romfslite', 'exefs', 'cheats') # a mod's parts that are no mod when straight in mods/ +# what a folder holds at its top when a mod starts there (runtime/mods.c find_root's places) +MOD_TOP = {'romfs', 'romfslite', 'exefs', 'cheats', 'atmosphere', 'contents', mod_files.TITLE.lower()} +UNWRAP_DEPTH = 4 +UNNAMED = 'Unnamed mod' +STAND_INS = 'Damaged file stand-ins' # the launcher's own (tools/romfs_check.py), shown as such +TITLE_ID = re.compile(r'^[0-9A-Fa-f]{16}$') + + +class ModError(Exception): + pass + + +# ---------------------------------------------------------------- the folder and the settings + +def mods_dir(game_dir): + """The mods folder the game reads: nx2navi.ini mods_dir, a relative one under the game folder, else mods.""" + try: + d = gameini.get(game_dir, 'MODS_DIR') + except OSError: + d = '' + p = Path(d) if d else Path('mods') + return p if p.is_absolute() or (len(d) > 1 and d[1] == ':') else Path(game_dir) / p + + +def ensure_mods_dir(game_dir): + d = mods_dir(game_dir) + d.mkdir(parents=True, exist_ok=True) + return d + + +def _names(game_dir, key): + try: + v = gameini.get(game_dir, key) + except OSError: + v = '' + return [n for n in v.split('|') if n] if v else [] + + +def _quoted(v): + """A value as runtime/cfg.c put_value writes it: quoted when the blanks at its ends or the quotes around it would + be lost.""" + if v and (v[0] in ' \t' or v[-1] in ' \t' or (len(v) >= 2 and v[0] == v[-1] == '"')): + return f'"{v}"' + return v + + +def order(names, wanted): + """names in the game's order (runtime/mods.c by_order): the ones wanted names first, in its order, then the rest by + name (byte order of their UTF-8, as strcmp).""" + pos = {n: i for i, n in reversed(list(enumerate(wanted)))} + return sorted(names, key=lambda n: (0, pos[n], b'') if n in pos else (1, 0, n.encode('utf-8', 'surrogateescape'))) + + +def set_enabled(game_dir, name, on): + """Switch a mod on or off for the game's next start (nx2navi.ini mods_disabled).""" + off = [n for n in _names(game_dir, 'MODS_DISABLED') if n != name] + if not on: + off.append(name) + gameini.put(game_dir, 'mods', 'mods_disabled', _quoted('|'.join(off))) + + +def move(game_dir, name, d): + """Move a mod d places up (negative) or down in the load order (nx2navi.ini mods_order, every mod named).""" + names = [m.name for m in scan(game_dir, details=False)] + if name not in names: + return + i = names.index(name) + j = max(0, min(len(names) - 1, i + d)) + names.insert(j, names.pop(i)) + gameini.put(game_dir, 'mods', 'mods_order', _quoted('|'.join(names))) + + +# ---------------------------------------------------------------- what a mod holds + +class Mod: + def __init__(self, path): + self.path, self.name = path, path.name + self.enabled = True + self.romfs = mod_files.part(path, 'romfs') or mod_files.part(path, 'romfslite') + self.exefs = mod_files.part(path, 'exefs') + self.cheats = mod_files.part(path, 'cheats') + self.files = [] # RomFS paths it replaces, lower case, '/' separated (details) + self.patches = 0 # ExeFS patch files for this game's main + self.other_version = 0 # ExeFS patch files for another version + self.cheat_file = None # its cheat file for this game's main + self.code_module = False # subsdk* or main in its exefs: code the port cannot run + self.other_game = '' # the title id of another game its Atmosphere folders name + self.several = [] # folders that each hold a mod (a download with options) + self.unsupported = '' # why the last build could not take an ExeFS patch file of it + self.waiting = False # code the last build did not take in + + @property + def usable(self): + return not self.problem + + @property + def problem(self): + """Why the game leaves this mod off whatever its switch says (runtime/mods.c scan_cb), '' when it does not.""" + if '|' in self.name: + return 'Its folder name holds "|", which the load order list uses: rename the folder.' + if self.code_module: + return ('It holds a code module (subsdk or main in its exefs folder), which NX2navi cannot run: the game ' + 'leaves the whole mod off.') + if self.unsupported: + return (f'An ExeFS patch file of it cannot be applied ({self.unsupported}): the game leaves the whole mod ' + 'off.') + if not (self.files or self.romfs or self.patches or self.cheat_file): + if self.other_game: + return f'It is made for another game (title {self.other_game}), not Ocarina of Time.' + if self.several: + return (f'It holds {len(self.several)} folders with mod files ({", ".join(self.several[:3])}' + f'{", ..." if len(self.several) > 3 else ""}), a download with options to pick from: move ' + 'the one you want into the mods folder.') + if self.other_version: + return 'Its code patches are for another version of the game, not 1.1.1.' + return ('It holds nothing the game reads: a mod\'s files go into a romfs, exefs or cheats folder (or ' + 'atmosphere\\contents\\' + mod_files.TITLE + '\\romfs).') + return '' + + +def _main_id(game_dir): + try: + with open(Path(game_dir) / 'exefs' / 'main', 'rb') as f: + head = f.read(0x60) + except OSError: + return '' + return head[0x40:0x60].hex() if head[:4] == b'NSO0' and len(head) == 0x60 else '' + + +def _same_build(hex_id, main_id): + """hex_id (trailing zeros may be left out) names main's build id; True when main's is unknown (runtime/mods.c).""" + if not main_id: + return True + h = hex_id.lower() + return 0 < len(h) <= 64 and all(c in '0123456789abcdef' for c in h) and main_id.startswith(h.rstrip('0')) \ + and set(main_id[len(h):]) <= {'0'} + + +def _pchtxt_build(data): + """The build id a pchtxt file names (its last @nsobid- line before @stop), '' when none.""" + found = '' + for line in data.decode('utf-8-sig', errors='replace').splitlines(): + s = line.strip() + if s.startswith('@stop'): + break + if s.startswith('@nsobid-'): + found = re.match(r'@nsobid-([0-9A-Fa-f]*)', s).group(1) + return found + + +def _fnv(text): + h = 0x811C9DC5 + for b in text.encode('utf-8'): + h = ((h ^ b) * 0x01000193) & 0xFFFFFFFF + return f'{h:08X}' + + +def _status(game_dir, name): + """{key: (kind, note)} of a build status file (build-cache/exefs.status, tools/exefs_patches.py).""" + out = {} + try: + text = (Path(game_dir) / 'build-cache' / name).read_text(encoding='utf-8', errors='replace') + except OSError: + return out + for line in text.splitlines(): + f = line.split('\t') + if len(f) >= 5: + out[f[0]] = (f[2], f[4]) + return out + + +def _details(m, main_id, exefs_status): + if m.romfs: + top = len(m.romfs.parts) + m.files = sorted('/'.join(Path(root, f).parts[top:]).lower() + for root, _dirs, files in os.walk(m.romfs) for f in files) + if m.exefs: + for f in m.exefs.iterdir(): + n = f.name.lower() + if not f.is_file(): + continue + if n.startswith('subsdk') or n == 'main': + m.code_module = True + for mod, rel, f in mod_files.patch_files(m.path.parent): + if mod != m.name: + continue + try: + bid = f.stem if f.suffix.lower() == '.ips' else _pchtxt_build(f.read_bytes()) + except OSError: + bid = '' + ok = bool(bid) and _same_build(bid, main_id) + if not ok: + m.other_version += 1 + continue + m.patches += 1 + kind, note = exefs_status.get(_fnv(f'{mod}/{rel}'), ('', '')) + if kind == 'unsupported' and not m.unsupported: + m.unsupported = f'{f.name}: {note}' if note else f.name + contents = m.path / 'atmosphere' / 'contents' + for where in (contents, m.path / 'contents'): + if where.is_dir(): + others = [d.name for d in where.iterdir() if d.is_dir() and TITLE_ID.match(d.name) + and d.name.upper() != mod_files.TITLE] + if others and not m.romfs and not m.exefs and not m.cheats: + m.other_game = others[0].upper() + if not (m.romfs or m.exefs or m.cheats or _is_mod_top(m.path)): + m.several = [d.name for d in _mod_folders_below(m.path)] + + +def _mod_folders_below(folder, depth=2): + """The folders under folder (down to depth levels) that hold a mod at their top.""" + out = [] + try: + subs = sorted((d for d in folder.iterdir() if d.is_dir()), key=lambda d: d.name.lower()) + except OSError: + return out + for d in subs: + if _is_mod_top(d): + out.append(d) + elif depth > 1: + out += _mod_folders_below(d, depth - 1) + return out + + +def _is_mod_top(folder): + try: + return any(d.is_dir() and d.name.lower() in MOD_TOP for d in folder.iterdir()) + except OSError: + return False + + +def scan(game_dir, details=True): + """The mods of the mods folder in the game's load order, with their switches; with details what each holds.""" + folder = mods_dir(game_dir) + mods = [Mod(p) for p in mod_files.mod_folders(folder)] + off = set(_names(game_dir, 'MODS_DISABLED')) + ranked = order([m.name for m in mods], _names(game_dir, 'MODS_ORDER')) + mods.sort(key=lambda m: ranked.index(m.name)) + main_id = _main_id(game_dir) if details else '' + exefs_status = _status(game_dir, 'exefs.status') if details else {} + waiting = set(code_waiting(game_dir)) if details else set() + for m in mods: + m.enabled = m.name not in off + if details: + _details(m, main_id, exefs_status) + m.cheat_file = next((f for mod, f in mod_files.cheat_files(folder, main_id[:16].upper()) if mod == m.name), + None) if main_id else None + m.waiting = m.name in waiting + return mods + + +def overlaps(mods): + """{mod name: [(other mod name, files both replace, whether this one wins)]} among the enabled mods with RomFS + files: the mod higher in the load order wins a file two replace.""" + on = [m for m in mods if m.enabled and m.files] + out = {m.name: [] for m in on} + sets = {m.name: set(m.files) for m in on} + for i, a in enumerate(on): + for j, b in enumerate(on): + if a is not b: + n = len(sets[a.name] & sets[b.name]) + if n: + out[a.name].append((b.name, n, i < j)) + return out + + +# ---------------------------------------------------------------- the mods' code and the build + +def built_code(game_dir): + """{"/": sha256} of the mods' code the last build took in (tools/make_game.py MOD_RECORD).""" + try: + return json.loads((Path(game_dir) / 'build-cache' / 'mod-code.json').read_text(encoding='utf-8')) + except (OSError, ValueError): + return {} + + +def code_waiting(game_dir): + """The enabled mods whose code cheats or ExeFS patches the game was not built with (or with other ones). Only the + game folder's own mods folder is built from.""" + folder = Path(game_dir) / 'mods' + if mods_dir(game_dir) != folder or not folder.is_dir(): + return [] + bid = mod_files.main_build_id(Path(game_dir) / 'exefs' / 'main') + if not bid: + return [] + off = set(_names(game_dir, 'MODS_DISABLED')) + built = built_code(game_dir) + out = [] + for key, sha in mod_files.code_record(folder, bid).items(): + mod = key.split('/', 1)[0] + if mod not in off and built.get(key) != sha and mod not in out: + out.append(mod) + return out + + +# ---------------------------------------------------------------- putting downloads in order + +def _unique(folder, name, file=False): + """folder / name, or the first free "name 2", "name 3" ... (for a file "stem (2).txt" ...).""" + p, k = folder / name, 2 + stem, suffix = (Path(name).stem, Path(name).suffix) if file else (name, '') + while p.exists(): + p = folder / (f'{stem} ({k}){suffix}' if file else f'{name} {k}') + k += 1 + return p + + +def _merge_into(src, dst): + """Move src's entries into dst (made when missing) and remove src: a folder dst holds already is merged into, and + a file of a name dst holds already moves in under a free name (a readme beside the wrapped folder).""" + dst.mkdir(parents=True, exist_ok=True) + for e in list(src.iterdir()): + t = dst / e.name + if t.is_dir() and e.is_dir(): + _merge_into(e, t) + else: + shutil.move(str(e), str(_unique(dst, e.name, file=True) if t.exists() else t)) + src.rmdir() + + +def unwrap(folder): + """A mod folder with nothing of a mod at its top and one folder inside (files beside it, a readme, do not count), + down to UNWRAP_DEPTH levels until the mod starts: that folder's contents move up. True when it did.""" + inner = folder + for _ in range(UNWRAP_DEPTH): + if _is_mod_top(inner): + break + subs = [d for d in inner.iterdir() if d.is_dir()] + if len(subs) != 1: + return False + inner = subs[0] + else: + if not _is_mod_top(inner): + return False + if inner == folder: + return False + # the chain's top folder may share its name with a folder of the mod (MyMod/romfs/romfs): moved aside first. The + # mod's folders go up first, then whatever lay beside them on the way down (a readme), so nothing stays behind + top = next(p for p in [inner, *inner.parents] if p.parent == folder) + aside = _unique(folder, '.unwrapping') + top.rename(aside) + inner = aside / inner.relative_to(top) + _merge_into(inner, folder) + if aside.exists(): + _merge_into(aside, folder) + return True + + +def _safe_members(names): + for n in names: + p = n.replace('\\', '/') + if p.startswith('/') or re.match(r'^[A-Za-z]:', p) or '..' in p.split('/'): + raise ModError(f'it holds a file outside its own folder ({n})') + + +def _tar(): + if os.name == 'nt': + t = Path(os.environ.get('SystemRoot', r'C:\Windows')) / 'System32' / 'tar.exe' + return str(t) if t.is_file() else None + return shutil.which('bsdtar') + + +def unpack(archive, dest): + """Unpack an archive into dest (a new folder). ModError when it cannot be.""" + dest.mkdir(parents=True) + try: + if archive.suffix.lower() == '.zip': + try: + with zipfile.ZipFile(archive) as z: + _safe_members(z.namelist()) + z.extractall(dest) + except (zipfile.BadZipFile, NotImplementedError, RuntimeError) as e: + raise ModError(f'it is not a zip file the launcher can read ({e})') + else: + tar = _tar() + if not tar: + raise ModError(f'this PC has no tool for {archive.suffix} files: unpack it yourself into a folder of ' + 'the mods folder') + flags = {'creationflags': 0x08000000} if os.name == 'nt' else {} + r = subprocess.run([tar, '-tf', str(archive)], capture_output=True, text=True, errors='replace', **flags) + if r.returncode: + raise ModError(f'it could not be read ({r.stderr.strip()[:200] or "tar " + str(r.returncode)}): ' + 'unpack it yourself into a folder of the mods folder') + _safe_members(r.stdout.splitlines()) + r = subprocess.run([tar, '-xf', str(archive), '-C', str(dest)], capture_output=True, text=True, + errors='replace', **flags) + if r.returncode: + raise ModError(f'it could not be unpacked ({r.stderr.strip()[:200] or "tar " + str(r.returncode)}): ' + 'unpack it yourself into a folder of the mods folder') + except BaseException: + shutil.rmtree(dest, ignore_errors=True) + raise + + +def prepare(game_dir, log=None): + """Put the mods folder in order for the game (see the top of this file). Returns what it did, one sentence each.""" + folder = mods_dir(game_dir) + done = [] + if not folder.is_dir(): + return done + + def note(text): + done.append(text) + if log: + log(f'mods: {text}') + + for a in sorted(p for p in folder.iterdir() if p.is_file() and p.suffix.lower() in ARCHIVES): + name = a.stem.strip().rstrip('.') + target = folder / name + if not name or target.exists(): + continue + try: + unpack(a, target) + except (ModError, OSError) as e: + note(f'{a.name} was not unpacked: {e}') + continue + unwrap(target) + note(f'{a.name} unpacked into {target.name}.') + loose = [p for p in folder.iterdir() if p.is_dir() and p.name.lower() in LOOSE_PARTS] + if loose: + target = _unique(folder, UNNAMED) + target.mkdir() + for p in loose: + shutil.move(str(p), str(target / p.name.lower())) + note(f'{", ".join(p.name for p in loose)} moved into a mod folder of its own, {target.name}: rename it as you ' + 'like.') + for d in mod_files.mod_folders(folder): + # an Atmosphere layout straight in the mods folder is a mod as it is; a link is the player's own arrangement + if d.name.lower() in MOD_TOP or d.is_symlink() or getattr(d, 'is_junction', lambda: False)(): + continue + try: + if unwrap(d): + note(f'{d.name}: its files moved up to where the game looks for them.') + except OSError as e: + note(f'{d.name}: its files could not be moved up ({e}).') + return done + + +def add(game_dir, src, log=None): + """Add an archive or a folder the player picked as a mod: unpacked or copied into the mods folder under its name + (a free one), in order for the game. Returns the new mod's folder.""" + src = Path(src) + folder = ensure_mods_dir(game_dir) + if src.is_dir(): + try: + src.resolve().relative_to(folder.resolve()) + raise ModError('it is in the mods folder already') + except ValueError: + pass + name = src.name if src.name.lower() not in MOD_TOP else UNNAMED + target = _unique(folder, name) + if src.name.lower() in MOD_TOP: # the player picked the mod's romfs (or atmosphere) folder itself + target.mkdir() + shutil.copytree(src, target / src.name.lower() if src.name.lower() in LOOSE_PARTS else target / src.name) + else: + shutil.copytree(src, target) + elif src.suffix.lower() in ARCHIVES: + target = _unique(folder, src.stem.strip().rstrip('.') or UNNAMED) + unpack(src, target) + else: + raise ModError('it is no archive (.zip, .7z or .rar) and no folder') + unwrap(target) + if log: + log(f'mods: added {src} as {target.name}') + return target + diff --git a/launcher/ui/app.py b/launcher/ui/app.py index 93007a4d..f9c10bcc 100644 --- a/launcher/ui/app.py +++ b/launcher/ui/app.py @@ -37,7 +37,7 @@ import tkinter.font as tkfont from pathlib import Path from tkinter import filedialog -from .. import NAME, build, feed, gameini, gamefiles, jobs, saves, steam, versions, winsys +from .. import NAME, build, feed, gameini, gamefiles, jobs, mods, saves, steam, versions, winsys from ..context import Context from . import art, picture, sound @@ -54,7 +54,9 @@ P = art.PALETTE # the colour scheme in use: art.set_theme change FRAME_MS = 15 # between two steps of the animations; they move by the time that really passed SPARKS = 22 # the dust drifting over the backdrop DEPTH = {'first_run': 0, 'home': 0, 'job': 1, 'saves': 1, 'settings': 1, 'customize': 1, 'uninstall': 2, - 'color': 2} + 'color': 2, 'mods': 1, 'mod_order': 2} +MODS_PAGE = 7 # mods on a page of the Mods screen, beside its other rows +ORDER_PAGE = 10 # ... of the Load Order screen # the Custom scheme's picker (draw_picker), logical pixels: its box (x, top, width, height), the colour square and # the hue bar (x0, y0, x1, y1) PICKER_BOX = (614, 100, 336, 452) @@ -241,6 +243,9 @@ class App: self.save_message = '' # the last import's result, shown in the import rows' descriptions self.export_message = '' # the last Export Save result, shown in its description self.steam_message = '' # the last Add to Steam result, shown in its description + self.mods_message = '' # what the last change on the Mods screen did, shown in its descriptions + self.mods_page = 0 # the page of the Mods and Load Order screens + self.mods_failed = [] # the mods whose code a build before Play could not take in: Play no longer waits self.steam_busy = False # Add to Steam is making the pictures or writing Steam's files self.pick = None # the colour picker: {'slot', 'h', 's', 'v'} (HSV in 0..1) of the colour being picked self.pick_drag = None # 'square' or 'bar' while the mouse drags on the picker @@ -620,7 +625,7 @@ class App: return next((i for i, r in enumerate(self.rows_home()) if r.label == label), None) def back(self): - if self.screen in ('settings', 'uninstall', 'saves', 'customize', 'job', 'color'): + if self.screen in ('settings', 'uninstall', 'saves', 'customize', 'job', 'color', 'mods', 'mod_order'): self.sounds.play('back') if self.screen == 'color': self.pick_done() @@ -630,6 +635,10 @@ class App: self.show('settings') elif self.screen == 'saves': self.show('home', self.home_row('Saves')) + elif self.screen == 'mods': + self.show('home', self.home_row('Mods')) + elif self.screen == 'mod_order': + self.show('mods', self.mods_back_focus()) elif self.screen == 'customize': self.show('home', self.home_row('Customize Launcher')) elif self.screen == 'job' and self.job and self.job.result is None: @@ -762,6 +771,9 @@ class App: self.after_update_notice() if self.install_staged('a staged update waited for the launcher to start'): return + if a.play: # Play from Steam with a mod's code to build in first (play_only) + self.play() + return if self.settings.get('auto_check'): self.check_now() else: @@ -997,6 +1009,7 @@ class App: desc=('Open the game folder. The game keeps your save in its profile folder, its settings in ' 'nx2navi.ini and its logs in logs. Its saves folder holds saves to import and backups.',)), self.saves_row(gd), + self.mods_row(gd), Row('value', 'Rebuild the Game', rebuild_value, rebuild_style, on=self.start_rebuild, desc=('Build the game again from the files of the last build. Only what changed is compiled. Use it ' 'after changing build tools, or if the game stopped starting.',)), @@ -1522,6 +1535,228 @@ class App: text = f'Save imported: {r["files"]} files from {r["source"].name}.' return text + (f' Your old save is in saves\\{r["backup"].name}.' if r['backup'] else '') + # ---- mods + + def mod_list(self, gd, details=True): + """The game folder's mods in the game's load order ([] before the game is installed or when the folder cannot + be read).""" + if not gd or not Path(gd).is_dir(): + return [] + try: + return mods.scan(gd, details=details) + except OSError as e: + self.ctx.log(f'mods: {e}') + return [] + + def mods_row(self, gd): + """Mods: the screen of the game's mods; orange while a mod's code waits to be built into the game.""" + found = self.mod_list(gd, details=False) + try: + waiting = mods.code_waiting(gd) if found else [] + except OSError: + waiting = [] + on = sum(1 for m in found if m.enabled) + desc = ('Mods change the game: its pictures, models, sounds, text or code. Put a mod you downloaded into the ' + 'mods folder, its .zip, .7z or .rar file as it is or its folder, or select Mods and then Add a Mod: ' + 'Play takes it in. Switch each mod on or off, and choose which wins where two change the same file.',) + if waiting: + desc += (f'{", ".join(waiting)} changes the game\'s code: Play first builds the game again with it, which ' + 'takes a few minutes.',) + return Row('value', 'Mods', 'Build Waiting' if waiting else f'{on} On' if found else 'None', + 'orange_arrow' if waiting else 'arrow', on=lambda: self.show('mods'), desc=desc) + + def mods_page_count(self, n, per): + return max(1, -(-n // per)) + + def rows_mods(self): + gd = self.settings.get('game_dir') + folder = mods.mods_dir(gd) if gd else Path('mods') + found = self.mod_list(gd) + over = mods.overlaps(found) + pages = self.mods_page_count(len(found), MODS_PAGE) + self.mods_page = max(0, min(pages - 1, self.mods_page)) + msg = (self.mods_message,) if self.mods_message else () + rows = [ + Row('value', 'Open Mods Folder', str(folder), on=self.open_mods, + desc=msg + ('Open the mods folder: one folder for each mod. A mod you downloaded goes in as it came, ' + 'an archive (.zip, .7z or .rar) or a folder; Play unpacks archives into folders of their ' + 'own and moves a mod\'s files to where the game looks for them.', + 'To remove a mod, delete its folder and its archive, or switch it off here.')), + Row('value', 'Add a Mod', 'Choose a File', 'arrow', on=lambda: self.add_mod(folder=False), + desc=msg + ('Choose a mod you downloaded, a .zip, .7z or .rar file: it is unpacked into the mods ' + 'folder, ready for the game\'s next start.',)), + Row('value', 'Add a Mod Folder', 'Choose a Folder', 'arrow', on=lambda: self.add_mod(folder=True), + desc=msg + ('Choose the folder of a mod (one with a romfs, exefs or cheats folder in it, or an ' + 'atmosphere folder): it is copied into the mods folder.',)), + ] + if found: + rows.append(Row('divider')) + for m in found[self.mods_page * MODS_PAGE:(self.mods_page + 1) * MODS_PAGE]: + label = fit(self.f['label'], m.name, self.X(LABEL_W - 30)) + if m.usable: + rows.append(Row('toggle', label, on=lambda v, m=m: self.switch_mod(m, v), get=lambda m=m: m.enabled, + desc=lambda m=m: self.mod_desc(m, over))) + else: + rows.append(Row('value', label, 'Not Usable', 'salmon', on=lambda: None, + desc=lambda m=m: self.mod_desc(m, over))) + if pages > 1: + rows.append(Row('enum', 'More Mods', get=lambda: f'Page {self.mods_page + 1} of {pages}', + left=lambda: self.mods_turn(-1, 'mods'), right=lambda: self.mods_turn(1, 'mods'), + desc=(f'{len(found)} mods, {MODS_PAGE} to a page: left and right turn the pages.',))) + if len(found) > 1: + rows.append(Row('value', 'Load Order', '', 'arrow', on=lambda: self.show('mod_order'), + desc=('Where two mods change the same file, the one higher in the load order wins. Set ' + 'the order here.',))) + rows.append(Row('action', 'Back', on=lambda: self.show('home', self.home_row('Mods')), height=40, + desc=('Back to the main menu.',))) + return rows + + def mods_turn(self, d, screen): + self.mods_page += d + self.show(screen, next((i for i, r in enumerate(getattr(self, 'rows_' + screen)()) if r.label == 'More Mods'), + None)) + + def mod_desc(self, m, over): + lines = [self.mods_message] if self.mods_message else [] + if not m.usable: + lines.append(m.problem) + what = [] + if m.files: + what.append(f'replaces {len(m.files):,} of the game\'s files' if len(m.files) > 1 else + 'replaces one of the game\'s files') + if m.patches: + what.append(f'patches the game\'s code ({m.patches} patch file{"s" if m.patches > 1 else ""})') + if m.cheat_file: + what.append('brings cheats, which the Cheats tab of the game\'s menu switches on and off') + if what: + lines.append('It ' + (', '.join(what[:-1]) + ' and ' + what[-1] if len(what) > 1 else what[0]) + '.') + if m.name == mods.STAND_INS: + lines.append('The launcher made it: stand-ins for game files its check found damaged, so that the game ' + 'can load what needs them.') + if m.other_version: + lines.append(f'{m.other_version} of its code patch files are for another version of the game and are ' + 'left out.') + for other, n, wins in over.get(m.name, [])[:3]: + lines.append(f'{n:,} of its files are in {other} too: {"this mod" if wins else other} wins, being higher ' + 'in the load order.') + if m.waiting and m.enabled: + lines.append('Its code is not in the game yet: Play first builds the game again with it, which takes a ' + 'few minutes.') + if m.usable: + lines.append('On: the game loads it at its next start. Off: it stays in the mods folder, unused.') + return {'title': m.name, 'text': lines} + + def game_busy(self, gd): + return bool((self.game_proc and self.game_proc.poll() is None) or (gd and saves.game_running(gd))) + + def switch_mod(self, m, on): + gd = self.settings.get('game_dir') + if self.game_busy(gd): + self.mods_message = 'Quit the game first: it reads the mods when it starts.' + return + try: + mods.set_enabled(gd, m.name, on) + except OSError as e: + self.mods_message = f'The game\'s settings file could not be written: {e}' + return + m.enabled, self.mods_message = bool(on), '' + self.ctx.log(f'mods: {m.name} {"on" if on else "off"}') + + def open_mods(self): + gd = self.settings.get('game_dir') + if not gd or not Path(gd).is_dir(): + self.mods_message = 'Install the game first.' + self.show('mods', self.focus) + return + try: + folder = mods.ensure_mods_dir(gd) + except OSError as e: + self.mods_message = f'The mods folder could not be made: {e}' + self.show('mods', self.focus) + return + self.open(folder) + + def add_mod(self, folder): + gd = self.settings.get('game_dir') + if not gd or not Path(gd).is_dir(): + self.mods_message = 'Install the game first.' + else: + if folder: + src = filedialog.askdirectory(parent=self.root, title='Choose the folder of the mod', + initialdir=str(Path.home() / 'Downloads'), mustexist=True) + else: + src = filedialog.askopenfilename(parent=self.root, title='Choose the mod', + initialdir=str(Path.home() / 'Downloads'), + filetypes=[('Mod', '*.zip *.7z *.rar'), ('All files', '*.*')]) + if not src: + return + try: + target = mods.add(gd, src, log=self.ctx.log) + except (mods.ModError, OSError) as e: + self.mods_message = f'{Path(src).name} was not added: {e}.' + else: + added = next((m for m in self.mod_list(gd) if m.name == target.name), None) + self.mods_message = f'Added {target.name}.' + (f' {added.problem}' if added and not added.usable else + ' It is on: the game loads it at its next start.') + self.show('mods', self.focus) + + def rows_mod_order(self): + gd = self.settings.get('game_dir') + found = self.mod_list(gd, details=False) + pages = self.mods_page_count(len(found), ORDER_PAGE) + self.mods_page = max(0, min(pages - 1, self.mods_page)) + rows = [] + first = self.mods_page * ORDER_PAGE + for k, m in enumerate(found[first:first + ORDER_PAGE], first + 1): + rows.append(Row('enum', fit(self.f['label'], m.name, self.X(LABEL_W - 30)), get=lambda k=k: f'{k}', + left=lambda m=m: self.order_move(m.name, -1), right=lambda m=m: self.order_move(m.name, 1), + desc=lambda m=m, k=k: {'title': m.name, 'text': [ + f'Number {k} of {len(found)} in the load order{" (off)" if not m.enabled else ""}. ' + 'Left moves it up: it wins over the mods below it where two change the same file. ' + 'Right moves it down.'] + ([self.mods_message] if self.mods_message else [])})) + if pages > 1: + rows.append(Row('enum', 'More Mods', get=lambda: f'Page {self.mods_page + 1} of {pages}', + left=lambda: self.mods_turn(-1, 'mod_order'), right=lambda: self.mods_turn(1, 'mod_order'), + desc=(f'{len(found)} mods, {ORDER_PAGE} to a page: left and right turn the pages.',))) + rows.append(Row('action', 'Back', on=lambda: self.show('mods', self.mods_back_focus()), height=40, + desc=('Back to the mods.',))) + return rows + + def mods_back_focus(self): + return next((i for i, r in enumerate(self.rows_mods()) if r.label == 'Load Order'), None) + + def order_move(self, name, d): + gd = self.settings.get('game_dir') + if self.game_busy(gd): + self.mods_message = 'Quit the game first: it reads the mods when it starts.' + return + try: + mods.move(gd, name, d) + except OSError as e: + self.mods_message = f'The game\'s settings file could not be written: {e}' + return + self.mods_message = '' + names = [m.name for m in self.mod_list(gd, details=False)] + at = names.index(name) if name in names else 0 + self.mods_page = at // ORDER_PAGE + self.show('mod_order', at % ORDER_PAGE) + + def mods_before_play(self, gd): + """Put the mods folder in order (mods.prepare) and say whether the game must be built again for a mod's code + first: the mods that wait for it, [] when none do.""" + try: + for line in mods.prepare(gd, log=self.ctx.log): + self.mods_message = line + if 'was not unpacked' in line: # on the home screen too, when the launcher comes back + self.message = line + except OSError as e: + self.ctx.log(f'mods: the mods folder could not be put in order: {e}') + try: + return mods.code_waiting(gd) if build.can_rebuild(gd) else [] + except OSError as e: + self.ctx.log(f'mods: {e}') + return [] + def update_desc(self): r = self.release nv, mine = versions.display(r.version), versions.display(self.layout.version) @@ -1598,7 +1833,8 @@ class App: if self.screen == 'home': self.show('home', self.focus) - def play(self): + def play(self, built=False): + """built: the build for the mods' code (ModsRebuildJob) just ended, so the game starts without asking again.""" gd = self.settings.get('game_dir') exe = Path(gd) / 'ocarina.exe' if not exe.is_file(): @@ -1611,6 +1847,19 @@ class App: self.message = 'The game is already running.' self.render() return + waiting = [] if built else self.mods_before_play(gd) + if waiting and waiting != self.mods_failed: + # a mod's code cheats or ExeFS patches the game was not built with: built in first (tools/make_game.py + # lift), then the game starts (job_finished). After a build that failed for the same mods the game + # starts without them, so that a build that cannot finish never keeps the player from the game. + self.ctx.log(f'play: the game is built again first for the code of {", ".join(waiting)}') + job = jobs.ModsRebuildJob(self.ctx, gd) + job.waiting = waiting + self.run_job(job, 'home') + self.message = f'Building the game with the code of {", ".join(waiting)}; it starts after.' + return + if waiting: + self.ctx.log(f'play: without the code of {", ".join(waiting)}, whose build did not finish') try: self.game_proc = winsys.start_game(exe) except OSError as e: @@ -1914,6 +2163,14 @@ class App: self.message = 'The update is ready: start NX2navi again to finish it.' elif j.after == 'staged': self.message = 'The update was downloaded; a checkout keeps running from the repository.' + elif j.after == 'play': # the mods' code is built in (play): the game starts + if not self.settings.get('keep_cache'): + self.drop_build_cache() + self.message, self.mods_failed = '', [] + self.show('home', self.home_row('Play')) + self.play(built=True) + return + self.mods_failed = [] if not self.settings.get('keep_cache') and j.kind in ('install', 'update', 'rebuild'): self.drop_build_cache() if j.kind == 'install': @@ -1930,8 +2187,10 @@ class App: if j.kind == 'install' and self.settings.get('auto_check'): self.check_now() elif j.result == 'cancelled': + self.mods_failed = getattr(j, 'waiting', self.mods_failed) self.show(self.job_back) else: + self.mods_failed = getattr(j, 'waiting', self.mods_failed) self.show('job') def drop_build_cache(self): @@ -2447,6 +2706,16 @@ def play_only(ctx): if saves.game_running(gd): ctx.log('play from Steam: the game is already running') return 0 + try: + for line in mods.prepare(gd, log=ctx.log): + pass + waiting = mods.code_waiting(gd) if build.can_rebuild(gd) else [] + except OSError as e: + ctx.log(f'play from Steam: mods: {e}') + waiting = [] + if waiting: # the window builds the mods' code in, then starts the game (App.start with --play) + ctx.log(f'play from Steam: the game is built again first for the code of {", ".join(waiting)}') + return None try: proc = winsys.start_game(exe) except OSError as e: diff --git a/launcher/ui/selftest.py b/launcher/ui/selftest.py index 7eeec6c1..a5db2193 100644 --- a/launcher/ui/selftest.py +++ b/launcher/ui/selftest.py @@ -216,12 +216,40 @@ def states(app): app.apply_look() app.show('home', 0) + was_game = s.get('game_dir') + + def mods_screen(): # a game folder of made-up mods, removed when the selftest closes + import atexit + import shutil + import tempfile + game = Path(tempfile.mkdtemp(prefix='nx2navi-selftest-')) + atexit.register(shutil.rmtree, game, True) + for name, files in (('HD Textures', 40), ('Better Menus', 12), ('Classic Music', 3)): + for k in range(files): + f = game / 'mods' / name / 'romfs' / ('UI' if k < 6 else name.split()[0]) / f'file{k}.bin' + f.parent.mkdir(parents=True, exist_ok=True) + f.write_bytes(b'0') + (game / 'mods' / 'Old Plugin' / 'exefs').mkdir(parents=True) + (game / 'mods' / 'Old Plugin' / 'exefs' / 'subsdk9').write_bytes(b'0') + (game / 'nx2navi.ini').write_text('[mods]\nmods_disabled = Classic Music\n', encoding='utf-8') + s['game_dir'] = str(game) + app.mods_page, app.mods_message = 0, '' + app.show('mods', next(i for i, r in enumerate(app.rows_mods()) if r.label == 'HD Textures')) + + def mod_order(): + app.show('mod_order', 1) + + def restore_game(): + s['game_dir'] = was_game + app.show('home', 0) + return [('1-first-run', first_run), ('2-installing', installing), ('3-home', home), ('4-update-ready', update_ready), ('5-updating', updating), ('6-settings', settings), ('7-build-failed', failed), ('8-uninstall', uninstall), ('9-import-saves', import_saves), ('10-shaders', shaders), ('11-update-staged', update_staged), ('12-export-save', export_save), ('13-customize', customize), ('14-color-scheme', color_scheme), ('15-custom-picker', custom_picker), - ('16-first-scheme', restore_scheme)] + ('16-first-scheme', restore_scheme), ('17-mods', mods_screen), ('18-mod-order', mod_order), + ('19-home-again', restore_game)] def run(app, shots): diff --git a/nx2navi.ini.example b/nx2navi.ini.example index 74a27c13..931959ff 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -731,13 +731,13 @@ [mods] # Folder with the player's mods, one subfolder per mod (/romfs, /exefs, /cheats). A relative folder -# is under the game's folder. Default: mods next to the game's exe. The setup builds the ExeFS patches and code cheats -# of the mods folder next to the exe only. +# is under the game's folder. Default: mods next to the game's exe. The build takes the ExeFS patches and code cheats +# of the mods folder next to the exe only. The launcher's Mods screen adds mods and puts downloads in order. # mods_dir = # Load order, top first, names separated by |; a mod not listed goes below the listed ones, by name. The top mod wins a -# file two mods replace. Set it here: the in-game menu has no Mods tab. +# file two mods replace. The launcher's Mods screen sets it (Load Order); the in-game menu has no Mods tab. # mods_order = -# Mods switched off, names separated by |. +# Mods switched off, names separated by |. The launcher's Mods screen sets it. # mods_disabled = # auto: when an enabled mod has cheats that change the game's code and the game was built without them, the game starts # the setup's rebuild at start (a few minutes) and starts again after it. off: the game starts with the code it has. diff --git a/tests/launcher_test.py b/tests/launcher_test.py index c3b54d30..4a3efcbe 100644 --- a/tests/launcher_test.py +++ b/tests/launcher_test.py @@ -1863,7 +1863,8 @@ class MakeGameTest(Tmp): def test_shaders_key_names_the_translator_the_build_uses(self): """A source checkout has its translator in build/xlate/ (tools/build_xlate.py), not in the kit's bin/: the shaders key follows its identity, so a rebuilt translator prepares the shaders again. A kit's bin/ comes first - when both exist, and a kit with no translator is refused with the way to build one.""" + when both exist. A kit with no translator has a key that says so (the keys are worked out before the steps + run, steps_to_run), and the build refuses it with the way to build one when it needs the translator.""" from unittest import mock kit = self.tmp / 'kit' (kit / 'tools').mkdir(parents=True) @@ -1887,8 +1888,9 @@ class MakeGameTest(Tmp): self.assertEqual(key(), from_bin) shutil.rmtree(kit / 'bin') shutil.rmtree(xlate) + self.assertTrue(key().endswith('|missing')) with contextlib.redirect_stderr(io.StringIO()) as err, self.assertRaises(SystemExit): - key() + self.mg.xlate_source() self.assertIn('tools/build_xlate.py', err.getvalue()) def test_prompts_step(self): @@ -1958,12 +1960,13 @@ class MakeGameTest(Tmp): self.assertEqual(build()[2], 3) (art / 'T_X_B_White.png').write_bytes(b'picture two') self.assertEqual(build()[2], 4) - # the lift running again makes the compile and the shaders run, not the prompts + # the lift running again makes the compile run, not the shaders (their key names the lift and the + # compile they depend on, make_game.py UNAFFECTED) nor the prompts st = json.loads((self.out / 'build-cache' / 'state.json').read_text()) del st['steps']['lift'] (self.out / 'build-cache' / 'state.json').write_text(json.dumps(st)) others, ev, runs = build() - self.assertEqual((others, states(ev), runs), (['compile', 'lift', 'shaders'], ['skip'], 4)) + self.assertEqual((others, states(ev), runs), (['compile', 'lift'], ['skip'], 4)) # a kit without art: skipped, the folders an earlier build drew stay shutil.rmtree(kit / 'prompt-art') self.assertEqual(build(), ([], [{'ev': 'step', 'step': 'prompts', 'state': 'skip'}], 4)) @@ -3086,7 +3089,8 @@ class MakeGameTest(Tmp): code, _ = self.main_rebuild(lambda out, cache: None, stop) self.assertEqual(code, 1) st = json.loads((self.out / 'build-cache' / 'state.json').read_text()) - self.assertEqual(sorted(st['steps']), ['exefs', 'lift', 'romfs']) + # the shaders' record stays as the lift left it: the step's key names what it depends on (UNAFFECTED) + self.assertEqual(sorted(s for s in st['steps'] if s != 'shaders'), ['exefs', 'lift', 'romfs']) self.assertEqual(build.built_version(self.out), '') # no finished build on record self.assertTrue(build.can_rebuild(self.out)) self.assertTrue(build.installed(self.out)) @@ -4806,7 +4810,143 @@ class WindowTest(Tmp): r = subprocess.run([sys.executable, '-m', 'launcher', '--root', str(self.tmp / 'w'), '--selftest'], cwd=ROOT, capture_output=True, text=True, timeout=180) self.assertEqual(r.returncode, 0, r.stderr) - self.assertIn('16 screens drawn', r.stdout) + self.assertIn('19 screens drawn', r.stdout) + + +# ---------------------------------------------------------------- mods + +class ModsTest(Tmp): + """The mods folder as the launcher puts it in order for the game (launcher/mods.py) and the build takes its code + in (tools/make_game.py, tools/mod_files.py).""" + + def setUp(self): + super().setUp() + from launcher import mods + self.mods = mods + self.game = self.tmp / 'game' + made_up_game(self.game) + self.folder = self.game / 'mods' + self.folder.mkdir() + + def file(self, rel, text='x'): + p = self.folder / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(text) + return p + + def tree(self): + return sorted(p.relative_to(self.folder).as_posix() for p in self.folder.rglob('*') if p.is_file()) + + def test_downloads_are_put_where_the_game_looks(self): + with zipfile.ZipFile(self.folder / 'Zipped.zip', 'w') as z: # a zip holding the mod's own folder + z.writestr('Zipped/romfs/Actor/a.bfres', 'a') + z.writestr('Zipped/README.md', 'r') + with zipfile.ZipFile(self.folder / 'Escape.zip', 'w') as z: + z.writestr('../outside.txt', 'e') + self.file('Wrapped/readme.txt') # wrapped twice, a readme on each level + self.file('Wrapped/Wrapped v2/readme.txt') + self.file('Wrapped/Wrapped v2/Wrapped/romfs/UI/b.txt') + self.file('romfs/Actor/c.bfres') # a mod's romfs straight in the mods folder + self.file('atmosphere/contents/0400486024DEC000/romfs/d.txt') # an Atmosphere layout: a mod as it is + self.file('Options/Option A/romfs/e') # a download with options: left alone + self.file('Options/Option B/romfs/e') + done = self.mods.prepare(self.game) + self.assertEqual(self.tree(), [ + 'Escape.zip', 'Options/Option A/romfs/e', 'Options/Option B/romfs/e', 'Unnamed mod/romfs/Actor/c.bfres', + 'Wrapped/readme (2).txt', 'Wrapped/readme.txt', 'Wrapped/romfs/UI/b.txt', 'Zipped.zip', + 'Zipped/README.md', 'Zipped/romfs/Actor/a.bfres', + 'atmosphere/contents/0400486024DEC000/romfs/d.txt']) + self.assertFalse((self.game / 'outside.txt').exists()) + self.assertTrue(any(line.startswith('Escape.zip was not unpacked') for line in done)) + self.assertEqual(self.mods.prepare(self.game), [line for line in done if 'Escape.zip' in line]) # once + found = {m.name: m for m in self.mods.scan(self.game)} + self.assertEqual(sorted(n for n, m in found.items() if m.usable), + ['Unnamed mod', 'Wrapped', 'Zipped', 'atmosphere']) + self.assertIn('options', found['Options'].problem) + self.assertEqual(found['Zipped'].files, ['actor/a.bfres']) + + def test_what_the_game_cannot_use_is_named(self): + self.file('Plugin/exefs/subsdk9') + self.file('Other/atmosphere/contents/01007EF00011E000/romfs/x') + self.file('Readme only/readme.txt') + self.file('Old patch/exefs/0123456789ABCDEF0123456789ABCDEF.ips', 'PATCHEOF') + self.file('Patch/exefs/E0F3CCFAF50065AE4EA01EB5AEB13B1A.ips', 'PATCHEOF') + found = {m.name: m for m in self.mods.scan(self.game)} + self.assertIn('code module', found['Plugin'].problem) + self.assertIn('01007EF00011E000', found['Other'].problem) + self.assertIn('nothing the game reads', found['Readme only'].problem) + self.assertIn('another version', found['Old patch'].problem) + self.assertEqual((found['Patch'].problem, found['Patch'].patches), ('', 1)) + + def test_switches_and_order_are_the_games_settings(self): + for name in ('beta', 'Alpha', ' Lead'): + self.file(f'{name}/romfs/same.txt', name) + self.assertEqual([m.name for m in self.mods.scan(self.game)], [' Lead', 'Alpha', 'beta']) # strcmp + self.mods.set_enabled(self.game, 'Alpha', False) + self.mods.move(self.game, 'beta', -2) + self.mods.set_enabled(self.game, ' Lead', False) + self.mods.set_enabled(self.game, 'Alpha', True) + self.assertEqual(gameini.get(self.game, 'MODS_ORDER'), 'beta| Lead|Alpha') + self.assertEqual(gameini.get(self.game, 'MODS_DISABLED'), ' Lead') + self.assertIn('mods_disabled = " Lead"', (self.game / 'nx2navi.ini').read_text()) # as runtime/cfg.c quotes + found = self.mods.scan(self.game) + self.assertEqual([(m.name, m.enabled) for m in found], [('beta', True), (' Lead', False), ('Alpha', True)]) + self.assertEqual(self.mods.overlaps(found), {'beta': [('Alpha', 1, True)], 'Alpha': [('beta', 1, False)]}) + + def test_a_picked_archive_or_folder_is_added(self): + src = self.tmp / 'Downloads' + src.mkdir() + with zipfile.ZipFile(src / 'Picked.zip', 'w') as z: + z.writestr('romfs/a.txt', 'a') + (src / 'Folder Mod' / 'Inner' / 'exefs').mkdir(parents=True) + (src / 'Folder Mod' / 'Inner' / 'exefs' / 'E0F3CCFAF50065AE.ips').write_text('PATCHEOF') + (src / 'romfs').mkdir() + (src / 'romfs' / 'b.txt').write_text('b') + self.assertEqual(self.mods.add(self.game, src / 'Picked.zip').name, 'Picked') + self.assertEqual(self.mods.add(self.game, src / 'Picked.zip').name, 'Picked 2') + self.assertEqual(self.mods.add(self.game, src / 'Folder Mod').name, 'Folder Mod') + self.assertEqual(self.mods.add(self.game, src / 'romfs').name, 'Unnamed mod') + self.assertEqual(self.tree(), ['Folder Mod/exefs/E0F3CCFAF50065AE.ips', 'Picked 2/romfs/a.txt', + 'Picked/romfs/a.txt', 'Unnamed mod/romfs/b.txt']) + self.assertTrue((src / 'Picked.zip').is_file()) # the player's download stays where it was + with self.assertRaises(self.mods.ModError): + self.mods.add(self.game, src / 'Folder Mod' / 'Inner' / 'exefs' / 'E0F3CCFAF50065AE.ips') + + def test_code_waits_for_a_build_that_saw_it(self): + import make_game + cheat = self.file('Cheats/cheats/E0F3CCFAF50065AE.txt', '[Moon jump]\n04000000 00000000 D503201F\n') + self.file('Textures/romfs/a.txt') + self.assertEqual(self.mods.code_waiting(self.game), ['Cheats']) + self.mods.set_enabled(self.game, 'Cheats', False) + self.assertEqual(self.mods.code_waiting(self.game), []) # a mod switched off waits for nothing + self.mods.set_enabled(self.game, 'Cheats', True) + (self.game / 'build-cache' / 'mod-code.json').write_text(json.dumps(make_game.mod_code(self.game))) + self.assertEqual(self.mods.code_waiting(self.game), []) # the last build took it up + cheat.write_text('[Moon jump]\n04000000 00000000 D65F03C0\n') + self.assertEqual(self.mods.code_waiting(self.game), ['Cheats']) # edited since + + def test_the_mods_code_is_in_the_lift_key_and_not_the_shaders_key(self): + import make_game + kit = self.tmp / 'kit' + for d in ('tools', 'recomp', 'runtime', 'bin'): + (kit / d).mkdir(parents=True) + for f in ('tools/prepare_shaders.py', 'tools/cheats.py', 'tools/exefs_patches.py', 'tools/mod_files.py', + 'recomp/patches-v111.txt', 'bin/' + make_game.XLATE_NAME, 'bin/nx2navi_xlate.id', 'VERSION'): + (kit / f).write_text('1\n') + + def keys(): + return {s: make_game.step_key(s, 'game', self.game) for s in ('lift', 'compile', 'shaders')} + with mock.patch.object(make_game, 'KIT', kit): + plain = keys() + self.file('Textures/romfs/a.txt') + self.assertEqual(keys(), plain) # files of the game's data: no build + self.assertNotIn('|mods ', plain['lift']) + self.file('Patch/exefs/E0F3CCFAF50065AE.ips', 'PATCHEOF') + with_patch = keys() + self.assertTrue(with_patch['lift'].startswith(plain['lift'] + '|mods ')) + self.assertNotEqual(with_patch['compile'], plain['compile']) + self.assertEqual(with_patch['shaders'], plain['shaders']) + self.assertEqual(list(make_game.mod_code(self.game)), ['Patch/exefs/E0F3CCFAF50065AE.ips']) if __name__ == '__main__': diff --git a/tools/cheats.py b/tools/cheats.py index 65df24de..db0c9c58 100644 --- a/tools/cheats.py +++ b/tools/cheats.py @@ -30,7 +30,7 @@ hash: the same over the cheat's words, so the runtime sees an edited cheat that A code cheat that calls or jumps into a code cave it writes (words where main has zeros) gets the cave's body inlined at the call site; a conditional skip of one store in the body (cbz wN, +8; str wN, ...) is dropped. -Standard library and recomp/ only. +Standard library, recomp/ and tools/mod_files.py only. """ import argparse import struct @@ -38,9 +38,11 @@ import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'recomp')) +sys.path.insert(0, str(Path(__file__).resolve().parent)) import flags # noqa: E402 from lifter import Lifter, Unsupported # noqa: E402 from nso import Nso # noqa: E402 +from mod_files import cheat_files # noqa: E402 BASE = 0x7000000000 # main's base in the layout (recomp/layout.py); only pc-relative decoding sees it NOP = 0xD503201F @@ -90,25 +92,6 @@ def parse(text, mod): return out -TITLE = '0400486024DEC000' # the game's title id: a mod's Atmosphere folders are named after it - - -def cheat_files(mods, bid): - """[(mod folder name, cheat file)] of the folders of mods, by name: /cheats/.txt or its Atmosphere places - (runtime/mods.c find_root), the file name in any case.""" - out = [] - for mod in sorted((d for d in mods.iterdir() if d.is_dir()), key=lambda d: d.name) if mods.is_dir() else []: - for where in ('cheats', f'atmosphere/contents/{TITLE}/cheats', f'contents/{TITLE}/cheats', f'{TITLE}/cheats'): - folder = mod / where - if folder.is_dir(): - f = next((p for p in sorted(folder.iterdir()) if p.is_file() and p.name.lower() == f'{bid}.txt'.lower()), - None) - if f: - out.append((mod.name, f)) - break - return out - - def ops(words): """[(type, [words])] of a cheat; type is the opcode nibble, 0xC0.. for the extended ones. ValueError when the last opcode is cut short.""" diff --git a/tools/exefs_patches.py b/tools/exefs_patches.py index 5280c7b8..1183a5c7 100644 --- a/tools/exefs_patches.py +++ b/tools/exefs_patches.py @@ -29,7 +29,7 @@ comma separated, or "-": the runtime switches them off while the mod is on, so t A file the rules cannot express is unsupported as a whole: half of a patch is worse than none. -Standard library, recomp/ and tools/cheats.py only. +Standard library, recomp/, tools/cheats.py and tools/mod_files.py only. """ import argparse import re @@ -38,29 +38,10 @@ import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) -from cheats import TITLE, Main, convert, fnv # noqa: E402 +from cheats import Main, convert, fnv # noqa: E402 +from mod_files import patch_files # noqa: E402 NSO_HEADER = 0x100 -EXEFS_PLACES = ('exefs', f'atmosphere/contents/{TITLE}/exefs', f'contents/{TITLE}/exefs', f'{TITLE}/exefs') - - -def patch_files(mods): - """[(mod folder name, path in the mod, file)] of the folders of mods, by name: the .ips and .pchtxt files of the - first exefs place that exists (runtime/mods.c find_root) and of atmosphere/exefs_patches//.""" - out = [] - for mod in sorted((d for d in mods.iterdir() if d.is_dir()), key=lambda d: d.name) if mods.is_dir() else []: - folders = [] - exefs = next((mod / w for w in EXEFS_PLACES if (mod / w).is_dir()), None) - if exefs: - folders.append(exefs) - patches = mod / 'atmosphere' / 'exefs_patches' - if patches.is_dir(): - folders += sorted(d for d in patches.iterdir() if d.is_dir()) - for folder in folders: - for f in sorted(folder.iterdir()): - if f.is_file() and f.suffix.lower() in ('.ips', '.pchtxt'): - out.append((mod.name, f.relative_to(mod).as_posix(), f)) - return out def same_build(name, build_id): diff --git a/tools/make_game.py b/tools/make_game.py index f271eea0..4a4c228d 100644 --- a/tools/make_game.py +++ b/tools/make_game.py @@ -12,7 +12,11 @@ Steps, each skipped when a run before finished it for the same inputs (state in 3. romfs every RomFS file, checked against the container's sha256, into one image -> /romfs/romfs.bin and index.json (16-byte aligned, in the byte order of the paths). When the lift and the compile follow, it runs on a thread of its own beside them: only the RomFS index and romfs.c wait for the image - 4. lift recomp/gen.py with recomp/patches-v111.txt -> /build-cache/lift, plus the two SDK bring-up fixes + 4. lift recomp/gen.py with recomp/patches-v111.txt -> /build-cache/lift, plus the two SDK bring-up fixes. + The code cheats and ExeFS patch files of the mods in /mods (tools/mod_files.py) become patch sets + first (tools/cheats.py, tools/exefs_patches.py), which the game switches on with their mod; the step's + key names those files, and build-cache/mod-code.json records the ones it took up (the launcher + rebuilds the game before it starts when a mod brings others, launcher/mods.py) 5. compile the lifted C and runtime/ with MSYS2's clang, linked into /ocarina.exe; the MSYS2 DLLs it loads, the shader translator and romfs-index.bin beside it 6. shaders every shader in the game files (tools/prepare_shaders.py, default bind state) translated, and each one's @@ -84,6 +88,7 @@ from pathlib import Path KIT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(KIT / 'tools')) import nup # noqa: E402 +import mod_files # noqa: E402 import prompt_art # noqa: E402 TITLE_ID = 0x0400486024DEC000 @@ -99,7 +104,10 @@ OWN_KEY_STEPS = ('prompts', 'scan') # (a step that runs, a later step that reads nothing it writes): the later step is not made to run again. The lift # reads only the unpacked ExeFS (step_key), so the RomFS step leaves it alone: an install that was cut short while the # image was still being packed, the lift beside it already done, packs the image again but does not lift again. -UNAFFECTED = {('romfs', 'lift')} +# The shaders step's key names the lift and the compile it depends on without the mods' code (step_key): a lift and a +# compile that run for a mod's code cheats or ExeFS patches leave the shaders prepared, as they leave the game's +# shaders as they were, and one that runs for anything else (another kit) changes that key and prepares them again. +UNAFFECTED = {('romfs', 'lift'), ('lift', 'shaders'), ('compile', 'shaders')} # The Python packages tools/pad_prompts.py imports (pip name: import name), checked before a build with button art. PROMPT_PACKAGES = {'zstandard': 'zstandard', 'fonttools': 'fontTools', 'pillow': 'PIL'} # The line tools/pad_prompts.py prints for each overlay folder it wrote. @@ -517,12 +525,35 @@ def lift_digest(): return h.hexdigest() -def step_lift(out, cache): - """gen.py into build-cache/lift. The folder is kept: gen.py rewrites only the chunks whose text changed and - removes the ones it no longer writes.""" - lift = cache / 'lift' - cmd = py_cmd(KIT / 'recomp' / 'gen.py', '--exefs', out / 'exefs', '--out', lift, - '--patches', KIT / 'recomp' / 'patches-v111.txt') +# The mods' code (tools/mod_files.py): the tools that make patch sets of it, the patch file each writes, the status +# file the runtime reads in build-cache next to the exe (runtime/cheats.c, runtime/mods.c) and what the log calls it +MOD_STAGES = (('cheats.py', 'patches-cheats.txt', 'cheats.status', 'code cheats'), + ('exefs_patches.py', 'patches-exefs.txt', 'exefs.status', 'ExeFS patches')) +# build-cache/: {"/": sha256} of the files the last lift took up, built in or refused (the status files say +# which): the launcher builds the game again before Play only for files no build has seen (launcher/mods.py) +MOD_RECORD = 'mod-code.json' + + +def mod_code(out): + """{"/": sha256} of the code cheats and ExeFS patch files of /mods for main's build.""" + return mod_files.code_record(out / 'mods', mod_files.main_build_id(out / 'exefs' / 'main')) + + +def mod_digest(out): + """The mods' part of the lift's key: '' without mod code (the key of a game without mods stays what it was), else + the files by name and content and the tools that turn them into patch sets.""" + code = mod_code(out) + if not code: + return '' + h = hashlib.sha256(json.dumps(code, sort_keys=True).encode()) + for tool in ('cheats.py', 'exefs_patches.py', 'mod_files.py'): + h.update(sha256_file(KIT / 'tools' / tool).encode() if (KIT / 'tools' / tool).is_file() else b'no tool') + return h.hexdigest() + + +def run_lifter(out, lift, patches): + """gen.py with the patch file into lift; its exit code.""" + cmd = py_cmd(KIT / 'recomp' / 'gen.py', '--exefs', out / 'exefs', '--out', lift, '--patches', patches) p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding='utf-8', errors='replace', bufsize=1) for line in p.stdout: @@ -531,8 +562,52 @@ def step_lift(out, cache): m = EMITTED.search(line) if m: emit('progress', step='lift', done=int(m.group(1)), total=int(m.group(2)), unit='chunks') - if p.wait(): - fail('the code generator failed (see above)') + return p.wait() + + +def mod_patches(out, cache, plain): + """The patch file with the mods' code cheats and ExeFS patches added (plain when there are none), the status files + written, and the mods' code. Each status file goes first: without it the runtime takes every such file as not + built. The files are the player's downloads, so one the tools refuse costs its stage, never the build.""" + patches, done = plain, [] + code = mod_code(out) + for tool, written, status, what in MOD_STAGES: + (cache / status).unlink(missing_ok=True) + if not code: + continue + cmd = py_cmd(KIT / 'tools' / tool, '--exefs', out / 'exefs', '--mods', out / 'mods', '--patches', patches, + '--out', cache / written, '--status', cache / status) + r = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding='utf-8', + errors='replace') + for line in r.stdout.splitlines(): + say(f' {line}') + if r.returncode: + (cache / status).unlink(missing_ok=True) + say(f' the mods\' {what} could not be built in (see above): the game is built without them') + else: + patches = cache / written + done.append(status) + return patches, done, code + + +def step_lift(out, cache): + """gen.py into build-cache/lift. The folder is kept: gen.py rewrites only the chunks whose text changed and + removes the ones it no longer writes. The mods' code cheats and ExeFS patches go in as patch sets (mod_patches); + should the lifter refuse those, the game is lifted without them.""" + lift = cache / 'lift' + plain = KIT / 'recomp' / 'patches-v111.txt' + (cache / MOD_RECORD).unlink(missing_ok=True) + patches, done, code = mod_patches(out, cache, plain) + if run_lifter(out, lift, patches): + if patches == plain: + fail('the code generator failed (see above)') + say(' the code generator refused the mods\' code: the game is lifted without it') + for status in done: + (cache / status).unlink(missing_ok=True) + if run_lifter(out, lift, plain): + fail('the code generator failed (see above)') + if code: + (cache / MOD_RECORD).write_text(json.dumps(code, indent=1, sort_keys=True), encoding='utf-8') for mod, old, new in LIFT_FIXES: hits = [p for p in sorted((lift / mod).glob('*.c')) if old in p.read_text(encoding='utf-8')] if len(hits) != 1 or hits[0].read_text(encoding='utf-8').count(old) != 1: @@ -1559,6 +1634,15 @@ def xlate_source(): fail(f'{XLATE_NAME} is missing: build it with tools/build_xlate.py') +def xlate_id(): + """The shader translator's id file's sha256 for the shaders step's key, 'missing' without a translator: the key is + worked out before the steps run (steps_to_run), and the shaders step itself says what is missing when it runs.""" + for d in (KIT / 'bin', KIT / 'build' / 'xlate'): + if (d / XLATE_NAME).is_file(): + return sha256_file(d / 'nx2navi_xlate.id') + return 'missing' + + # The x86-64 level the lifted code was compiled for, beside the objects (lifted_level). LIFTED_BUILD = 'lifted-build.json' LIFTED_OBJECTS = ('main_', 'sdk_', 'subsdk0_') # the objects of the lifted code, by the module their names start with @@ -2208,10 +2292,12 @@ def step_scan(out): def step_key(step, identity, out=None): - """What a step's output depends on. exefs and romfs: the game files only; lift: the unpacked ExeFS and the - lifter; compile: the lift, the runtime sources, this script and the processor (cpu_tag); prompts: the game files, - the kit's button art, the version of the tools that draw it (PROMPT_TOOLS), the bind_ lines of nx2navi.ini, and - whether every folder it writes is still there; scan: the game files, the check's version (SCAN_TOOL) and whether its report is still there.""" + """What a step's output depends on. exefs and romfs: the game files only; lift: the unpacked ExeFS, the lifter and + the mods' code cheats and ExeFS patches (mod_digest); compile: the lift, the runtime sources, this script and the + processor (cpu_tag); shaders: the compile without the mods' code, the preparing script and the translator; + prompts: the game files, the kit's button art, the version of the tools that draw it (PROMPT_TOOLS), the bind_ + lines of nx2navi.ini, and whether every folder it writes is still there; scan: the game files, the check's version + (SCAN_TOOL) and whether its report is still there.""" if step in ('exefs', 'romfs'): return f'{step}-1|{identity}' if step == 'scan': @@ -2223,17 +2309,21 @@ def step_key(step, identity, out=None): written = all((out / f / 'manifest.tsv').is_file() for f in prompt_art.OVERLAYS) return (f'prompts-1|{identity}|{tree_digest(prompt_art_dir(), ("**/*.png",))}|{PROMPT_TOOLS}|{binds}' f'|{"written" if written else "missing"}') - lift = f'lift|{exefs_digest(out)}|{lift_digest()}' + base = f'lift|{exefs_digest(out)}|{lift_digest()}' + mods = mod_digest(out) if step in ('lift', 'compile') else '' + lift = f'{base}|mods {mods}' if mods else base if step == 'lift': return lift - compile_key = (f'compile|{lift}|{tree_digest(KIT / "runtime", ("**/*",))}|{sha256_file(Path(__file__))}|{kit_version()}' - f'|cpu {cpu_tag()}') + + def compile_of(lift): + return (f'compile|{lift}|{tree_digest(KIT / "runtime", ("**/*",))}|{sha256_file(Path(__file__))}' + f'|{kit_version()}|cpu {cpu_tag()}') if step == 'compile': - return compile_key - # shaders: the built game and the translator that prepared them, the one xlate_source finds (the kit's bin/, or a - # source checkout's build/xlate/) - return (f'shaders-1|{compile_key}|{sha256_file(KIT / "tools" / "prepare_shaders.py")}' - f'|{sha256_file(xlate_source() / "nx2navi_xlate.id")}') + return compile_of(lift) + # shaders: the built game without the mods' code (UNAFFECTED) and the translator that prepared them, the one + # xlate_source finds (the kit's bin/, or a source checkout's build/xlate/) + return (f'shaders-1|{compile_of(base)}|{sha256_file(KIT / "tools" / "prepare_shaders.py")}' + f'|{xlate_id()}') def out_path(arg): diff --git a/tools/make_kit.py b/tools/make_kit.py index f71a0cf5..bc304307 100644 --- a/tools/make_kit.py +++ b/tools/make_kit.py @@ -31,9 +31,11 @@ NAME = 'NX2navi-BuildKit' TOOLS = ('make_game.py', 'nup.py', 'romfs_index.py', 'check_content.py', 'hostmem.py', 'prepare_shaders.py', 'shader_census.py', 'shader_list.py', 'pipeline_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py', 'romfs_check.py', 'build_game.py', 'check_isa.py', 'build_shaders.py', 'make_dlaa_mask.py', 'make_mvobj_fs.py', - 'pack_shader_cache.py', 'vulkan_pin.py') # make_game.py imports build_game's vma_include and vulkan_include on -# Linux for the VMA header and for Vulkan headers newer than the system's (which build_game takes from vulkan_pin), and -# build_game.py imports check_isa and build_shaders, which reads the two make_ scripts' shaders + 'pack_shader_cache.py', 'vulkan_pin.py', 'mod_files.py', 'cheats.py', 'exefs_patches.py') +# make_game.py imports build_game's vma_include and vulkan_include on Linux for the VMA header and for Vulkan headers +# newer than the system's (which build_game takes from vulkan_pin), and build_game.py imports check_isa and +# build_shaders, which reads the two make_ scripts' shaders; the lift runs cheats.py and exefs_patches.py for the mods' +# code, and both import mod_files.py TOP = ('LICENSE', 'LICENSE-ADDITIONS', 'THIRD_PARTY_NOTICES.md', 'VERSION', 'nx2navi.ini.example') SKIP_DIRS = {'__pycache__', '.pytest_cache'} SKIP_SUFFIXES = {'.o', '.obj', '.pyc', '.exe', '.dll', '.a', '.lib'} diff --git a/tools/make_launcher.py b/tools/make_launcher.py index d154869a..98e8d815 100644 --- a/tools/make_launcher.py +++ b/tools/make_launcher.py @@ -80,7 +80,7 @@ PINS = { } TOOLS = ('make_game.py', 'nup.py', 'romfs_index.py', 'check_content.py', 'hostmem.py', 'prepare_shaders.py', 'shader_census.py', 'shader_list.py', 'pipeline_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py', 'romfs_check.py', - 'pack_shader_cache.py') + 'pack_shader_cache.py', 'mod_files.py', 'cheats.py', 'exefs_patches.py') TOP = ('LICENSE', 'LICENSE-ADDITIONS', 'THIRD_PARTY_NOTICES.md', 'nx2navi.ini.example') SKIP_DIRS = {'__pycache__', '.pytest_cache', 'native'} SKIP_SUFFIXES = {'.o', '.obj', '.pyc', '.exe', '.dll', '.a', '.lib'} diff --git a/tools/make_release.py b/tools/make_release.py index 0b36ee0b..9df4d11f 100644 --- a/tools/make_release.py +++ b/tools/make_release.py @@ -114,7 +114,7 @@ LLVM_SKIP_SUFFIXES = {'.idl', '.tlb'} TOOLS_FILES = ['build_game.py', 'build_shaders.py', 'button_icons.py', 'check_content.py', 'check_isa.py', 'extract.py', 'hostmem.py', 'make_romfs_patches.py', 'ncz.py', 'pack_shader_cache.py', 'prepare_shaders.py', 'romfs_index.py', 'shader_census.py', 'shader_list.py', 'pipeline_list.py', 'ultrawide_layout.py', 'cheats.py', - 'exefs_patches.py'] + 'exefs_patches.py', 'mod_files.py'] TOOLS_DIRS = ['vendor/macnativegames', 'data'] # data/: expected hashes RUNTIME_SUFFIXES = {'.h', '.inc', '.ico'} # .ico: the installer window icon diff --git a/tools/mod_files.py b/tools/mod_files.py new file mode 100644 index 00000000..14f85920 --- /dev/null +++ b/tools/mod_files.py @@ -0,0 +1,99 @@ +"""The player's mods folder as the game reads it (runtime/mods.c): where each mod keeps its parts, and the files of the +mods that change the game's code. + +One folder per mod (a symbolic link or junction to a folder counts). A part (romfs, romfslite, exefs, cheats) is the +first of /, /atmosphere/contents//<part>, <mod>/contents/<title>/<part> and +<mod>/<title>/<part> that is a folder (runtime/mods.c find_root). + +The files that change the game's code are its Atmosphere cheat file for main's build (<cheats>/<build id>.txt, the +name in any case) and the ExeFS patch files (.ips, .pchtxt) of its exefs part and of +<mod>/atmosphere/exefs_patches/<name>/. The lifted game cannot be patched while it runs, so the build turns them into +patch sets (tools/cheats.py, tools/exefs_patches.py) and the runtime switches those on with their mod; a file the +build did not see waits for the next one. tools/make_game.py builds them in and records what it built +(build-cache/mod-code.json, code_record); the launcher compares that record with the folder (launcher/mods.py) to +rebuild the game before it starts when a mod brought new ones. + +Standard library only: the launcher imports it from the kit's tools folder. +""" +import hashlib +from pathlib import Path + +TITLE = '0400486024DEC000' # the game's title id: a mod's Atmosphere folders are named after it +PLACES = ('{}', f'atmosphere/contents/{TITLE}/{{}}', f'contents/{TITLE}/{{}}', f'{TITLE}/{{}}') +PATCH_SUFFIXES = ('.ips', '.pchtxt') + + +def mod_folders(mods): + """The mod folders of the mods folder, by name (as the setup's tools list them: Python's order of the names).""" + mods = Path(mods) + if not mods.is_dir(): + return [] + return sorted((d for d in mods.iterdir() if d.is_dir()), key=lambda d: d.name) + + +def part(mod, name): + """The folder of a mod's part (romfs, romfslite, exefs, cheats): the first of its places that is a folder, or + None.""" + return next((Path(mod) / w.format(name) for w in PLACES if (Path(mod) / w.format(name)).is_dir()), None) + + +def cheat_files(mods, bid): + """[(mod folder name, cheat file)] of the folders of mods, by name: <cheats>/<bid>.txt, the name in any case. bid is + the first 8 bytes of main's build id in hex, as cheat files name themselves (tools/cheats.py build_id).""" + out = [] + for mod in mod_folders(mods): + folder = part(mod, 'cheats') + if folder: + f = next((p for p in sorted(folder.iterdir()) if p.is_file() and p.name.lower() == f'{bid}.txt'.lower()), + None) + if f: + out.append((mod.name, f)) + return out + + +def patch_files(mods): + """[(mod folder name, path in the mod, file)] of the folders of mods, by name: the .ips and .pchtxt files of the + exefs part and of atmosphere/exefs_patches/<name>/.""" + out = [] + for mod in mod_folders(mods): + folders = [] + exefs = part(mod, 'exefs') + if exefs: + folders.append(exefs) + patches = mod / 'atmosphere' / 'exefs_patches' + if patches.is_dir(): + folders += sorted(d for d in patches.iterdir() if d.is_dir()) + for folder in folders: + for f in sorted(folder.iterdir()): + if f.is_file() and f.suffix.lower() in PATCH_SUFFIXES: + out.append((mod.name, f.relative_to(mod).as_posix(), f)) + return out + + +def main_build_id(main): + """The first 8 bytes of main's build id in upper-case hex (the cheat files' name), '' when main cannot be read.""" + try: + with open(main, 'rb') as f: + head = f.read(0x48) + except OSError: + return '' + return head[0x40:0x48].hex().upper() if len(head) == 0x48 and head[:4] == b'NSO0' else '' + + +def code_files(mods, bid): + """[(mod folder name, path in the mod, file)] of every file of the mods folder that changes the game's code: the + cheat files for bid, then the ExeFS patch files.""" + return ([(m, f.relative_to(Path(mods) / m).as_posix(), f) for m, f in cheat_files(mods, bid)] + + patch_files(mods)) + + +def code_record(mods, bid): + """{"<mod>/<path in the mod>": sha256 of its bytes} of code_files: what a build that lifts them in takes. A file + that cannot be read is left out (the tools would fail on it too, and the next build tries again).""" + out = {} + for mod, rel, f in code_files(mods, bid): + try: + out[f'{mod}/{rel}'] = hashlib.sha256(f.read_bytes()).hexdigest() + except OSError: + pass + return out