diff --git a/docs/README-BuildKit.md b/docs/README-BuildKit.md new file mode 100644 index 00000000..ad348b5f --- /dev/null +++ b/docs/README-BuildKit.md @@ -0,0 +1,72 @@ +# Ocarina Recomp build kit + +This kit builds *The Legend of Zelda: Ocarina of Time* (Nintendo Switch 2, game version 1.1.1) as a native Windows +program **on your PC, from your own copy of the game**. The kit contains no game code or data. Everything the build +writes (`ocarina.exe`, the RomFS image, the shader cache, your saves) comes from your game: keep it to yourself. + +## What you need + +- 64-bit Windows 10 or 11, 16 GB of RAM, a Vulkan 1.3 graphics card with mesh shaders and a current driver (tested on + NVIDIA). +- Your own dump of the game as a `.nup` file (or its `exefs` and `romfs` folders), with the 1.1.1 update: program + `0400486024DEC000`, main build `e0f3ccfa`. Other versions are refused. +- About 30 GB free on the disk you build to, besides the dump. + +## One-time setup (about 15 minutes) + +1. **Python 3.12 or later** from [python.org](https://www.python.org/downloads/windows/). Tick "Add python.exe to PATH" + in the installer. Then, in a Command Prompt: + + ```bat + py -m pip install numpy lz4 zstandard zbic + ``` + + `zbic` unpacks the game's compressed program files. It is a separate GPL-2.0 package you install yourself; the kit + only calls it. + +2. **MSYS2** from [msys2.org](https://www.msys2.org/), installed to the default `C:\msys64`. Open **MSYS2 UCRT64** from + the Start menu and paste: + + ```sh + pacman -S --needed mingw-w64-ucrt-x86_64-clang mingw-w64-ucrt-x86_64-gcc mingw-w64-ucrt-x86_64-sdl3 mingw-w64-ucrt-x86_64-opus mingw-w64-ucrt-x86_64-libvpx mingw-w64-ucrt-x86_64-dav1d mingw-w64-ucrt-x86_64-zstd mingw-w64-ucrt-x86_64-vulkan-headers mingw-w64-ucrt-x86_64-vulkan-loader mingw-w64-ucrt-x86_64-vulkan-memory-allocator + ``` + + Answer `Y` when it asks. If MSYS2 is somewhere other than `C:\msys64`, add `--msys \ucrt64` to the build + command. + +## Build (about 10 to 25 minutes, once) + +Unpack this kit anywhere, open a Command Prompt in its folder and run: + +```bat +py -I tools\make_game.py --nup "C:\path\to\your game.nup" --out "C:\Games\Ocarina" +``` + +With extracted folders instead, use `--exefs "C:\path\exefs" --romfs "C:\path\romfs"` in place of `--nup`. To only +check that your dump is the right version, add `--check` (it builds nothing). + +The build checks the version, unpacks the program files, packs the 26 GB RomFS image (every file is checked against +the dump's own checksums), turns the game's code into C and compiles it (about 6 minutes on a 16-thread Ryzen 7; longer on fewer cores). +You can use your PC meanwhile. If it stops, +run the same command again: finished steps are skipped. + +When it says `finished`, start `C:\Games\Ocarina\ocarina.exe`. You can delete `C:\Games\Ocarina\build-cache` (about +5 GB) afterwards. + +## Playing + +- **F1** opens the settings menu. Settings are saved in `nx2navi.ini` beside the game; saves go to `profile\`. +- The first time a shader appears it is translated, which can stutter briefly; it is cached from then on. After an + update of the kit or your graphics driver, the next start shows "Preparing shaders" once. +- Logs are in `logs\`. When reporting a problem, say where you were in the game and which graphics card you have, and + attach `logs\nx2navi.log`. Never send game files. + +## If something goes wrong + +| Message | What to do | +|---|---| +| `not game version 1.1.1` | Install the 1.1.1 update and dump the game again | +| `the Python package ... is missing` | Run the `pip install` line above | +| `...\clang.exe is missing` | Run the `pacman` line above in MSYS2 UCRT64 | +| `needs ... GB free` | Free space on the `--out` disk, or build to another one | +| The game opens and closes | Update your graphics driver, then send `logs\nx2navi.log` | diff --git a/tools/make_game.py b/tools/make_game.py new file mode 100644 index 00000000..ce45bba9 --- /dev/null +++ b/tools/make_game.py @@ -0,0 +1,433 @@ +"""Build Ocarina of Time (Switch 2, game 1.1.1) for Windows on the player's PC, from the player's own game files. + + python -I tools/make_game.py --nup "C:\\path\\game.nup" --out "C:\\Games\\Ocarina" + python -I tools/make_game.py --exefs "C:\\path\\exefs" --romfs "C:\\path\\romfs" --out "C:\\Games\\Ocarina" + python -I tools/make_game.py --nup "C:\\path\\game.nup" --check # only checks the game's version + +Steps, each skipped when a run before finished it for the same input (state in /build-cache/state.json): + 1. check the title (0400486024DEC000) and main's build id (game 1.1.1) + 2. exefs the five ExeFS files, every module's segments unpacked (ZBIC through the player's own `zbic` package) + and stored plainly, as the runtime's loader reads them -> /exefs + 3. romfs every RomFS file, checked against the container's sha256, into one image -> /romfs/romfs.bin and + index.json (16-byte aligned, in the byte order of the paths) + 4. lift recomp/gen.py with recomp/patches-v111.txt -> /build-cache/lift, plus the two SDK bring-up fixes + 5. compile the lifted C and runtime/ with MSYS2's clang, linked into /ocarina.exe; the MSYS2 DLLs it loads, + the shader translator and romfs-index.bin beside it + +Nothing is written outside --out. The game folder then holds only what this PC built from the player's files: never +share it. Needs Python 3.10+ with numpy, lz4, zstandard and zbic (pip), and MSYS2 UCRT64's clang and libraries +(README-BuildKit.md lists the commands). Delete /build-cache afterwards to free about 5 GB. +""" +import argparse +import concurrent.futures +import hashlib +import json +import os +import shutil +import struct +import subprocess +import sys +import time +from pathlib import Path + +KIT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(KIT / 'tools')) +import nup # noqa: E402 + +TITLE_ID = 0x0400486024DEC000 +MAIN_BUILD_ID = 'e0f3ccfaf50065ae4ea01eb5aeb13b1a' # main of game 1.1.1 +EXEFS_FILES = ('main', 'main.npdm', 'sdk', 'subsdk0', 'rtld') +MODULES = ('main', 'sdk', 'subsdk0', 'rtld') +ALIGN = 16 +STATE_VERSION = 1 + +# Bring-up fixes in the lifted SDK, each anchored on exact generated text that must occur once. +LIFT_FIXES = [ + # nn::oe::SetupGpuErrorHandler (sdk+0x59a430) registers a system worker the host does not provide: GPU error + # reporting is not needed, so it returns at once. + ('sdk', 'void f_700579a430(Cpu *c) {\n FN_ENTER(0x700579a430ull);\n', + 'void f_700579a430(Cpu *c) {\n c->x[0] = 0; return; /* bring-up: GpuErrorHandler (make_game.py) */\n' + ' FN_ENTER(0x700579a430ull);\n'), + # The thread-start path loads TPIDR_EL0 from TLR+0x1F0 before the SDK stored its TLS there for a host-created + # thread: a zero slot keeps the TLS pointer the host seeded. + ('sdk', ' { uint64_t ea = c->x[19] + 0x1b8ull; ST64(ea, x8); }\n x8 = (uint64_t)(c->tpidrro);\n' + ' { uint64_t ea = x8 + 0x1f0ull; uint64_t v = LD64(ea); x8 = (uint64_t)(v); }\n c->tpidr = x8;\n', + ' { uint64_t ea = c->x[19] + 0x1b8ull; ST64(ea, x8); }\n x8 = (uint64_t)(c->tpidrro);\n' + ' { uint64_t ea = x8 + 0x1f0ull; uint64_t v = LD64(ea); x8 = (uint64_t)(v); }\n' + ' if (x8) c->tpidr = x8; /* zero slot: keep the host\'s TLS (make_game.py) */\n'), +] +LINK_LIBS = ['-lvulkan-1', '-lSDL3', '-lopus', '-lvpx', '-l:libdav1d.a', '-lzstd', '-l:libatomic.a', '-lbcrypt', + '-limm32', '-luuid', '-lole32', '-luser32', '-lshell32', '-lgdi32', '-ladvapi32', '-lwinmm'] +SYSTEM_DLLS_SKIP = {'vulkan-1.dll'} # the graphics driver's loader, never a copy + + +def say(msg): + print(msg, flush=True) + + +def fail(msg): + print(f'\nerror: {msg}', file=sys.stderr, flush=True) + sys.exit(1) + + +# ---------------------------------------------------------------- input + +class Source: + """The game files, from a .nup container or from extracted exefs/romfs folders.""" + + def __init__(self, a): + self.n = None + if a.nup: + try: + self.n = nup.Nup(a.nup) + except (OSError, nup.NupError) as e: + fail(f'cannot read {a.nup}: {e}') + pkgs = [p for p in self.n.packages if p['title_id'] == TITLE_ID] + if not pkgs: + ids = ', '.join(f"{p['title_id']:016X}" for p in self.n.packages) + fail(f'this .nup holds {ids}, not Ocarina of Time (0400486024DEC000)') + p = pkgs[-1] + self.exefs = dict(self.n.tree(p['exefs'])) + self.romfs = list(self.n.tree(p['romfs'])) + st = os.stat(a.nup) + self.identity = f'nup {os.path.abspath(a.nup)} {st.st_size} {int(st.st_mtime)}' + else: + self.exefs_dir, self.romfs_dir = Path(a.exefs), Path(a.romfs) + for d in (self.exefs_dir, self.romfs_dir): + if not d.is_dir(): + fail(f'{d} is not a folder') + self.exefs = {name: self.exefs_dir / name for name in EXEFS_FILES if (self.exefs_dir / name).is_file()} + self.romfs = [(p.relative_to(self.romfs_dir).as_posix(), p) for p in self.romfs_dir.rglob('*') if p.is_file()] + self.identity = f'folders {self.exefs_dir.resolve()} {self.romfs_dir.resolve()} {len(self.romfs)}' + missing = [x for x in EXEFS_FILES if x not in self.exefs] + if missing: + fail(f'the ExeFS lacks {", ".join(missing)}') + if not self.romfs: + fail('the RomFS is empty') + + def read_exefs(self, name): + return self.n.read(self.exefs[name]) if self.n else self.exefs[name].read_bytes() + + def romfs_size(self, ref): + return self.n.size(ref) if self.n else ref.stat().st_size + + def romfs_chunks(self, ref): + if self.n: + yield from self.n.chunks(ref) + return + with open(ref, 'rb') as f: + while b := f.read(nup.CHUNK): + yield b + + +def check(src): + main = src.read_exefs('main') + if main[:4] != b'NSO0': + fail('main is not an NSO module') + build = main[0x40:0x50].hex() + if build != MAIN_BUILD_ID: + fail(f'this is not game version 1.1.1: main has build {build[:8]}, the recomp supports build {MAIN_BUILD_ID[:8]}. ' + 'Install the 1.1.1 update and dump the game again.') + return main + + +# ---------------------------------------------------------------- exefs + +def flat_nso(raw, name): + """An NSO with its segments stored uncompressed (the runtime's loader reads only those); segment hashes kept.""" + flags = struct.unpack_from('> i & 1 else size)] + if flags >> i & 1: + if flags & 0x80: + try: + import zbic + except ImportError: + fail('the game\'s modules need the zbic package: pip install zbic') + data = zbic.decompress(data) + else: + import lz4.block + data = lz4.block.decompress(data, uncompressed_size=size) + if len(data) != size: + fail(f'{name}: {seg} unpacks to {len(data):#x} bytes, its header says {size:#x}') + if flags >> (3 + i) & 1 and hashlib.sha256(data).digest() != raw[0xA0 + 0x20 * i:0xC0 + 0x20 * i]: + fail(f'{name}: the {seg} segment is damaged (sha256 mismatch)') + struct.pack_into(' {len(data) / 1e6:6.1f} MB') + + +# ---------------------------------------------------------------- romfs + +def step_romfs(src, out, jobs): + dst = out / 'romfs' + dst.mkdir(parents=True, exist_ok=True) + files = sorted(src.romfs, key=lambda e: e[0].encode()) + total = sum(src.romfs_size(ref) for _, ref in files) + free = shutil.disk_usage(dst).free + if free < total + (1 << 30): + fail(f'{dst} needs {total / 1e9:.1f} GB free for the RomFS image, it has {free / 1e9:.1f} GB') + say(f' {len(files)} files, {total / 1e9:.1f} GB') + entries, h, pos, done, t0 = [], hashlib.sha256(), 0, 0, time.time() + part = dst / 'romfs.bin.part' + + def load(e): # small files are read whole on a worker; big ones are streamed by the writer + rel, ref = e + return None if src.romfs_size(ref) > (64 << 20) else b''.join(src.romfs_chunks(ref)) + + with open(part, 'wb', buffering=16 << 20) as img, concurrent.futures.ThreadPoolExecutor(jobs) as ex: + batch = 256 + for start in range(0, len(files), batch): + group = files[start:start + batch] + for (rel, ref), data in zip(group, ex.map(load, group)): + pad = -pos % ALIGN + if pad: + img.write(b'\0' * pad) + h.update(b'\0' * pad) + pos += pad + size = src.romfs_size(ref) + entries.append({'path': rel, 'offset': pos, 'size': size}) + for chunk in ((data,) if data is not None else src.romfs_chunks(ref)): + img.write(chunk) + h.update(chunk) + pos += size + done += size + if start // batch % 40 == 0: + say(f' {done / 1e9:5.1f} of {total / 1e9:.1f} GB, {time.time() - t0:.0f} s') + part.replace(dst / 'romfs.bin') + index = {'status': 'pass', 'romfs_size': pos, 'romfs_sha256': h.hexdigest(), 'files': entries} + (dst / 'index.json').write_text(json.dumps(index), encoding='utf-8') + say(f' image {pos / 1e9:.2f} GB in {time.time() - t0:.0f} s') + + +# ---------------------------------------------------------------- lift + +def step_lift(out, cache): + lift = cache / 'lift' + if lift.exists(): + shutil.rmtree(lift) + cmd = [sys.executable, str(KIT / 'recomp' / 'gen.py'), '--exefs', str(out / 'exefs'), '--out', str(lift), + '--patches', str(KIT / 'recomp' / 'patches-v111.txt')] + if subprocess.run(cmd).returncode: + fail('the code generator failed (see above)') + for mod, old, new in LIFT_FIXES: + hits = [p for p in sorted((lift / mod).glob('*.c')) if old in p.read_text(encoding='utf-8')] + if len(hits) != 1 or hits[0].read_text(encoding='utf-8').count(old) != 1: + fail(f'a bring-up fix found its anchor {len(hits)} times in the lifted {mod}, not once') + p = hits[0] + p.write_text(p.read_text(encoding='utf-8').replace(old, new), encoding='utf-8', newline='\n') + + +# ---------------------------------------------------------------- compile + +def sha256_file(path): + h = hashlib.sha256() + with open(path, 'rb') as f: + while b := f.read(8 << 20): + h.update(b) + return h.hexdigest() + + +def modules_c(lift, out, gen): + layout = json.loads((lift / 'layout.json').read_text(encoding='utf-8')) + L = ['#include "dispatch.h"', '#include "host.h"', '#include "layout.h"', ''] + for m in layout: + n = m['name'] + L += [f'extern const uint64_t {n}_fn_addrs[];', f'extern const GuestFn {n}_fn_ptrs[];', + f'extern const uint64_t {n}_fn_count;'] + L += ['', 'const RtLayoutEntry rt_layout[] = {'] + L += [f' {{"{m["name"]}", MOD_{m["name"].upper()}_BASE, MOD_{m["name"].upper()}_SIZE}},' for m in layout] + L += ['};', f'const int rt_layout_count = {len(layout)};', '', 'int rt_linked_modules(RtModule *out, int max) {', + ' int n = 0;'] + for m in layout: + n, u = m['name'], m['name'].upper() + L.append(f' if (n < max) out[n++] = (RtModule){{"{n}", MOD_{u}_BASE, MOD_{u}_SIZE, {n}_fn_addrs, {n}_fn_ptrs, ' + f'{n}_fn_count}};') + L += [' return n;', '}', '', 'const RtFileHash rt_exefs_sha256[] = {'] + L += [f' {{"{name}", "{sha256_file(out / "exefs" / name)}"}},' for name in EXEFS_FILES] + L += ['};', f'const int rt_exefs_sha256_count = {len(EXEFS_FILES)};', '', + f'const char rt_default_exefs[] = {json.dumps(str(out / "exefs"))};', + f'const char rt_default_romfs[] = {json.dumps(str(out / "romfs"))};', + f'const char rt_default_profile[] = {json.dumps(str(out / "profile"))};', + 'const int rt_game_version = 111;', + 'const unsigned long long rt_app_pool_bytes = 9809ull << 20; /* the Switch 2 application pool */', + 'const int rt_nvn_api_version[2] = {231, 40}; /* NVN2: main asks for major 231, minor 40 or later */', + f'const char rt_port_version[] = {json.dumps(kit_version())};'] + p = gen / 'modules.c' + p.write_text('\n'.join(L) + '\n', encoding='utf-8') + (gen / 'shim.c').write_text('/* Defined by the SDL window path only; this build uses the Win32 one. */\n' + 'void menu_sdl_event(void *e) { (void)e; }\n', encoding='utf-8') + return p + + +def kit_version(): + v = KIT / 'VERSION' + return v.read_text(encoding='utf-8').strip() if v.is_file() else '0.0.1-ocarina' + + +def pe_imports(path): + """The DLL names a PE file imports (its import directory), lower case.""" + d = Path(path).read_bytes() + pe = struct.unpack_from(' [--out build/kit] [--zip] + +The kit holds this repository's runtime/, recomp/, the build tools, the shader translator built from tools/xlate and +README-BuildKit.md; no game code, data or anything generated from the game. check_content runs over the folder before +it is zipped. +""" +import argparse +import shutil +import sys +import zipfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +NAME = 'OcarinaRecomp-BuildKit' +TOOLS = ('make_game.py', 'nup.py', 'romfs_index.py', 'check_content.py', 'hostmem.py') +TOP = ('LICENSE', 'LICENSE-ADDITIONS', 'THIRD_PARTY_NOTICES.md', 'VERSION', 'nx2navi.ini.example') +SKIP_DIRS = {'__pycache__', '.pytest_cache'} +SKIP_SUFFIXES = {'.o', '.obj', '.pyc', '.exe', '.dll', '.a', '.lib'} + + +def copy_tree(src, dst): + for p in sorted(src.rglob('*')): + rel = p.relative_to(src) + if any(part in SKIP_DIRS for part in rel.parts) or p.is_dir() or p.suffix.lower() in SKIP_SUFFIXES: + continue + (dst / rel).parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(p, dst / rel) + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument('--xlate', type=Path, required=True, help='folder with nx2navi_xlate.exe and nx2navi_xlate.id') + ap.add_argument('--out', type=Path, default=ROOT / 'build' / 'kit') + ap.add_argument('--zip', action='store_true') + a = ap.parse_args() + kit = a.out / NAME + if kit.exists(): + shutil.rmtree(kit) + kit.mkdir(parents=True) + for d in ('runtime', 'recomp'): + copy_tree(ROOT / d, kit / d) + (kit / 'tools').mkdir() + for t in TOOLS: + shutil.copy2(ROOT / 'tools' / t, kit / 'tools' / t) + for t in TOP: + if (ROOT / t).is_file(): + shutil.copy2(ROOT / t, kit / t) + shutil.copy2(ROOT / 'docs' / 'README-BuildKit.md', kit / 'README-BuildKit.md') + (kit / 'bin').mkdir() + for n in ('nx2navi_xlate.exe', 'nx2navi_xlate.id'): + shutil.copy2(a.xlate / n, kit / 'bin' / n) + sys.path.insert(0, str(ROOT / 'tools')) + import check_content + bad = [f'{p.relative_to(kit)}: {m}' for p in sorted(kit.rglob('*')) if p.is_file() + for m in check_content.problems_for(p, allow_large=p.suffix == '.exe')] + if bad: + sys.exit('check_content refuses these kit files, not zipped:\n ' + '\n '.join(bad[:20])) + size = sum(p.stat().st_size for p in kit.rglob('*') if p.is_file()) + print(f'{kit}: {size / 1e6:.1f} MB') + if a.zip: + z = a.out / f'{NAME}.zip' + with zipfile.ZipFile(z, 'w', zipfile.ZIP_DEFLATED, compresslevel=9) as zf: + for p in sorted(kit.rglob('*')): + if p.is_file(): + zf.write(p, Path(NAME) / p.relative_to(kit)) + print(f'{z}: {z.stat().st_size / 1e6:.1f} MB') + + +if __name__ == '__main__': + main() diff --git a/tools/nup.py b/tools/nup.py new file mode 100644 index 00000000..fe2c944c --- /dev/null +++ b/tools/nup.py @@ -0,0 +1,126 @@ +"""Reader for the .nup game container (little endian, nothing encrypted). + + 0x00 superblock, 0x80 bytes: "NUP\\x01", u16 0x80, u16 1, u64 toc offset, u64 toc size, sha256(toc), + u64 blob offset, u64 file size, u32 package count, u32 0, 16-byte unique id, 32 zero bytes + TOC: u32 "TOC\\x01", u32 packages, u64 dirs, u64 files, u64 name pool offset (from the TOC), u64 pool length, u64 0 + packages, 0x40 each: u64 title id, u32 root dir, u32 exefs dir, u32 romfs dir, u32 version, u8 kind + (0 base, 1 update, 2 dlc), u8 flags, u16, u32, u64 meta offset, u32 meta length, 20 bytes + dirs, 0x18 each: u32 name offset, u32 name length, u32 first child, u32 child count, u64 + files, 0x50 each: u32 name offset, u32 name length, u64 size, u64 data offset (absolute), u64 stored size, + sha256 of the file's bytes, u8 codec (0 stored, 1 one zstd frame), padding + children: u32 each, a directory below the dir count, else dir count + file index + names: the pool + +Each thread reads through its own handle: seeks on one shared handle race. Standard library plus zstandard. +""" +import hashlib +import struct +import threading + +SUPER = struct.Struct('<4sHHQQ32sQQII16s32s') +PACKAGE = struct.Struct('