Build Ocarina natively on Linux with tools/make_game.py, and pack the kit as an AppImage.

make_game.py only knew MSYS2. On Linux it now does what tools/build_game.py's Linux path does:
- clang, clang++ and lld from PATH (checked at start), the system's headers and libraries, vk_mem_alloc.h from the
  system or the pinned copy build_game.py fetches into toolchain/vma;
- the runtime's *_sdl.* files and platform/linux.c in place of the Win32 ones; no shim.c (ui/menu_sdl.cpp defines
  menu_sdl_event), no windres, no DLLs to copy;
- -fdeclspec for C++, as third_party/ffx_api/ffx_api.h declares __declspec(dllexport) unconditionally;
- build_game.py's Linux link flags (-no-pie, noexecstack, rpath $ORIGIN, libstdc++ static) and libraries, plus
  dav1d (the AV1 movies), libatomic and libunwind (the previous commit's plat_frame_capture);
- ocarina and nx2navi_xlate without .exe; the translator from the kit's bin/ or a checkout's build/xlate.
Windows is unchanged: every difference is under os.name.

tools/make_appimage.py packs the kit (tools, recomp, runtime, the built translator, VMA) with a relocatable CPython 3.14
and numpy, lz4, zstandard and fonttools into NX2navi-Linux-<version>-x86_64.AppImage. Its AppRun checks for the
compiler and headers and names the packages for Arch, Fedora and Debian/Ubuntu, installs zbic (GPL-2.0, not bundled)
with pip into ~/.local/share/nx2navi/pydeps on the first run, picks --jobs from the memory, builds and writes play.sh
beside the game. README-BuildKit.md has a Linux section.

Tested on CachyOS (kernel 7.3, clang 23, RTX 3080, NVIDIA 620.70.06, 16 GB): make_game.py --nup ... --jobs 4 builds
ocarina in 9.9 minutes from a 1.1.1 dump, which passes --probe-cfg; a build with the same sources and flags
(compiled by hand before this script could) starts, loads a save and plays. With one compile per thread on 16 GB
the build filled memory and swap and the kernel stopped desktop processes, hence the --jobs advice.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
loteran
2026-11-08 14:48:31 +01:00
committed by chrg6
parent 844940c6fb
commit f3daf7e2b4
3 changed files with 307 additions and 29 deletions

View File

@@ -41,6 +41,11 @@ version (fast_clang: the same objects, about a fifth faster).
python tools/make_game.py --make-runtime-lib <kit>/lib --msys <ucrt64> # the release's prebuilt runtime
On Linux the same steps make <out>/ocarina with the clang, clang++ and lld on PATH and the system's libraries
(Vulkan, SDL3, Opus, libvpx, dav1d, zstd, libunwind; libstdc++ static, rpath $ORIGIN), as tools/build_game.py's Linux
build does: the runtime's *_sdl.* and platform/linux.c take the place of its *_win.*, platform/win32.c and
nx2navi.rc, and --msys is not used. README-BuildKit.md, "Linux", lists the packages.
Nothing is written outside --out (but for --make-runtime-lib DIR, which writes DIR). The game folder then holds
only what this PC built from the player's files: never
share it. Needs Python 3.10+ with numpy, lz4, zstandard and zbic (pip), with fonttools and pillow too when the kit has
@@ -101,10 +106,17 @@ LIFT_FIXES = [
' { uint64_t ea = x8 + 0x1f0ull; uint64_t v = LD64(ea); x8 = (uint64_t)(v); }\n'
' if (x8) c->tpidr = x8; /* zero slot: keep the host\'s TLS (make_game.py) */\n'),
]
LINK_LIBS = ['-lvulkan-1', '-lSDL3', '-lopus', '-lvpx', '-l:libdav1d.a', '-lzstd', '-l:libatomic.a', '-lbcrypt',
'-limm32', '-luuid', '-lole32', '-luser32', '-lshell32', '-lgdi32', '-ladvapi32', '-lwinmm']
WIN = os.name == 'nt'
if WIN:
LINK_LIBS = ['-lvulkan-1', '-lSDL3', '-lopus', '-lvpx', '-l:libdav1d.a', '-lzstd', '-l:libatomic.a', '-lbcrypt',
'-limm32', '-luuid', '-lole32', '-luser32', '-lshell32', '-lgdi32', '-ladvapi32', '-lwinmm']
else: # tools/build_game.py's Linux libraries, plus dav1d (the AV1 movies), libatomic and libunwind (platform/linux.c's
# plat_frame_capture walks the host stack with it, issue #72)
LINK_LIBS = ['-lvulkan', '-lSDL3', '-lopus', '-lvpx', '-ldav1d', '-lzstd', '-l:libstdc++.a', '-latomic', '-lunwind',
'-lm', '-lpthread', '-ldl']
SYSTEM_DLLS_SKIP = {'vulkan-1.dll'} # the graphics driver's loader, never a copy
EXE_NAME = 'ocarina.exe'
EXE_NAME = 'ocarina.exe' if WIN else 'ocarina'
XLATE_NAME = 'nx2navi_xlate.exe' if WIN else 'nx2navi_xlate'
# Headers this build writes per install (gen.py, romfs_index.py): a runtime source that includes one, directly or
# through a runtime header, is compiled on the player's PC; every other runtime source can come prebuilt.
GENERATED_HEADERS = ('layout.h', 'patches.h', 'romfs_config.h')
@@ -456,8 +468,9 @@ def modules_c(lift, out, gen):
f'const char rt_port_version[] = {json.dumps(kit_version())};']
p = gen / 'modules.c'
write_if_changed(p, '\n'.join(L) + '\n')
write_if_changed(gen / 'shim.c', '/* Defined by the SDL window path only; this build uses the Win32 one. */\n'
'void menu_sdl_event(void *e) { (void)e; }\n')
if WIN: # on Linux ui/menu_sdl.cpp defines it
write_if_changed(gen / 'shim.c', '/* Defined by the SDL window path only; this build uses the Win32 one. */\n'
'void menu_sdl_event(void *e) { (void)e; }\n')
return p
@@ -724,12 +737,13 @@ def compile_all(srcs, cmd_for, cache, jobs, env, progress=None, salt='', stop=No
def runtime_sources():
"""(source, object name) of every runtime file the Windows build compiles."""
"""(source, object name) of every runtime file this OS's build compiles: the Win32 files on Windows, the SDL
window's (*_sdl.*) and platform/linux.c elsewhere (tools/build_game.py's foreign_os)."""
rt = KIT / 'runtime'
out = []
for pat in RUNTIME_PATTERNS:
for p in sorted(rt.glob(pat)):
if p.stem.endswith('_sdl') or p.name == 'linux.c':
if (p.stem.endswith('_sdl') or p.name == 'linux.c') if WIN else (p.stem.endswith('_win') or p.name == 'win32.c'):
continue
out.append((p, 'rt_' + p.relative_to(rt).as_posix().replace('/', '_').rsplit('.', 1)[0] + '.o'))
return out
@@ -800,6 +814,8 @@ def fast_clang(msys, clang, env):
built with profile-guided optimisation, about a fifth faster on the lifted code. With UCRT64's headers
(--sysroot) it writes the same objects as UCRT64's clang, byte for byte but the COFF timestamp; it is used only
when it reports the same version and target. None otherwise."""
if not WIN:
return None
fast = Path(msys).parent / 'clang64' / 'bin' / 'clang.exe'
if not fast.is_file():
return None
@@ -818,8 +834,10 @@ def runtime_cmd(clang, clangxx, inc, level=0, fast=None, msys=None):
guest = dst.name.startswith(('main_', 'sdk_', 'subsdk0_'))
march = [f'-march=x86-64-v{level}'] if guest and level else []
cc = [str(fast), f'--sysroot={msys}'] if guest and fast else [str(clangxx if cpp else clang)]
# -fdeclspec: third_party/ffx_api/ffx_api.h declares its entry points __declspec(dllexport) unconditionally
declspec = ['-fdeclspec'] if cpp and not WIN else []
return [*cc, '-c', '-std=gnu++17' if cpp else '-std=gnu11', '-O1' if guest else '-O2',
*march, '-fno-strict-aliasing', '-w', *inc, '-o', str(dst), str(src)]
*march, '-fno-strict-aliasing', '-w', *declspec, *inc, '-o', str(dst), str(src)]
return cmd_for
@@ -923,6 +941,26 @@ def compile_resources(msys, rt, obj, env):
return rco
def toolchain(msys):
"""(clang, clang++, environment, the folder holding vk_mem_alloc.h): MSYS2 UCRT64's on Windows, the ones on PATH
elsewhere, with tools/build_game.py's VMA lookup (the system's header, else the pinned one fetched into
toolchain/vma)."""
if WIN:
return (msys / 'bin' / 'clang.exe', msys / 'bin' / 'clang++.exe',
dict(os.environ, PATH=str(msys / 'bin') + os.pathsep + os.environ.get('PATH', '')), msys / 'include' / 'vma')
from build_game import vma_include
return Path(shutil.which('clang')), Path(shutil.which('clang++')), dict(os.environ), vma_include()
def xlate_source():
"""The folder of the shader translator to copy beside the game: the kit's bin/, else a source checkout's
build/xlate/ (tools/build_xlate.py)."""
for d in (KIT / 'bin', KIT / 'build' / 'xlate'):
if (d / XLATE_NAME).is_file():
return d
fail(f'{XLATE_NAME} is missing: build it with tools/build_xlate.py')
def step_compile(out, cache_dir, msys, jobs, romfs_job=None):
"""romfs_job: the RomFS step still running beside this one (a first build). Everything that does not include
romfs_config.h compiles meanwhile; the RomFS index and the few files that include it follow once it is done."""
@@ -930,8 +968,7 @@ def step_compile(out, cache_dir, msys, jobs, romfs_job=None):
gen.mkdir(parents=True, exist_ok=True)
obj.mkdir(parents=True, exist_ok=True)
rt = KIT / 'runtime'
clang, clangxx = msys / 'bin' / 'clang.exe', msys / 'bin' / 'clang++.exe'
env = dict(os.environ, PATH=str(msys / 'bin') + os.pathsep + os.environ.get('PATH', ''))
clang, clangxx, env, vma = toolchain(msys)
if not romfs_job:
romfs_index(out, gen)
srcs = [(p, obj / f'{mod}_{p.stem}.o') for mod in ('main', 'sdk', 'subsdk0') for p in sorted((lift / mod).glob('*.c'))]
@@ -941,9 +978,10 @@ def step_compile(out, cache_dir, msys, jobs, romfs_job=None):
runtime = [(p, o) for p, o in runtime if o.name not in prebuilt[1]]
say(f' the runtime comes prebuilt; {len(runtime)} runtime files are compiled here')
srcs += runtime
srcs += [(modules_c(lift, out, gen), obj / 'modules.o'), (gen / 'shim.c', obj / 'shim.o')]
inc = ['-I', str(rt), '-I', str(lift), '-I', str(gen), '-I', str(rt / 'third_party' / 'imgui'),
'-I', str(msys / 'include' / 'vma')]
srcs.append((modules_c(lift, out, gen), obj / 'modules.o'))
if WIN:
srcs.append((gen / 'shim.c', obj / 'shim.o'))
inc = ['-I', str(rt), '-I', str(lift), '-I', str(gen), '-I', str(rt / 'third_party' / 'imgui'), '-I', str(vma)]
cid = compiler_id(clang, env)
level = cpu_level(clang, env)
say(f' the lifted code is built for x86-64-v{level}, this processor\'s level' if level else
@@ -970,7 +1008,7 @@ def step_compile(out, cache_dir, msys, jobs, romfs_job=None):
late = []
if romfs_job: # the lifted chunks include only cpu.h and simd.h: only the runtime and generated files are read
tainted = per_install_names(('romfs_config.h',))
late = [(s, o) for s, o in runtime + srcs[-2:] if per_install(s, tainted)]
late = [(s, o) for s, o in runtime + srcs[-(2 if WIN else 1):] if per_install(s, tainted)]
say(f' {len(late)} files wait for the RomFS index: {", ".join(s.name for s, _ in late)}')
early = [j for j in srcs if j not in late]
compiled, reused, bad = compile_all(early, cmd_for, cache, jobs, env, lambda d, t: progress(d, len(srcs)),
@@ -990,14 +1028,16 @@ def step_compile(out, cache_dir, msys, jobs, romfs_job=None):
print(f'--- {src}\n{err}', file=sys.stderr)
fail(f'{len(bad)} files did not compile')
rco = compile_resources(msys, rt, obj, env)
objs = [o for _, o in srcs] + [rco]
objs = [o for _, o in srcs] + ([compile_resources(msys, rt, obj, env)] if WIN else [])
exe = out / EXE_NAME
rsp = cache_dir / 'link.rsp'
rsp.write_text('\n'.join('"%s"' % o.as_posix() for o in objs), encoding='utf-8')
# lld links several times faster than GNU ld; the stack matches tools/build_game.py's build
# 16 MiB for the main thread with either linker (GNU ld takes the same flag): the default 2 MiB is too little
ld = (['-fuse-ld=lld'] if (msys / 'bin' / 'ld.lld.exe').is_file() else []) + ['-Wl,--stack,16777216']
if WIN:
ld = (['-fuse-ld=lld'] if (msys / 'bin' / 'ld.lld.exe').is_file() else []) + ['-Wl,--stack,16777216']
else: # tools/build_game.py's Linux flags; the main thread's stack is the process's (ulimit -s), not the linker's
ld = (['-fuse-ld=lld'] if shutil.which('ld.lld') else []) + ['-no-pie', '-Wl,-z,noexecstack', '-Wl,-rpath,$ORIGIN']
# the link map feeds tools/prof_report.py (symbols for an NX2NAVI_PROFILE sampling profile); both linkers write one
ld.append('-Wl,--Map=' + (cache_dir / 'ocarina.map').as_posix())
lib = ['-Wl,--whole-archive', str(prebuilt[0]), '-Wl,--no-whole-archive'] if prebuilt else []
@@ -1007,13 +1047,14 @@ def step_compile(out, cache_dir, msys, jobs, romfs_job=None):
link_key = sha256_bytes(json.dumps([LINK_LIBS, ld, cid] + stamp).encode())
state_link = cache_dir / 'link.key'
if exe.is_file() and state_link.is_file() and state_link.read_text(encoding='utf-8') == link_key:
say(' ocarina.exe is current, not linked again')
say(f' {EXE_NAME} is current, not linked again')
else:
emit('phase', step='compile', phase='link')
say(' linking')
tmp = out / (EXE_NAME + '.new')
p = subprocess.run([str(clangxx), *ld, '-o', str(tmp), '@' + str(rsp), *lib, '-L', str(msys / 'lib'),
*LINK_LIBS], capture_output=True, text=True, env=env, errors='replace')
lib_dir = ['-L', str(msys / 'lib')] if WIN else []
p = subprocess.run([str(clangxx), *ld, '-o', str(tmp), '@' + str(rsp), *lib, *lib_dir, *LINK_LIBS],
capture_output=True, text=True, env=env, errors='replace')
if p.returncode:
print(p.stderr[-4000:], file=sys.stderr)
tmp.unlink(missing_ok=True)
@@ -1025,14 +1066,16 @@ def step_compile(out, cache_dir, msys, jobs, romfs_job=None):
fail(f'{exe} is in use: close the game and try again')
state_link.write_text(link_key, encoding='utf-8')
copy_if_changed(gen / 'romfs-index.bin', out / 'romfs-index.bin')
for name in ('nx2navi_xlate.exe', 'nx2navi_xlate.id'):
copy_if_changed(KIT / 'bin' / name, out / name)
xlate = xlate_source()
for name in (XLATE_NAME, 'nx2navi_xlate.id'):
copy_if_changed(xlate / name, out / name)
if (KIT / 'bin' / 'amd_fidelityfx_vk.dll').is_file(): # FSR 3.1 (runtime/vk_ffx.cpp)
copy_if_changed(KIT / 'bin' / 'amd_fidelityfx_vk.dll', out / 'amd_fidelityfx_vk.dll')
kept = place_dlss(out) # NVIDIA DLSS (runtime/vk_dlaa.c)
kept = place_dlss(out) if WIN else None # NVIDIA DLSS (runtime/vk_dlaa.c): its library is a Windows DLL
if kept:
say(f' {kept}')
say(f' {EXE_NAME} {exe.stat().st_size / 1e6:.0f} MB; beside it: {", ".join(copy_dlls(exe, msys / "bin", out))}')
beside = copy_dlls(exe, msys / 'bin', out) if WIN else []
say(f' {EXE_NAME} {exe.stat().st_size / 1e6:.0f} MB' + (f'; beside it: {", ".join(beside)}' if beside else ''))
(out / 'profile').mkdir(exist_ok=True)
@@ -1067,7 +1110,7 @@ def step_shaders(out):
Its output is read as it comes: the counts nx2navi.exe prints become progress events, the other lines go on to
the log, and while nothing moves an event every few seconds keeps saying the step is running and for how long."""
cmd = [sys.executable, '-u', str(KIT / 'tools' / 'prepare_shaders.py'), str(out), '--romfs', str(out / 'romfs'),
'--exefs', str(out / 'exefs'), '--no-list', '--exe', 'ocarina.exe']
'--exefs', str(out / 'exefs'), '--no-list', '--exe', EXE_NAME]
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding='utf-8',
errors='replace', bufsize=1)
t0 = time.monotonic()
@@ -1238,7 +1281,8 @@ def main():
help='build again from the game files the last run unpacked into --out (after a kit update)')
ap.add_argument('--out', help='the folder for the finished game')
ap.add_argument('--check', action='store_true', help='only check the game\'s version; builds nothing')
ap.add_argument('--msys', default=r'C:\msys64\ucrt64', help='MSYS2 UCRT64 folder (default C:\\msys64\\ucrt64)')
ap.add_argument('--msys', default=r'C:\msys64\ucrt64',
help='MSYS2 UCRT64 folder (default C:\\msys64\\ucrt64; Windows only: Linux takes clang and lld from PATH)')
ap.add_argument('--jobs', type=int, default=os.cpu_count() or 4, help='parallel compiles')
ap.add_argument('--events', action='store_true', help='machine-readable progress lines on stdout ("@@" + JSON)')
ap.add_argument('--make-runtime-lib', metavar='DIR', help='build the prebuilt runtime library of a release into DIR')
@@ -1270,9 +1314,14 @@ def main():
if not out:
ap.error('--out is needed to build')
msys = Path(a.msys)
for need in ('bin/clang.exe', 'bin/clang++.exe', 'bin/SDL3.dll', 'lib/libSDL3.dll.a', 'include/vma'):
if not (msys / need).exists():
fail(f'{msys / need} is missing: install MSYS2 and the packages README-BuildKit.md lists')
if WIN:
for need in ('bin/clang.exe', 'bin/clang++.exe', 'bin/SDL3.dll', 'lib/libSDL3.dll.a', 'include/vma'):
if not (msys / need).exists():
fail(f'{msys / need} is missing: install MSYS2 and the packages README-BuildKit.md lists')
else:
for tool in ('clang', 'clang++', 'ld.lld'):
if not shutil.which(tool):
fail(f'{tool} is not on PATH: install the packages README-BuildKit.md lists under "Linux"')
packages = {'numpy': 'numpy', 'lz4': 'lz4', **({} if a.rebuild else {'zbic': 'zbic'}),
**(PROMPT_PACKAGES if prompt_art_dir() else {})}
for pip_name, mod in packages.items():