Build the Linux kit against pinned Vulkan headers and without #embed, and try the compiler before the long steps.

A player's AppImage build of 0.7.1 stopped in its compile step after eight minutes of RomFS work: four
files did not compile, because the kit builds with the system's compiler and headers and theirs were
older than the runtime needs. vk_pipeline.c and vk_device.c name VK_KHR_pipeline_binary,
VK_KHR_dynamic_rendering_local_read and VK_KHR_shader_maximal_reconvergence, which their Vulkan headers
did not declare, and ui/menu.cpp takes its font in with #embed, which clang has from version 19.

- Vulkan headers: tools/vulkan_pin.py holds the release the other builds use (1.4.363, the archive and
  hash tools/make_release_linux.py already pinned, which now reads them from there). The AppImage carries
  its C headers in toolchain/vulkan-headers and builds against them whatever the system has; a kit or a
  source checkout uses the system's when they are at least that release and fetches the pinned ones
  otherwise (build_game.py vulkan_include). Their folder goes first on the include path of the runtime's
  sources only, so the lifted code's objects keep their keys, and a runtime object compiled before
  against the system's headers is taken over instead of compiled again. AppRun no longer asks for the
  system's Vulkan headers.
- #embed: tools/build_shaders.py also writes the font's bytes as runtime/ui/roboto_ttf.h (ignored by git,
  carried by a kit), and menu.cpp includes that where the compiler has no __has_embed. With #embed
  nothing changes.
- Before its long steps a Linux build that will compile now checks every runtime source that needs no
  header of the install with the compile step's own command (clang -fsyntax-only) and links an empty
  program against the game's libraries (make_game.py preflight): an old compiler, an old header or a
  missing development package stops it in the first seconds with the compiler's own errors. A failed
  compile names every file that failed, not only the three whose errors it prints.

Checked on Windows: tests/launcher_test.py passes with four new tests (117); the header of bytes and
#embed give the same 162,588 bytes, and both branches of menu.cpp compile; the early check passes on the
Windows file set with clang 23 and stops on files made to fail; the 34 local headers of release 363 are
the upstream tag's files by git blob id. Not run: anything on Linux, the pinned archive's download, a
compiler older than 23.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
chrg6
2026-11-10 12:42:17 -05:00
parent 005d30a29c
commit 61cd4784f8
13 changed files with 465 additions and 38 deletions

View File

@@ -884,6 +884,81 @@ def compile_all(srcs, cmd_for, cache, jobs, env, progress=None, salt='', stop=No
return len(todo), reused, bad
def report_failures(bad):
"""The files that did not compile, to stderr: the compiler's errors of the first three, and the names of all of
them, so that a paste of the build's end says which they were."""
for src, err in bad[:3]:
print(f'--- {src}\n{err}', file=sys.stderr)
print('did not compile: ' + ', '.join(Path(src).name for src, _ in bad), file=sys.stderr, flush=True)
def runtime_headers(rt):
"""The headers tools/build_shaders.py builds into runtime/ (the runtime's own shaders, the menu font's bytes): in
the repository as source only, so from a source checkout they are not there until they are built, and every
runtime file that includes one would fail to compile. A kit carries them built; a release's kit carries neither
tools/build_shaders.py nor the compilers it calls, and skips this."""
try:
import build_shaders
except ModuleNotFoundError:
return
stale = build_shaders.stale(rt)
if stale:
say(f' building {len(stale)} of the runtime\'s own headers (runtime/*_spv.h, the menu font)')
build_shaders.build(rt, stale)
def preflight(msys, jobs, sources=None):
"""Linux, before the long steps of a build whose compile step will run: this machine's compiler, headers and
libraries are tried on the runtime itself. Every runtime source that needs no header of this install is checked
with the compile step's own command (clang -fsyntax-only), and an empty program is linked against the libraries
the game links. The build takes the compiler, the linker and the libraries from the system (only the Vulkan
headers and VMA come with the kit), and a distribution's older compiler, older headers or a missing development
package stopped the compile step only after the RomFS image and the lift, ten minutes and more into the build.
sources: the files to check instead (tests)."""
clang, clangxx, env, vma = toolchain(msys)
version = clang_version(clang, env)
say(f'check: the compiler is {version}')
rt = KIT / 'runtime'
runtime_headers(rt)
inc = ['-I', str(rt), '-I', str(rt / 'third_party' / 'imgui'), '-I', str(vma)]
vulkan = vulkan_headers(clang, env)
cmd_for = runtime_cmd(clang, clangxx, inc, first=vulkan)
if sources is None:
tainted = per_install_names()
sources = [p for p, _ in runtime_sources() if not per_install(p, tainted)]
def check(src):
cmd = [c for c in cmd_for(src, Path('unused.o')) if c not in ('-c', '-o', 'unused.o')]
p = subprocess.run([cmd[0], '-fsyntax-only', *cmd[1:]], capture_output=True, text=True, env=env, errors='replace')
return src, p.returncode, p.stderr[-1500:]
t0 = time.time()
with concurrent.futures.ThreadPoolExecutor(max(1, jobs)) as ex:
bad = [(src, err) for src, rc, err in ex.map(check, sources) if rc]
if bad:
report_failures(bad)
lacking = sorted(set(re.findall(r"fatal error: '([^']+)' file not found", ''.join(e for _, e in bad))))
fail(f'{len(bad)} of the runtime\'s {len(sources)} files do not compile on this machine, so the build would stop '
f'in its compile step. ' +
(f'These headers are missing: {", ".join(lacking)}. Install the development packages '
f'README-BuildKit.md lists under "Linux".' if lacking else
f'The compiler ({version}) or a library\'s headers are older than the runtime needs: the releases are '
f'built with clang 23, and the Vulkan headers come ' +
(f'from {vulkan[1]}' if vulkan else 'from the system') + '. The errors above name what is missing.'))
with tempfile.TemporaryDirectory(prefix='nx2navi-check-') as tmp:
probe = Path(tmp) / 'probe.c'
probe.write_text('int main(void) { return 0; }\n', encoding='ascii')
ld = (['-fuse-ld=lld'] if shutil.which('ld.lld') else []) + ['-no-pie']
p = subprocess.run([str(clangxx), *ld, '-o', str(Path(tmp) / 'probe'), '-x', 'c', str(probe), '-x', 'none',
*LINK_LIBS], capture_output=True, text=True, env=env, errors='replace')
if p.returncode:
print(p.stderr[-2000:], file=sys.stderr)
fail('the libraries the game links are not all there (the linker\'s lines above name the missing ones): '
'install the development packages README-BuildKit.md lists under "Linux". '
'On Fedora the static C++ library is its own package, libstdc++-static.')
say(f'check: {len(sources)} runtime files and the libraries pass, {time.time() - t0:.0f} s')
def runtime_sources():
"""(source, object name) of every runtime file this OS's build compiles: the Win32 files on Windows, the SDL
window's (*_sdl.*) and platform/linux.c elsewhere (tools/build_game.py's foreign_os)."""
@@ -985,8 +1060,10 @@ def fast_clang(msys, clang, env):
return fast if mine and clang_ident(fast, env) == mine else None
def runtime_cmd(clang, clangxx, inc, level=0, fast=None, msys=None):
"""fast: fast_clang's compiler for the lifted code, with msys as its sysroot."""
def runtime_cmd(clang, clangxx, inc, level=0, fast=None, msys=None, first=()):
"""fast: fast_clang's compiler for the lifted code, with msys as its sysroot. first: include flags that go
ahead of inc for the runtime's sources only (vulkan_headers); the lifted code includes none of those headers,
and its command, which is part of its objects' keys, stays as it was."""
def cmd_for(src, dst):
cpp = src.suffix in ('.cpp', '.cxx')
# the lifted code at -O1 (its size makes -O2 slow to build), the runtime at -O2; neither may assume strict
@@ -999,7 +1076,7 @@ def runtime_cmd(clang, clangxx, inc, level=0, fast=None, msys=None):
# -fdeclspec: third_party/ffx_api/ffx_api.h declares its entry points __declspec(dllexport) unconditionally
declspec = ['-fdeclspec'] if cpp and not WIN else []
return [*cc, '-c', '-std=gnu++17' if cpp else '-std=gnu11', '-O1' if guest else '-O2',
*march, '-fno-strict-aliasing', '-w', *declspec, *inc, '-o', str(dst), str(src)]
*march, '-fno-strict-aliasing', '-w', *declspec, *([] if guest else first), *inc, '-o', str(dst), str(src)]
return cmd_for
@@ -1018,12 +1095,12 @@ def make_runtime_lib(dest, msys, jobs, obj=None):
inc = ['-I', str(rt), '-I', str(rt / 'third_party' / 'imgui'), '-I', str(vma)]
cache = ObjectCache(obj / 'objects.json', {'<kit>': KIT})
t0 = time.time()
compiled, reused, bad = compile_all(srcs, runtime_cmd(clang, clangxx, inc), cache, jobs, env,
compiled, reused, bad = compile_all(srcs, runtime_cmd(clang, clangxx, inc, first=vulkan_headers(clang, env)),
cache, jobs, env,
lambda d, t: (d % 20 == 0 or d == t) and say(f' {d} of {t}'),
salt=compiler_id(clang, env))
if bad:
for src, err in bad[:3]:
print(f'--- {src}\n{err}', file=sys.stderr)
report_failures(bad)
fail(f'{len(bad)} runtime files did not compile')
lib = dest / RUNTIME_LIB
lib.unlink(missing_ok=True)
@@ -1168,6 +1245,21 @@ def toolchain(msys):
return Path(shutil.which('clang')), Path(shutil.which('clang++')), dict(os.environ), vma
def vulkan_headers(clang, env):
"""The include flags that put the kit's Vulkan headers ahead of the system's for the runtime's sources: the
pinned release in <kit>/toolchain/vulkan-headers (tools/vulkan_pin.py), which the AppImage's kit carries and a
source checkout fetches when the system's headers are older (tools/build_game.py vulkan_include). None on
Windows, where MSYS2 supplies the headers, and none where the system's headers are new enough."""
if WIN:
return []
try:
from build_game import vulkan_include
folder = vulkan_include(clang, env)
except ImportError: # a kit without tools/vulkan_pin.py: the system's headers, as before it existed
return []
return ['-I', str(folder)] if folder else []
def xlate_source():
"""The folder of the shader translator to copy beside the game: the kit's bin/, else a source checkout's
build/xlate/ (tools/build_xlate.py)."""
@@ -1223,17 +1315,7 @@ def step_compile(out, cache_dir, msys, jobs, romfs_job=None, update=False):
gen.mkdir(parents=True, exist_ok=True)
obj.mkdir(parents=True, exist_ok=True)
rt = KIT / 'runtime'
# The runtime's own shaders are in the repository as source only: from a source checkout their headers are not
# there until they are built, and every runtime file that includes one would fail to compile. A kit carries them
# built and does not carry tools/build_shaders.py (nor the compilers it calls), so a kit build skips this step.
try:
import build_shaders
except ModuleNotFoundError:
build_shaders = None
stale = build_shaders.stale(rt) if build_shaders else []
if stale:
say(f' compiling {len(stale)} of the runtime\'s own shaders (runtime/*_spv.h)')
build_shaders.build(rt, stale)
runtime_headers(rt)
clang, clangxx, env, vma = toolchain(msys)
if not romfs_job:
romfs_index(out, gen)
@@ -1260,15 +1342,23 @@ def step_compile(out, cache_dir, msys, jobs, romfs_job=None, update=False):
fast = fast_clang(msys, clang, env)
if fast:
say(f' the lifted code compiles with {fast}')
cmd_for = runtime_cmd(clang, clangxx, inc, level, fast, msys)
vulkan = vulkan_headers(clang, env)
if vulkan:
say(f' the Vulkan headers come from {vulkan[1]}')
cmd_for = runtime_cmd(clang, clangxx, inc, level, fast, msys, first=vulkan)
# The keys earlier versions gave the same compiles (compile_all alts): 0.6.0's, with the compiler's file time and,
# with the fast compiler, its id and --sysroot; 0.5.0's and earlier's, with their kit's folder in the paths.
# with the fast compiler, its id and --sysroot; 0.5.0's and earlier's, with their kit's folder in the paths; and
# up to 0.7.1 the runtime's without the Vulkan headers' folder ahead of the system's (an object that compiled
# against the system's headers is as good).
plain = runtime_cmd(clang, clangxx, inc, level, None, msys)
no_vulkan = runtime_cmd(clang, clangxx, inc, level, fast, msys)
fast_old = (cid_old + compiler_ids(fast, env)[1]) if fast else None
kits = earlier_kits()
def alts(src, dst):
keys = [object_key(plain(src, dst), src, dst, cache, cid_old, sysroot=True)]
if vulkan:
keys.append(object_key(no_vulkan(src, dst), src, dst, cache, cid))
if fast:
keys.append(object_key(cmd_for(src, dst), src, dst, cache, fast_old, sysroot=True))
keys += [legacy_key(src, dst, cache, cid_old, clang, clangxx, k, lift, gen, vma) for k in kits]
@@ -1306,8 +1396,7 @@ def step_compile(out, cache_dir, msys, jobs, romfs_job=None, update=False):
compiled, reused = compiled + c2, reused + r2
say(f' {compiled} compiled, {reused} unchanged since the last build')
if bad:
for src, err in bad[:3]:
print(f'--- {src}\n{err}', file=sys.stderr)
report_failures(bad)
fail(f'{len(bad)} files did not compile')
if cache.keep_only({o.name for _, o in srcs}):
cache.save()
@@ -1846,6 +1935,10 @@ def main():
# the compile step's key does not name the level: without this a finished compile reads as done before
state['steps'].pop('compile', None)
keep_awake()
# Linux: the compiler, headers and libraries are the system's, so they are tried on the runtime before the long
# steps of a build that will reach the compile step (the first step whose key is not current makes the rest run)
if not WIN and any(state['steps'].get(n) != step_key(n, src.identity, out) for n in STEPS[:STEPS.index('compile') + 1]):
preflight(msys, a.jobs)
t0 = time.time()
romfs_job = None
runs = {'exefs': lambda: step_exefs(src, out), 'romfs': lambda: step_romfs(src, out, min(a.jobs, 8)),