Merge branch 'claude/linux-old-headers' into test-all-fixes

This commit is contained in:
chrg6
2026-11-10 14:18:29 -05:00
13 changed files with 465 additions and 38 deletions

View File

@@ -61,6 +61,7 @@ import ctypes.wintypes as wt # the job-object structures below; importing it w
import hashlib
import json
import os
import re
import shutil
import subprocess
import sys
@@ -180,6 +181,40 @@ def vma_include():
return VMA_LOCAL
def system_vulkan_version(clang, env=None):
"""VK_HEADER_VERSION of the <vulkan/vulkan_core.h> this compiler finds by itself (its own search path, CPATH
included), 0 when it finds none."""
probe = '#include <vulkan/vulkan_core.h>\nnx2navi_vulkan_headers VK_HEADER_VERSION\n'
try:
p = subprocess.run([str(clang), '-E', '-P', '-x', 'c', '-'], input=probe, capture_output=True, text=True,
env=env, errors='replace')
except OSError:
return 0
m = re.search(r'nx2navi_vulkan_headers\s+(\d+)', p.stdout) if p.returncode == 0 else None
return int(m.group(1)) if m else 0
def vulkan_include(clang=None, env=None):
"""A build with the system's own headers (Linux without --deps): the folder to put ahead of them for
<vulkan/vulkan.h>, or None when the system's serve. The runtime names Vulkan extensions newer than the headers
many distributions package, and every file that does fails to compile there (tools/vulkan_pin.py). So:
toolchain/vulkan-headers when it holds the pinned release (a kit carries it; a checkout once it was fetched), else
nothing when the system's headers are at least that release, else the pinned release, fetched there once.
Windows takes MSYS2's headers and returns None."""
if WIN:
return None
import vulkan_pin
local = ROOT / vulkan_pin.FOLDER
if vulkan_pin.have(local):
return local / 'include'
found = system_vulkan_version(clang or CLANG, env)
if found >= vulkan_pin.HEADER_VERSION:
return None
print(f'build_game: the system\'s Vulkan headers are {f"release {found}" if found else "missing"}; the runtime is '
f'built against release {vulkan_pin.HEADER_VERSION} ({vulkan_pin.VERSION}) or newer', flush=True)
return vulkan_pin.resolve(local) / 'include'
# --toolchain: an llvm-mingw tree. The libraries, headers and DLLs MSYS2 supplies come from the deps directory instead.
LLVM_LINK_LIBS = ['-lpsapi', '-lbcrypt', '-luser32', '-lgdi32', '-ldwmapi', '-limm32', '-lwinmm', '-lole32', '-lshell32', '-luuid', '-lvulkan-1', '-lSDL3', '-lopus',
'-l:libdav1d.a', '-l:libvpx.a', '-l:libzstd.a', '-l:libc++.a', '-l:libc++abi.a', '-l:libunwind.a', '-l:libwinpthread.a']
@@ -380,6 +415,9 @@ def ninja_file(gen, out, mods, pool_depth, skip=(), prebuilt=False, runtime_lib=
vma = f'{cmd_path(DEPS / "include")} -I {cmd_path(DEPS / "include" / "vma")}' if DEPS else cmd_path(vma_include())
imgui = cmd_path(RUNTIME / 'third_party' / 'imgui')
rt_inc = f'{inc} -I {cmd_path(out)} -I {vma} -I {imgui}' # romfs_config.h, vk_mem_alloc.h (+ deps headers), imgui.h
vulkan = None if DEPS else vulkan_include() # deps carries the Vulkan headers itself
if vulkan: # first, so that no other folder on the path answers for <vulkan/vulkan.h> (VMA's may be /usr/include)
rt_inc = f'-I {cmd_path(vulkan)} {rt_inc}'
libs = [f'-L {cmd_path(DEPS / "lib")}'] + (LLVM_LINK_LIBS if WIN else LINK_LIBS) if DEPS else LINK_LIBS
# A compiler cache (sccache) may front clang: every lift rewrites the generated chunks with the same contents, so
# the cached objects save the compile step on the next build.

View File

@@ -1,4 +1,4 @@
"""Build the runtime's own shaders into C headers of SPIR-V words.
"""Build the runtime's own shaders into C headers of SPIR-V words, and the files it embeds into headers of bytes.
The repository holds the shaders as source only. Each becomes runtime/<name>_spv.h, which declares
`static const uint32_t <name>_spv[]`; the headers are build outputs (ignored by git) and tools/build_game.py builds the
@@ -8,6 +8,9 @@ missing and the stale ones before it compiles the runtime. Three kinds of source
runtime/shaders/<name>.spvasm SPIR-V assembly, assembled with spirv-as and checked with spirv-val
tools/make_dlaa_mask.py, make_mvobj_fs.py shaders assembled in Python (ASSEMBLED); they need no other tool
One more kind of header is no shader: a file the runtime embeds (EMBEDDED: the menu's font) as a list of its bytes,
for a compiler without #embed (runtime/ui/menu.cpp includes it there). It needs no tool.
A compiled or assembled header records the SHA-256 of its source and is stale when that no longer matches; a Python
one is stale when its text is not what its script gives. glslc, spirv-as and spirv-val come from MSYS2 UCRT64 on
Windows (packages mingw-w64-ucrt-x86_64-shaderc and mingw-w64-ucrt-x86_64-spirv-tools) and from PATH elsewhere
@@ -36,10 +39,28 @@ ASM = '.spvasm'
ASM_TARGET = 'vulkan1.3'
# header name -> the script beside this one that assembles it; the script's HEADER is the header's text
ASSEMBLED = {'dlaa_mask': 'make_dlaa_mask.py', 'mvobj_fs': 'make_mvobj_fs.py'}
# a file below runtime/ the runtime embeds -> the header of its bytes, below runtime/ too (git ignores it)
EMBEDDED = {'ui/fonts/Roboto-Medium.ttf': 'ui/roboto_ttf.h'}
def embedded(src):
"""True for a file of EMBEDDED: a binary file, read as it is."""
return src.suffix not in STAGES and src.suffix not in (ASM, '.py')
def source_hash(src):
return hashlib.sha256(src.read_bytes().replace(b'\r\n', b'\n')).hexdigest() # the same on a CRLF checkout
data = src.read_bytes()
return hashlib.sha256(data if embedded(src) else data.replace(b'\r\n', b'\n')).hexdigest() # text: the same on a CRLF checkout
def bytes_header(src, runtime):
"""The header of an embedded file: its bytes as decimal numbers, to include inside an array's braces."""
data = src.read_bytes()
lines = [f'// Generated by tools/build_shaders.py from runtime/{src.relative_to(runtime).as_posix()}; do not edit.',
f'// source sha256 {source_hash(src)}',
f'// The file\'s {len(data)} bytes, for a compiler without #embed.']
lines += [','.join(map(str, data[i:i + 32])) + ',' for i in range(0, len(data), 32)]
return '\n'.join(lines) + '\n'
def header_of(src, words):
@@ -64,6 +85,7 @@ def sources(runtime):
found = sorted(p for p in shaders.iterdir() if p.suffix in STAGES or p.suffix == ASM) if shaders.is_dir() else []
out = [(p, runtime / f'{p.stem}_spv.h') for p in found]
out += [(TOOLS / script, runtime / f'{name}_spv.h') for name, script in ASSEMBLED.items() if (TOOLS / script).is_file()]
out += [(runtime / src, runtime / header) for src, header in EMBEDDED.items() if (runtime / src).is_file()]
return out
@@ -130,12 +152,14 @@ def build(runtime, only=None):
todo = stale(runtime) if only is None else list(only)
headers = dict(sources(runtime))
for src in todo:
text = assembled_text(src) if src.suffix == '.py' else header_of(src, words_of(src))
text = (assembled_text(src) if src.suffix == '.py' else bytes_header(src, runtime) if embedded(src)
else header_of(src, words_of(src)))
header = headers[src]
part = header.with_name(header.name + '.part') # whole or not at all: another build may be reading the folder
part.write_text(text, encoding='utf-8', newline='\n')
part.replace(header)
print(f'{header.name}: {text.count("0x")} words from {src.name}')
print(f'{header.name}: {src.stat().st_size} bytes of {src.name}' if embedded(src)
else f'{header.name}: {text.count("0x")} words from {src.name}')
return len(todo)

View File

@@ -8,9 +8,10 @@ with --events and reads its progress lines).
The AppImage holds no game files: the kit's tools/, recomp/, runtime/ and docs, the shader translator built by
tools/build_xlate.py (bin/nx2navi_xlate), NVIDIA's DLSS library with its notice (bin/, tools/dlss_pin.py: downloaded
from NVIDIA's DLSS SDK repository and checked, or --dlss; not with --no-dlss), the pinned Vulkan Memory Allocator
header (an AppImage made without the library downloads it after a build on machines with NVIDIA's driver: AppRun's
get_dlss), and a relocatable CPython 3.14 (python-build-standalone, as `uv python install 3.14` puts in
from NVIDIA's DLSS SDK repository and checked, or --dlss; not with --no-dlss: an AppImage made without the library
downloads it after a build on machines with NVIDIA's driver, AppRun's get_dlss), the pinned Vulkan Memory Allocator
header, the pinned Vulkan headers (toolchain/vulkan-headers, tools/vulkan_pin.py: the runtime names Vulkan extensions
newer than the headers many distributions package), and a relocatable CPython 3.14 (python-build-standalone, as `uv python install 3.14` puts in
~/.local/share/uv/python) with numpy, lz4, zstandard, fonttools and pillow, and with --prompt-art the button art
make_game.py's prompts step draws from (as make_kit.py --prompt-art: the pictures tools/prompt_art.py art_files names,
into the kit's prompt-art folder; without it that step is skipped and the game keeps the Switch prompts). zbic, the GPL
@@ -18,7 +19,9 @@ package that unpacks the game's program files, is not bundled: the AppImage inst
~/.local/share/nx2navi/pydeps the first time it runs, as the Windows launcher downloads it.
The compiler and the libraries come from the system (README-BuildKit.md, "Linux"): the AppImage checks for clang,
clang++, lld and the headers before it starts, and names the packages for Arch, Fedora and Debian/Ubuntu.
clang++, lld and the headers before it starts, and names the packages for Arch, Fedora and Debian/Ubuntu. Whether
they are new enough is found out by trying them: tools/make_game.py checks the runtime's sources with that compiler
and links against those libraries before the build's long steps (preflight).
"""
import argparse
import os
@@ -72,7 +75,8 @@ ev step deps '"state":"start"'
missing=()
for tool in clang clang++ ld.lld; do command -v "$tool" >/dev/null || missing+=("$tool"); done
for h in SDL3/SDL.h vulkan/vulkan.h opus/opus.h vpx/vpx_decoder.h dav1d/dav1d.h zstd.h; do
# The Vulkan headers come with this file (the kit's toolchain/vulkan-headers): a distribution's may be too old.
for h in SDL3/SDL.h opus/opus.h vpx/vpx_decoder.h dav1d/dav1d.h zstd.h; do
[ -e "/usr/include/$h" ] || [ -e "/usr/local/include/$h" ] || missing+=("$h")
done
if [ ${#missing[@]} -gt 0 ]; then
@@ -226,6 +230,10 @@ def main():
dlss_pin.place(dlss, kit / 'bin', linux=True)
(kit / 'toolchain' / 'vma').mkdir(parents=True)
shutil.copy2(ROOT / 'toolchain' / 'vma' / 'vk_mem_alloc.h', kit / 'toolchain' / 'vma' / 'vk_mem_alloc.h')
# The Vulkan headers the runtime is written against, ahead of the system's on the player's machine
# (tools/build_game.py vulkan_include finds them in the kit): fetched and checked here when not there yet
import vulkan_pin
shutil.copytree(vulkan_pin.resolve(ROOT / vulkan_pin.FOLDER), kit / vulkan_pin.FOLDER)
if a.prompt_art:
import prompt_art
n, size = prompt_art.copy_art(a.prompt_art, kit / prompt_art.FOLDER)

View File

@@ -922,6 +922,81 @@ def compile_all(srcs, cmd_for, cache, jobs, env, progress=None, salt='', stop=No
return len(todo), reused, bad
def report_failures(bad):
"""The files that did not compile, to stderr: the compiler's errors of the first three, and the names of all of
them, so that a paste of the build's end says which they were."""
for src, err in bad[:3]:
print(f'--- {src}\n{err}', file=sys.stderr)
print('did not compile: ' + ', '.join(Path(src).name for src, _ in bad), file=sys.stderr, flush=True)
def runtime_headers(rt):
"""The headers tools/build_shaders.py builds into runtime/ (the runtime's own shaders, the menu font's bytes): in
the repository as source only, so from a source checkout they are not there until they are built, and every
runtime file that includes one would fail to compile. A kit carries them built; a release's kit carries neither
tools/build_shaders.py nor the compilers it calls, and skips this."""
try:
import build_shaders
except ModuleNotFoundError:
return
stale = build_shaders.stale(rt)
if stale:
say(f' building {len(stale)} of the runtime\'s own headers (runtime/*_spv.h, the menu font)')
build_shaders.build(rt, stale)
def preflight(msys, jobs, sources=None):
"""Linux, before the long steps of a build whose compile step will run: this machine's compiler, headers and
libraries are tried on the runtime itself. Every runtime source that needs no header of this install is checked
with the compile step's own command (clang -fsyntax-only), and an empty program is linked against the libraries
the game links. The build takes the compiler, the linker and the libraries from the system (only the Vulkan
headers and VMA come with the kit), and a distribution's older compiler, older headers or a missing development
package stopped the compile step only after the RomFS image and the lift, ten minutes and more into the build.
sources: the files to check instead (tests)."""
clang, clangxx, env, vma = toolchain(msys)
version = clang_version(clang, env)
say(f'check: the compiler is {version}')
rt = KIT / 'runtime'
runtime_headers(rt)
inc = ['-I', str(rt), '-I', str(rt / 'third_party' / 'imgui'), '-I', str(vma)]
vulkan = vulkan_headers(clang, env)
cmd_for = runtime_cmd(clang, clangxx, inc, first=vulkan)
if sources is None:
tainted = per_install_names()
sources = [p for p, _ in runtime_sources() if not per_install(p, tainted)]
def check(src):
cmd = [c for c in cmd_for(src, Path('unused.o')) if c not in ('-c', '-o', 'unused.o')]
p = subprocess.run([cmd[0], '-fsyntax-only', *cmd[1:]], capture_output=True, text=True, env=env, errors='replace')
return src, p.returncode, p.stderr[-1500:]
t0 = time.time()
with concurrent.futures.ThreadPoolExecutor(max(1, jobs)) as ex:
bad = [(src, err) for src, rc, err in ex.map(check, sources) if rc]
if bad:
report_failures(bad)
lacking = sorted(set(re.findall(r"fatal error: '([^']+)' file not found", ''.join(e for _, e in bad))))
fail(f'{len(bad)} of the runtime\'s {len(sources)} files do not compile on this machine, so the build would stop '
f'in its compile step. ' +
(f'These headers are missing: {", ".join(lacking)}. Install the development packages '
f'README-BuildKit.md lists under "Linux".' if lacking else
f'The compiler ({version}) or a library\'s headers are older than the runtime needs: the releases are '
f'built with clang 23, and the Vulkan headers come ' +
(f'from {vulkan[1]}' if vulkan else 'from the system') + '. The errors above name what is missing.'))
with tempfile.TemporaryDirectory(prefix='nx2navi-check-') as tmp:
probe = Path(tmp) / 'probe.c'
probe.write_text('int main(void) { return 0; }\n', encoding='ascii')
ld = (['-fuse-ld=lld'] if shutil.which('ld.lld') else []) + ['-no-pie']
p = subprocess.run([str(clangxx), *ld, '-o', str(Path(tmp) / 'probe'), '-x', 'c', str(probe), '-x', 'none',
*LINK_LIBS], capture_output=True, text=True, env=env, errors='replace')
if p.returncode:
print(p.stderr[-2000:], file=sys.stderr)
fail('the libraries the game links are not all there (the linker\'s lines above name the missing ones): '
'install the development packages README-BuildKit.md lists under "Linux". '
'On Fedora the static C++ library is its own package, libstdc++-static.')
say(f'check: {len(sources)} runtime files and the libraries pass, {time.time() - t0:.0f} s')
def runtime_sources():
"""(source, object name) of every runtime file this OS's build compiles: the Win32 files on Windows, the SDL
window's (*_sdl.*) and platform/linux.c elsewhere (tools/build_game.py's foreign_os)."""
@@ -1023,8 +1098,10 @@ def fast_clang(msys, clang, env):
return fast if mine and clang_ident(fast, env) == mine else None
def runtime_cmd(clang, clangxx, inc, level=0, fast=None, msys=None):
"""fast: fast_clang's compiler for the lifted code, with msys as its sysroot."""
def runtime_cmd(clang, clangxx, inc, level=0, fast=None, msys=None, first=()):
"""fast: fast_clang's compiler for the lifted code, with msys as its sysroot. first: include flags that go
ahead of inc for the runtime's sources only (vulkan_headers); the lifted code includes none of those headers,
and its command, which is part of its objects' keys, stays as it was."""
def cmd_for(src, dst):
cpp = src.suffix in ('.cpp', '.cxx')
# the lifted code at -O1 (its size makes -O2 slow to build), the runtime at -O2; neither may assume strict
@@ -1037,7 +1114,7 @@ def runtime_cmd(clang, clangxx, inc, level=0, fast=None, msys=None):
# -fdeclspec: third_party/ffx_api/ffx_api.h declares its entry points __declspec(dllexport) unconditionally
declspec = ['-fdeclspec'] if cpp and not WIN else []
return [*cc, '-c', '-std=gnu++17' if cpp else '-std=gnu11', '-O1' if guest else '-O2',
*march, '-fno-strict-aliasing', '-w', *declspec, *inc, '-o', str(dst), str(src)]
*march, '-fno-strict-aliasing', '-w', *declspec, *([] if guest else first), *inc, '-o', str(dst), str(src)]
return cmd_for
@@ -1056,12 +1133,12 @@ def make_runtime_lib(dest, msys, jobs, obj=None):
inc = ['-I', str(rt), '-I', str(rt / 'third_party' / 'imgui'), '-I', str(vma)]
cache = ObjectCache(obj / 'objects.json', {'<kit>': KIT})
t0 = time.time()
compiled, reused, bad = compile_all(srcs, runtime_cmd(clang, clangxx, inc), cache, jobs, env,
compiled, reused, bad = compile_all(srcs, runtime_cmd(clang, clangxx, inc, first=vulkan_headers(clang, env)),
cache, jobs, env,
lambda d, t: (d % 20 == 0 or d == t) and say(f' {d} of {t}'),
salt=compiler_id(clang, env))
if bad:
for src, err in bad[:3]:
print(f'--- {src}\n{err}', file=sys.stderr)
report_failures(bad)
fail(f'{len(bad)} runtime files did not compile')
lib = dest / RUNTIME_LIB
lib.unlink(missing_ok=True)
@@ -1206,6 +1283,21 @@ def toolchain(msys):
return Path(shutil.which('clang')), Path(shutil.which('clang++')), dict(os.environ), vma
def vulkan_headers(clang, env):
"""The include flags that put the kit's Vulkan headers ahead of the system's for the runtime's sources: the
pinned release in <kit>/toolchain/vulkan-headers (tools/vulkan_pin.py), which the AppImage's kit carries and a
source checkout fetches when the system's headers are older (tools/build_game.py vulkan_include). None on
Windows, where MSYS2 supplies the headers, and none where the system's headers are new enough."""
if WIN:
return []
try:
from build_game import vulkan_include
folder = vulkan_include(clang, env)
except ImportError: # a kit without tools/vulkan_pin.py: the system's headers, as before it existed
return []
return ['-I', str(folder)] if folder else []
def xlate_source():
"""The folder of the shader translator to copy beside the game: the kit's bin/, else a source checkout's
build/xlate/ (tools/build_xlate.py)."""
@@ -1261,17 +1353,7 @@ def step_compile(out, cache_dir, msys, jobs, romfs_job=None, update=False):
gen.mkdir(parents=True, exist_ok=True)
obj.mkdir(parents=True, exist_ok=True)
rt = KIT / 'runtime'
# The runtime's own shaders are in the repository as source only: from a source checkout their headers are not
# there until they are built, and every runtime file that includes one would fail to compile. A kit carries them
# built and does not carry tools/build_shaders.py (nor the compilers it calls), so a kit build skips this step.
try:
import build_shaders
except ModuleNotFoundError:
build_shaders = None
stale = build_shaders.stale(rt) if build_shaders else []
if stale:
say(f' compiling {len(stale)} of the runtime\'s own shaders (runtime/*_spv.h)')
build_shaders.build(rt, stale)
runtime_headers(rt)
clang, clangxx, env, vma = toolchain(msys)
if not romfs_job:
romfs_index(out, gen)
@@ -1298,15 +1380,23 @@ def step_compile(out, cache_dir, msys, jobs, romfs_job=None, update=False):
fast = fast_clang(msys, clang, env)
if fast:
say(f' the lifted code compiles with {fast}')
cmd_for = runtime_cmd(clang, clangxx, inc, level, fast, msys)
vulkan = vulkan_headers(clang, env)
if vulkan:
say(f' the Vulkan headers come from {vulkan[1]}')
cmd_for = runtime_cmd(clang, clangxx, inc, level, fast, msys, first=vulkan)
# The keys earlier versions gave the same compiles (compile_all alts): 0.6.0's, with the compiler's file time and,
# with the fast compiler, its id and --sysroot; 0.5.0's and earlier's, with their kit's folder in the paths.
# with the fast compiler, its id and --sysroot; 0.5.0's and earlier's, with their kit's folder in the paths; and
# up to 0.7.1 the runtime's without the Vulkan headers' folder ahead of the system's (an object that compiled
# against the system's headers is as good).
plain = runtime_cmd(clang, clangxx, inc, level, None, msys)
no_vulkan = runtime_cmd(clang, clangxx, inc, level, fast, msys)
fast_old = (cid_old + compiler_ids(fast, env)[1]) if fast else None
kits = earlier_kits()
def alts(src, dst):
keys = [object_key(plain(src, dst), src, dst, cache, cid_old, sysroot=True)]
if vulkan:
keys.append(object_key(no_vulkan(src, dst), src, dst, cache, cid))
if fast:
keys.append(object_key(cmd_for(src, dst), src, dst, cache, fast_old, sysroot=True))
keys += [legacy_key(src, dst, cache, cid_old, clang, clangxx, k, lift, gen, vma) for k in kits]
@@ -1344,8 +1434,7 @@ def step_compile(out, cache_dir, msys, jobs, romfs_job=None, update=False):
compiled, reused = compiled + c2, reused + r2
say(f' {compiled} compiled, {reused} unchanged since the last build')
if bad:
for src, err in bad[:3]:
print(f'--- {src}\n{err}', file=sys.stderr)
report_failures(bad)
fail(f'{len(bad)} files did not compile')
if cache.keep_only({o.name for _, o in srcs}):
cache.save()
@@ -1891,6 +1980,10 @@ def main():
# the compile step's key does not name the level: without this a finished compile reads as done before
state['steps'].pop('compile', None)
keep_awake()
# Linux: the compiler, headers and libraries are the system's, so they are tried on the runtime before the long
# steps of a build that will reach the compile step (the first step whose key is not current makes the rest run)
if not WIN and any(state['steps'].get(n) != step_key(n, src.identity, out) for n in STEPS[:STEPS.index('compile') + 1]):
preflight(msys, a.jobs)
t0 = time.time()
romfs_job = None
runs = {'exefs': lambda: step_exefs(src, out), 'romfs': lambda: step_romfs(src, out, min(a.jobs, 8)),

View File

@@ -30,7 +30,8 @@ NAME = 'NX2navi-BuildKit'
TOOLS = ('make_game.py', 'nup.py', 'romfs_index.py', 'check_content.py', 'hostmem.py', 'prepare_shaders.py',
'shader_census.py', 'shader_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py', 'romfs_check.py',
'build_game.py', 'check_isa.py', 'build_shaders.py', 'make_dlaa_mask.py', 'make_mvobj_fs.py',
'pack_shader_cache.py') # make_game.py imports build_game's vma_include on Linux for the VMA header, and
'pack_shader_cache.py', 'vulkan_pin.py') # make_game.py imports build_game's vma_include and vulkan_include on
# Linux for the VMA header and for Vulkan headers newer than the system's (which build_game takes from vulkan_pin), and
# build_game.py imports check_isa and build_shaders, which reads the two make_ scripts' shaders
TOP = ('LICENSE', 'LICENSE-ADDITIONS', 'THIRD_PARTY_NOTICES.md', 'VERSION', 'nx2navi.ini.example')
SKIP_DIRS = {'__pycache__', '.pytest_cache'}

View File

@@ -40,6 +40,7 @@ from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
import build_game # noqa: E402
import make_release as mr # noqa: E402
import vulkan_pin # noqa: E402
ROOT = mr.ROOT
MAIN_ROOT = mr.MAIN_ROOT
@@ -69,8 +70,7 @@ PINS = {
'5749cbc4e668273514150a80e387a957f933c6ed3f5f11e03fb30955e2bbead6'),
'zstd-src': ('https://github.com/facebook/zstd/releases/download/v1.5.7/zstd-1.5.7.tar.gz',
'eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3'),
'vulkan-headers-src': ('https://github.com/KhronosGroup/Vulkan-Headers/archive/refs/tags/v1.4.363.tar.gz',
'cbaf687d3c59b9666fe080e7f8396b6b4f4d344a768ee6b337c59852f4526b68'),
'vulkan-headers-src': vulkan_pin.SOURCE, # the release a build with the system's headers takes too
'vma-header': build_game.VMA_FETCH,
# appimagetool and the type-2 runtime from release tags, not "continuous", whose uploads change under the same
# URL and would stop the build at the hash check
@@ -169,8 +169,8 @@ def build_deps(out, llvm, sysroot):
d = out / 'runtime' / 'deps'
(d / 'include' / 'vma').mkdir(parents=True, exist_ok=True)
(d / 'lib').mkdir(parents=True, exist_ok=True)
vk = tar_extract('vulkan-headers-src', WORK / 'vulkan-headers', ['Vulkan-Headers-1.4.363/include',
'Vulkan-Headers-1.4.363/LICENSE.md'], strip=1)
vk = tar_extract('vulkan-headers-src', WORK / 'vulkan-headers', [f'{vulkan_pin.TOP}include',
f'{vulkan_pin.TOP}LICENSE.md'], strip=1)
for sub in ('vulkan', 'vk_video'):
shutil.copytree(vk / 'include' / sub, d / 'include' / sub)
shutil.copy2(mr.fetch('vma-header'), d / 'include' / 'vma' / 'vk_mem_alloc.h')

109
tools/vulkan_pin.py Normal file
View File

@@ -0,0 +1,109 @@
"""The Vulkan headers the runtime is compiled against where the system's are older.
The runtime names structures and constants of Vulkan extensions as they come out (runtime/vk_device.c asks for
VK_KHR_pipeline_binary, VK_KHR_dynamic_rendering_local_read and VK_KHR_shader_maximal_reconvergence when the driver
has them), so it needs headers at least as new as the ones it was written against. On Windows MSYS2 supplies them and
the Linux release (tools/make_release_linux.py) carries them in runtime/deps. A build with the system's own compiler
and headers (tools/make_appimage.py's kit, a source checkout) met distributions whose packaged headers are years
older: every file that names a newer extension then fails to compile. Such a build takes the headers from here
instead: the release the others use, from Khronos's Vulkan-Headers repository, checked against its SHA-256.
Only the C headers are kept (include/vulkan/*.h and include/vk_video/*.h, which vulkan_core.h includes) with the
repository's licence note. The headers are Apache-2.0 OR MIT (THIRD_PARTY_NOTICES.md). Standard library only: the
build kit's Python runs this.
python tools/vulkan_pin.py [--dest toolchain/vulkan-headers] fetch and unpack the pinned headers
"""
import argparse
import hashlib
import io
import re
import shutil
import sys
import tarfile
import urllib.request
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
VERSION = '1.4.363'
HEADER_VERSION = 363 # VK_HEADER_VERSION of that release; the numbers only grow, across 1.3 and 1.4 alike
# (address, SHA-256): the archive GitHub makes of the release's tag
SOURCE = (f'https://github.com/KhronosGroup/Vulkan-Headers/archive/refs/tags/v{VERSION}.tar.gz',
'cbaf687d3c59b9666fe080e7f8396b6b4f4d344a768ee6b337c59852f4526b68')
TOP = f'Vulkan-Headers-{VERSION}/' # the archive's top folder
FOLDER = Path('toolchain') / 'vulkan-headers' # below a checkout's or a kit's root: include/ and LICENSE.md
STAMP = 'pinned.txt' # in the folder: the release it holds, written last
KEEP = re.compile(r'include/(vulkan|vk_video)/[A-Za-z0-9_]+\.h|LICENSE\.md')
def have(folder):
"""True when folder holds the pinned headers, complete (the stamp is written after the last file)."""
try:
return (folder / STAMP).read_text(encoding='ascii').strip() == VERSION and \
(folder / 'include' / 'vulkan' / 'vulkan_core.h').is_file()
except OSError:
return False
def unpack(archive, folder):
"""The C headers and the licence note of the release's archive (bytes) into folder, replacing what is there."""
shutil.rmtree(folder, ignore_errors=True)
n = 0
with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar:
for m in tar:
rel = m.name[len(TOP):] if m.name.startswith(TOP) else ''
if not m.isfile() or not KEEP.fullmatch(rel): # a name the pattern takes has no '..' and no other folder
continue
dst = folder / rel
dst.parent.mkdir(parents=True, exist_ok=True)
dst.write_bytes(tar.extractfile(m).read())
n += 1
if not (folder / 'include' / 'vulkan' / 'vulkan_core.h').is_file() or not (folder / 'LICENSE.md').is_file():
shutil.rmtree(folder, ignore_errors=True)
sys.exit(f'vulkan_pin: the archive of Vulkan-Headers {VERSION} lacks vulkan_core.h or LICENSE.md')
(folder / STAMP).write_text(VERSION + '\n', encoding='ascii')
return n
def resolve(folder=None, archive=None):
"""The folder holding the pinned headers (default <root>/toolchain/vulkan-headers), fetched and unpacked there
when it does not hold them yet. archive: the release's archive on disk instead of the download; it must be the
pinned file either way. Returns the folder; its include/ goes on the include path."""
folder = Path(folder) if folder else ROOT / FOLDER
if have(folder):
return folder
url, want = SOURCE
if archive:
data = Path(archive).read_bytes()
else:
print(f'vulkan_pin: fetching the Vulkan headers {VERSION} into {folder}', flush=True)
try:
data = urllib.request.urlopen(url, timeout=120).read()
except OSError as e:
sys.exit(f'vulkan_pin: fetching {url} failed ({e}). Install Vulkan headers {VERSION} or newer, or put the '
f'archive on disk and run tools/vulkan_pin.py --archive <file>')
if hashlib.sha256(data).hexdigest() != want:
sys.exit(f'vulkan_pin: {archive or url} is not the pinned archive of Vulkan-Headers {VERSION} (SHA-256 differs)')
unpack(data, folder)
return folder
def header_version(text):
"""VK_HEADER_VERSION as vulkan_core.h defines it in text, 0 when it does not."""
m = re.search(r'^[ \t]*#[ \t]*define[ \t]+VK_HEADER_VERSION[ \t]+(\d+)', text, re.M)
return int(m.group(1)) if m else 0
def main():
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
ap.add_argument('--dest', type=Path, default=ROOT / FOLDER, help='where the headers go (default toolchain/vulkan-headers)')
ap.add_argument('--archive', type=Path, help=f'the archive of Vulkan-Headers {VERSION} on disk instead of the download')
a = ap.parse_args()
folder = resolve(a.dest, a.archive)
n = sum(1 for _ in (folder / 'include').rglob('*.h'))
print(f'{folder}: Vulkan headers {VERSION}, {n} files')
return 0
if __name__ == '__main__':
sys.exit(main())