diff --git a/.gitignore b/.gitignore index 66d8ebec..464aaae3 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,8 @@ __pycache__/ # tools/make_*.py assemblers when the game is built; tools/check_content.py refuses them in a commit) /runtime/*_spv.h *.spv +# The menu font's bytes as a header for compilers without #embed (tools/build_shaders.py EMBEDDED) +/runtime/ui/roboto_ttf.h # Build outputs *.exe diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 26317abd..83a29525 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -123,6 +123,7 @@ folder's `licenses/` and in each component's own folder. | zlib 1.4 (MSYS2 `mingw-w64-ucrt-x86_64-zlib`), static library | https://zlib.net, Copyright (C) Jean-loup Gailly and Mark Adler | zlib | linked into `NX2navi-Setup.exe` (its unzip) | | mingw-w64 CRT startup code and winpthreads 14.0.0, GCC 16.2.0's `libgcc` (MSYS2 packages), static | https://www.mingw-w64.org, https://gcc.gnu.org | mingw-w64: ZPL-2.1 / MIT / public domain; libgcc: GPL-3.0 with the GCC Runtime Library Exception | linked into `NX2navi.exe` and `NX2navi-Setup.exe` | | NVIDIA DLSS 310.9.1 (`nvngx_dlss.dll`, NVIDIA's own build, unmodified; pinned with its size and sha256 in `tools/dlss_pin.py`) | https://github.com/NVIDIA/DLSS @ `v310.9.1` (`lib/Windows_x86_64/rel/nvngx_dlss.dll`), Copyright NVIDIA Corporation | NVIDIA RTX SDKs License (https://github.com/NVIDIA/DLSS/blob/v310.9.1/LICENSE.txt); licensed for systems with NVIDIA GPUs only | `app//bin/` with `nvngx_dlss-NOTICE.txt`, also in the build kit's `bin/`; `tools/make_game.py` copies both beside the built game, where NGX loads the library (`runtime/vk_dlaa.c`) | +| Vulkan headers 1.4.363 (`include/vulkan/*.h` and `include/vk_video/*.h`, unmodified; the release pinned with its sha256 in `tools/vulkan_pin.py`) | https://github.com/KhronosGroup/Vulkan-Headers @ `v1.4.363`, Copyright 2015-2026 The Khronos Group Inc. | Apache-2.0 OR MIT | the Linux build kit's `toolchain/vulkan-headers/` (the AppImage of `tools/make_appimage.py`) with the repository's `LICENSE.md`; `tools/make_game.py` compiles the runtime against them on the player's PC, ahead of the system's headers. Never in this repository: a build fetches them | | NVIDIA DLSS 310.9.1 for Linux (`libnvidia-ngx-dlss.so.310.9.1`, NVIDIA's own build, unmodified; pinned with its size and sha256 in `tools/dlss_pin.py`) | https://github.com/NVIDIA/DLSS @ `v310.9.1` (`lib/Linux_x86_64/rel/libnvidia-ngx-dlss.so.310.9.1`), Copyright NVIDIA Corporation | NVIDIA RTX SDKs License (https://github.com/NVIDIA/DLSS/blob/v310.9.1/LICENSE.txt), the one licence of the repository's Windows and Linux libraries; licensed for systems with NVIDIA GPUs only | the Linux build kit's `bin/` (the AppImage of `tools/make_appimage.py`, unless made with `--no-dlss`) with `libnvidia-ngx-dlss-NOTICE.txt`; `tools/make_game.py` copies both beside the built game, where NGX loads the library (`runtime/vk_dlaa.c`) | | NVIDIA Streamline 2.14.16 runtime (`sl.interposer.dll`, `sl.common.dll`, `sl.dlss_g.dll`, `sl.pcl.dll`, `sl.reflex.dll`, NVIDIA's own builds, unmodified) and NVIDIA DLSS Frame Generation 310.9.16 (`nvngx_dlssg.dll`, as the Streamline SDK ships it) | https://github.com/NVIDIA-RTX/Streamline, SDK release 2.14.16 (`bin/x64/`), Copyright NVIDIA Corporation | Streamline: MIT; `nvngx_dlssg.dll`: NVIDIA RTX SDKs License, licensed for systems with NVIDIA GPUs only | `app//bin/dlssg/` with both licence texts and `dlssg-NOTICE.txt` (each file's sha256), when the release is made with `tools/make_launcher.py --dlssg`, and the build kit's `bin/dlssg/` with `tools/make_kit.py --dlssg` (the SDK release pinned with its size and sha256 and each file's sha256 in `tools/streamline_pin.py`); `tools/make_game.py` copies them beside the built game, where `nx2navi_dlssg.dll` (this repository's `runtime/dlssg/`, built with the Streamline SDK's MIT headers) loads them (`runtime/vk_dlssg.c`) | | Button prompt art: the 128x128 PNGs of the Input Prompts Pack V7 (`XGamepad`, `P5Gamepad` and `Keyboard_Mouse` styles), unmodified | the pack as licensed to the repository owner; never in this repository | the pack's licence, which allows redistributing these pictures in NX2navi's releases (confirmed by the repository owner, 2026-11-09) | `app//prompt-art/` and the build kit's `prompt-art/` when made with `--prompt-art` (`tools/prompt_art.py`); `tools/pad_prompts.py` draws the game's Xbox, PlayStation, Steam Deck (from the Xbox and PlayStation pictures) and keyboard and mouse prompts from them on the player's PC | diff --git a/docs/README-BuildKit.md b/docs/README-BuildKit.md index 5f29d7c6..bd459a32 100644 --- a/docs/README-BuildKit.md +++ b/docs/README-BuildKit.md @@ -78,6 +78,17 @@ RADV). It takes clang, clang++ and lld from `PATH` and links against the system' (GPL-2.0): `pip install --user zbic`, or in a virtual environment with the packages above. Vulkan Memory Allocator is one header: the system's (AUR `vulkan-memory-allocator`), else the build fetches the pinned 3.4.0 into `toolchain/vma/` and checks its hash. + The Vulkan headers must be release 363 (1.4.363) or newer: the runtime names extensions that older headers do not + declare (`VK_KHR_pipeline_binary` among them), and distributions with long support package older ones. The + AppImage carries the pinned headers (`toolchain/vulkan-headers/`, `tools/vulkan_pin.py`) and builds against them + whatever the system has; a build from a kit or a source checkout uses the system's when they are new enough and + otherwise fetches the pinned ones there once and checks their hash. The loader's library still comes from the + system (`vulkan-icd-loader`, `libvulkan-dev`, `vulkan-loader-devel`). + The compiler is the system's too. The releases are built with clang 23; an older one works as far as it compiles + the runtime (one without `#embed`, before clang 19, takes the menu's font from a header the kit carries). Before + its long steps the build checks every runtime source with that compiler and links against the libraries, so a + compiler, a header or a library it cannot use stops it in the first seconds, with the compiler's own words, and + not after the RomFS image. 2. The shader translator: a kit with `bin/nx2navi_xlate` has it; from a source checkout, build it once with `python tools/build_xlate.py` (needs the .NET 10 SDK). The build copies it beside the game. A kit also carries the @@ -132,8 +143,9 @@ switches at once, with a *DLSS multiplier* of 2x to 6x (above 2x on RTX 50 cards frames needed to reach the display's refresh rate (`framegen = dlss` and `fg_mult` in `nx2navi.ini`). The log says `ngxfg: DLSS Frame Generation available` when NGX found the library. -`tools/make_appimage.py` packs this kit into one `NX2navi-Linux--x86_64.AppImage` with its own Python, for -players without a source checkout: it checks for the packages above, installs zbic with pip on its first run, picks +`tools/make_appimage.py` packs this kit into one `NX2navi-Linux--x86_64.AppImage` with its own Python and the +pinned Vulkan headers, for players without a source checkout: it checks for the packages above (their Vulkan headers +are not needed), installs zbic with pip on its first run, picks `--jobs` from the memory, builds into `~/Games/Ocarina` (or the folder given after the dump) and writes `play.sh` beside the game. diff --git a/docs/building.md b/docs/building.md index 1a5d9286..de709c79 100644 --- a/docs/building.md +++ b/docs/building.md @@ -94,6 +94,9 @@ of them. .venv\Scripts\python tools\build_game.py --version ``` + The menu's font comes into `runtime/ui/menu.cpp` with `#embed`; for a compiler without it (before clang 19, which + a Linux build with the distribution's compiler can meet) the same step writes the font's bytes as + `runtime/ui/roboto_ttf.h` (git ignores it), which `menu.cpp` then includes. The runtime's own shaders (upscaling, HDR, texture decode and the like) are in the repository as source only: GLSL and SPIR-V assembly in `runtime\shaders\`, and two small ones assembled by `tools\make_dlaa_mask.py` and `tools\make_mvobj_fs.py`. `build_game.py` compiles them into `runtime\_spv.h` (git ignores these) before it @@ -219,6 +222,10 @@ Requirements: Vulkan Memory Allocator is a single header (AUR `vulkan-memory-allocator`). Without it, `build_game.py` fetches the version the Windows release pins (3.4.0) into `toolchain/vma/` on first use and checks its hash. + The Vulkan headers must be release 363 (1.4.363, the one MSYS2 supplies on Windows) or newer: the runtime names + extensions older headers lack. Where the system's are older, `build_game.py` and `make_game.py` fetch that release + into `toolchain/vulkan-headers/` on first use, check its hash and put it ahead of the system's (`tools/vulkan_pin.py`, + `build_game.py` `vulkan_include`). The steps are the Windows ones with POSIX paths: `.venv/bin/python` in place of `.venv\Scripts\python`, and `nx2navi` in place of `nx2navi.exe`. `build_game.py` takes clang, lld and Ninja from PATH (or the ones in `/bin` with diff --git a/runtime/ui/menu.cpp b/runtime/ui/menu.cpp index ff048e40..957d9f3c 100644 --- a/runtime/ui/menu.cpp +++ b/runtime/ui/menu.cpp @@ -82,11 +82,18 @@ void cheats_frame(uint64_t frame); // cheats.c bool tab_cheats(float scale); // ui/cheats_tab.cpp; true: restart into the rebuild } -// Roboto Medium (Apache-2.0, ui/fonts/LICENSE-Roboto.txt) with Latin and Cyrillic glyphs. +// Roboto Medium (Apache-2.0, ui/fonts/LICENSE-Roboto.txt) with Latin and Cyrillic glyphs. The file comes in with +// #embed where the compiler has it (clang 19 and later define __has_embed). An older one, which a Linux build with the +// distribution's compiler can meet, includes the same bytes as numbers from roboto_ttf.h, which tools/build_shaders.py +// writes beside this file from the font when the game is built (a kit carries it). #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wc23-extensions" // #embed in C++ static const unsigned char roboto_ttf[] = { +#ifdef __has_embed #embed "fonts/Roboto-Medium.ttf" +#else +#include "roboto_ttf.h" +#endif }; #pragma clang diagnostic pop diff --git a/tests/launcher_test.py b/tests/launcher_test.py index 915ee895..55707c2a 100644 --- a/tests/launcher_test.py +++ b/tests/launcher_test.py @@ -1954,6 +1954,131 @@ class MakeGameTest(Tmp): self.assertNotEqual(self.mg.object_key(plain, src, dst, cache, 'id', sysroot=True), self.mg.object_key(fast, src, dst, cache, 'id', sysroot=True)) + def test_vulkan_headers_go_first_for_the_runtime_only(self): + """The folder of the pinned Vulkan headers goes ahead of every other include folder for the runtime's sources, + so that no folder that also is a system one answers for ; the lifted code's command, which is + part of its objects' keys, is the one it was.""" + inc = ['-I', 'rt', '-I', '/usr/include'] + with_vk = self.mg.runtime_cmd('clang', 'clang++', inc, 3, first=['-I', 'kit/vulkan']) + without = self.mg.runtime_cmd('clang', 'clang++', inc, 3) + guest = (Path('lift/main_0001.c'), Path('obj/main_main_0001.o')) + self.assertEqual(with_vk(*guest), without(*guest)) + for src in (Path('rt/vk_device.c'), Path('rt/ui/menu.cpp')): + cmd, plain = with_vk(src, Path('obj/rt_x.o')), without(src, Path('obj/rt_x.o')) + self.assertEqual(cmd.count('-I'), plain.count('-I') + 1) + self.assertLess(cmd.index('kit/vulkan'), cmd.index('rt')) + self.assertLess(cmd.index('kit/vulkan'), cmd.index('/usr/include')) + + def test_early_check_tries_the_runtime_before_the_long_steps(self): + """The check of a Linux build runs the compile step's own command with -fsyntax-only on each source and links + an empty program against the game's libraries. When files fail it stops with every one of them named (the + compile step named three of four), and a missing header is told from an old one.""" + srcs = [self.tmp / n for n in ('good.c', 'bad1.c', 'bad2.c', 'bad3.c', 'bad4.cpp')] + for s in srcs: + s.write_text('int x;\n') + ran = [] + + def compiler(errors): + def run(cmd, **kw): + ran.append([str(c) for c in cmd]) + name = next((Path(c).name for c in cmd if Path(str(c)).name.startswith(('good', 'bad'))), '') + bad = name.startswith('bad') + return subprocess.CompletedProcess(cmd, 1 if bad else 0, '', errors.get(name, 'error: old') if bad else '') + return run + + def check(errors, sources): + ran.clear() + err = io.StringIO() + with mock.patch.multiple(self.mg, toolchain=lambda msys: (Path('clang'), Path('clang++'), {}, Path('vma')), + clang_version=lambda clang, env: 'clang version 18.1.3', + vulkan_headers=lambda clang, env: ['-I', 'kit/vulkan'], + runtime_headers=lambda rt: None, EVENTS=False), \ + mock.patch.object(self.mg.subprocess, 'run', compiler(errors)), \ + contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(err): + try: + self.mg.preflight(self.msys, 2, sources) + return None, err.getvalue() + except SystemExit: + return 'stopped', err.getvalue() + + stopped, text = check({}, srcs) + self.assertEqual(stopped, 'stopped') + self.assertIn('did not compile: bad1.c, bad2.c, bad3.c, bad4.cpp', text) + self.assertIn('4 of the runtime\'s 5 files', text) + self.assertIn('clang version 18.1.3', text) + self.assertIn('kit/vulkan', text) # where the Vulkan headers came from + compiles = [c for c in ran if '-fsyntax-only' in c] + self.assertEqual(len(compiles), 5) + for c in compiles: + self.assertEqual(c[1], '-fsyntax-only') + self.assertFalse({'-c', '-o'} & set(c)) + self.assertLess(c.index('kit/vulkan'), c.index(str(self.mg.KIT / 'runtime'))) + self.assertEqual(len(ran), 5) # no link is tried once sources failed + stopped, text = check({'bad1.c': "bad1.c:1:10: fatal error: 'SDL3/SDL.h' file not found"}, srcs[:2]) + self.assertIn('These headers are missing: SDL3/SDL.h', text) + # every source passes: the empty program is linked against the game's libraries + stopped, text = check({}, srcs[:1]) + self.assertIsNone(stopped) + link = ran[-1] + self.assertNotIn('-fsyntax-only', link) + self.assertEqual(link[-len(self.mg.LINK_LIBS):], self.mg.LINK_LIBS) + + def test_pinned_vulkan_headers(self): + """tools/vulkan_pin.py keeps the C headers and the licence note of the pinned archive and nothing else, takes + no other archive, and says the folder holds them only once the last file is written.""" + import vulkan_pin + buf = io.BytesIO() + with tarfile.open(fileobj=buf, mode='w:gz') as tar: + for name, data in (('include/vulkan/vulkan_core.h', b'#define VK_HEADER_VERSION 363\n'), + ('include/vulkan/vulkan.h', b'#include "vulkan_core.h"\n'), + ('include/vk_video/vulkan_video_codecs_common.h', b'\n'), + ('include/vulkan/vulkan.hpp', b'// C++\n'), ('registry/vk.xml', b''), + ('include/vulkan/../../outside.h', b'x'), ('LICENSE.md', b'Apache-2.0 OR MIT\n')): + info = tarfile.TarInfo(vulkan_pin.TOP + name) + info.size = len(data) + tar.addfile(info, io.BytesIO(data)) + archive = self.tmp / 'headers.tar.gz' + archive.write_bytes(buf.getvalue()) + dest = self.tmp / 'toolchain' / 'vulkan-headers' + with self.assertRaises(SystemExit), contextlib.redirect_stdout(io.StringIO()): + vulkan_pin.resolve(dest, archive) # not the pinned archive + self.assertFalse(vulkan_pin.have(dest)) + pinned = (vulkan_pin.SOURCE[0], hashlib.sha256(buf.getvalue()).hexdigest()) + with mock.patch.object(vulkan_pin, 'SOURCE', pinned): + self.assertEqual(vulkan_pin.resolve(dest, archive), dest) + files = sorted(f.relative_to(dest).as_posix() for f in dest.rglob('*') if f.is_file()) + self.assertEqual(files, ['LICENSE.md', 'include/vk_video/vulkan_video_codecs_common.h', + 'include/vulkan/vulkan.h', 'include/vulkan/vulkan_core.h', vulkan_pin.STAMP]) + self.assertFalse((self.tmp / 'outside.h').exists() or (self.tmp / 'toolchain' / 'outside.h').exists()) + self.assertTrue(vulkan_pin.have(dest)) + self.assertEqual(vulkan_pin.resolve(dest), dest) # there already: nothing is fetched + self.assertEqual(vulkan_pin.header_version((dest / 'include' / 'vulkan' / 'vulkan_core.h').read_text()), 363) + self.assertEqual(vulkan_pin.header_version('#define VK_HEADER_VERSION_COMPLETE 1\n'), 0) + (dest / vulkan_pin.STAMP).unlink() # a folder cut short while it was written + self.assertFalse(vulkan_pin.have(dest)) + + def test_embedded_file_as_a_header_of_bytes(self): + """tools/build_shaders.py writes the menu font's bytes as a header for compilers without #embed: the numbers + are the file's bytes, and the header is stale once the file changes.""" + import build_shaders + rt = self.tmp / 'runtime' + font = rt / 'ui' / 'fonts' / 'Roboto-Medium.ttf' + font.parent.mkdir(parents=True) + data = bytes(range(256)) * 3 + b'\r\n\x00tail' + font.write_bytes(data) + header = rt / 'ui' / 'roboto_ttf.h' + with mock.patch.object(build_shaders, 'ASSEMBLED', {}), contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(build_shaders.stale(rt), [font]) + self.assertEqual(build_shaders.build(rt), 1) + self.assertEqual(build_shaders.stale(rt), []) + numbers = ''.join(ln for ln in header.read_text(encoding='utf-8').splitlines() if not ln.startswith('//')) + self.assertEqual(bytes(int(n) for n in numbers.split(',') if n), data) + font.write_bytes(data + b'!') + self.assertEqual(build_shaders.stale(rt), [font]) + menu = (ROOT / 'runtime' / 'ui' / 'menu.cpp').read_text(encoding='utf-8') + self.assertIn('#ifdef __has_embed\n#embed "fonts/Roboto-Medium.ttf"\n#else\n#include "roboto_ttf.h"\n#endif', menu) + self.assertEqual(build_shaders.EMBEDDED, {'ui/fonts/Roboto-Medium.ttf': 'ui/roboto_ttf.h'}) + def test_objects_of_an_earlier_version_are_taken_over(self): """An entry under a key an earlier version gave the same compile is re-recorded under the new key when its files read as they did; a header in the earlier kit's folder is read in this kit's (that folder may be gone).""" diff --git a/tools/build_game.py b/tools/build_game.py index e4eaad0b..7323de3e 100644 --- a/tools/build_game.py +++ b/tools/build_game.py @@ -61,6 +61,7 @@ import ctypes.wintypes as wt # the job-object structures below; importing it w import hashlib import json import os +import re import shutil import subprocess import sys @@ -180,6 +181,40 @@ def vma_include(): return VMA_LOCAL +def system_vulkan_version(clang, env=None): + """VK_HEADER_VERSION of the this compiler finds by itself (its own search path, CPATH + included), 0 when it finds none.""" + probe = '#include \nnx2navi_vulkan_headers VK_HEADER_VERSION\n' + try: + p = subprocess.run([str(clang), '-E', '-P', '-x', 'c', '-'], input=probe, capture_output=True, text=True, + env=env, errors='replace') + except OSError: + return 0 + m = re.search(r'nx2navi_vulkan_headers\s+(\d+)', p.stdout) if p.returncode == 0 else None + return int(m.group(1)) if m else 0 + + +def vulkan_include(clang=None, env=None): + """A build with the system's own headers (Linux without --deps): the folder to put ahead of them for + , or None when the system's serve. The runtime names Vulkan extensions newer than the headers + many distributions package, and every file that does fails to compile there (tools/vulkan_pin.py). So: + toolchain/vulkan-headers when it holds the pinned release (a kit carries it; a checkout once it was fetched), else + nothing when the system's headers are at least that release, else the pinned release, fetched there once. + Windows takes MSYS2's headers and returns None.""" + if WIN: + return None + import vulkan_pin + local = ROOT / vulkan_pin.FOLDER + if vulkan_pin.have(local): + return local / 'include' + found = system_vulkan_version(clang or CLANG, env) + if found >= vulkan_pin.HEADER_VERSION: + return None + print(f'build_game: the system\'s Vulkan headers are {f"release {found}" if found else "missing"}; the runtime is ' + f'built against release {vulkan_pin.HEADER_VERSION} ({vulkan_pin.VERSION}) or newer', flush=True) + return vulkan_pin.resolve(local) / 'include' + + # --toolchain: an llvm-mingw tree. The libraries, headers and DLLs MSYS2 supplies come from the deps directory instead. LLVM_LINK_LIBS = ['-lpsapi', '-lbcrypt', '-luser32', '-lgdi32', '-ldwmapi', '-limm32', '-lwinmm', '-lole32', '-lshell32', '-luuid', '-lvulkan-1', '-lSDL3', '-lopus', '-l:libdav1d.a', '-l:libvpx.a', '-l:libzstd.a', '-l:libc++.a', '-l:libc++abi.a', '-l:libunwind.a', '-l:libwinpthread.a'] @@ -380,6 +415,9 @@ def ninja_file(gen, out, mods, pool_depth, skip=(), prebuilt=False, runtime_lib= vma = f'{cmd_path(DEPS / "include")} -I {cmd_path(DEPS / "include" / "vma")}' if DEPS else cmd_path(vma_include()) imgui = cmd_path(RUNTIME / 'third_party' / 'imgui') rt_inc = f'{inc} -I {cmd_path(out)} -I {vma} -I {imgui}' # romfs_config.h, vk_mem_alloc.h (+ deps headers), imgui.h + vulkan = None if DEPS else vulkan_include() # deps carries the Vulkan headers itself + if vulkan: # first, so that no other folder on the path answers for (VMA's may be /usr/include) + rt_inc = f'-I {cmd_path(vulkan)} {rt_inc}' libs = [f'-L {cmd_path(DEPS / "lib")}'] + (LLVM_LINK_LIBS if WIN else LINK_LIBS) if DEPS else LINK_LIBS # A compiler cache (sccache) may front clang: every lift rewrites the generated chunks with the same contents, so # the cached objects save the compile step on the next build. diff --git a/tools/build_shaders.py b/tools/build_shaders.py index feddcca7..69088412 100644 --- a/tools/build_shaders.py +++ b/tools/build_shaders.py @@ -1,4 +1,4 @@ -"""Build the runtime's own shaders into C headers of SPIR-V words. +"""Build the runtime's own shaders into C headers of SPIR-V words, and the files it embeds into headers of bytes. The repository holds the shaders as source only. Each becomes runtime/_spv.h, which declares `static const uint32_t _spv[]`; the headers are build outputs (ignored by git) and tools/build_game.py builds the @@ -8,6 +8,9 @@ missing and the stale ones before it compiles the runtime. Three kinds of source runtime/shaders/.spvasm SPIR-V assembly, assembled with spirv-as and checked with spirv-val tools/make_dlaa_mask.py, make_mvobj_fs.py shaders assembled in Python (ASSEMBLED); they need no other tool +One more kind of header is no shader: a file the runtime embeds (EMBEDDED: the menu's font) as a list of its bytes, +for a compiler without #embed (runtime/ui/menu.cpp includes it there). It needs no tool. + A compiled or assembled header records the SHA-256 of its source and is stale when that no longer matches; a Python one is stale when its text is not what its script gives. glslc, spirv-as and spirv-val come from MSYS2 UCRT64 on Windows (packages mingw-w64-ucrt-x86_64-shaderc and mingw-w64-ucrt-x86_64-spirv-tools) and from PATH elsewhere @@ -36,10 +39,28 @@ ASM = '.spvasm' ASM_TARGET = 'vulkan1.3' # header name -> the script beside this one that assembles it; the script's HEADER is the header's text ASSEMBLED = {'dlaa_mask': 'make_dlaa_mask.py', 'mvobj_fs': 'make_mvobj_fs.py'} +# a file below runtime/ the runtime embeds -> the header of its bytes, below runtime/ too (git ignores it) +EMBEDDED = {'ui/fonts/Roboto-Medium.ttf': 'ui/roboto_ttf.h'} + + +def embedded(src): + """True for a file of EMBEDDED: a binary file, read as it is.""" + return src.suffix not in STAGES and src.suffix not in (ASM, '.py') def source_hash(src): - return hashlib.sha256(src.read_bytes().replace(b'\r\n', b'\n')).hexdigest() # the same on a CRLF checkout + data = src.read_bytes() + return hashlib.sha256(data if embedded(src) else data.replace(b'\r\n', b'\n')).hexdigest() # text: the same on a CRLF checkout + + +def bytes_header(src, runtime): + """The header of an embedded file: its bytes as decimal numbers, to include inside an array's braces.""" + data = src.read_bytes() + lines = [f'// Generated by tools/build_shaders.py from runtime/{src.relative_to(runtime).as_posix()}; do not edit.', + f'// source sha256 {source_hash(src)}', + f'// The file\'s {len(data)} bytes, for a compiler without #embed.'] + lines += [','.join(map(str, data[i:i + 32])) + ',' for i in range(0, len(data), 32)] + return '\n'.join(lines) + '\n' def header_of(src, words): @@ -64,6 +85,7 @@ def sources(runtime): found = sorted(p for p in shaders.iterdir() if p.suffix in STAGES or p.suffix == ASM) if shaders.is_dir() else [] out = [(p, runtime / f'{p.stem}_spv.h') for p in found] out += [(TOOLS / script, runtime / f'{name}_spv.h') for name, script in ASSEMBLED.items() if (TOOLS / script).is_file()] + out += [(runtime / src, runtime / header) for src, header in EMBEDDED.items() if (runtime / src).is_file()] return out @@ -130,12 +152,14 @@ def build(runtime, only=None): todo = stale(runtime) if only is None else list(only) headers = dict(sources(runtime)) for src in todo: - text = assembled_text(src) if src.suffix == '.py' else header_of(src, words_of(src)) + text = (assembled_text(src) if src.suffix == '.py' else bytes_header(src, runtime) if embedded(src) + else header_of(src, words_of(src))) header = headers[src] part = header.with_name(header.name + '.part') # whole or not at all: another build may be reading the folder part.write_text(text, encoding='utf-8', newline='\n') part.replace(header) - print(f'{header.name}: {text.count("0x")} words from {src.name}') + print(f'{header.name}: {src.stat().st_size} bytes of {src.name}' if embedded(src) + else f'{header.name}: {text.count("0x")} words from {src.name}') return len(todo) diff --git a/tools/make_appimage.py b/tools/make_appimage.py index 79991371..571dccd7 100644 --- a/tools/make_appimage.py +++ b/tools/make_appimage.py @@ -8,9 +8,10 @@ with --events and reads its progress lines). The AppImage holds no game files: the kit's tools/, recomp/, runtime/ and docs, the shader translator built by tools/build_xlate.py (bin/nx2navi_xlate), NVIDIA's DLSS library with its notice (bin/, tools/dlss_pin.py: downloaded -from NVIDIA's DLSS SDK repository and checked, or --dlss; not with --no-dlss), the pinned Vulkan Memory Allocator -header (an AppImage made without the library downloads it after a build on machines with NVIDIA's driver: AppRun's -get_dlss), and a relocatable CPython 3.14 (python-build-standalone, as `uv python install 3.14` puts in +from NVIDIA's DLSS SDK repository and checked, or --dlss; not with --no-dlss: an AppImage made without the library +downloads it after a build on machines with NVIDIA's driver, AppRun's get_dlss), the pinned Vulkan Memory Allocator +header, the pinned Vulkan headers (toolchain/vulkan-headers, tools/vulkan_pin.py: the runtime names Vulkan extensions +newer than the headers many distributions package), and a relocatable CPython 3.14 (python-build-standalone, as `uv python install 3.14` puts in ~/.local/share/uv/python) with numpy, lz4, zstandard, fonttools and pillow, and with --prompt-art the button art make_game.py's prompts step draws from (as make_kit.py --prompt-art: the pictures tools/prompt_art.py art_files names, into the kit's prompt-art folder; without it that step is skipped and the game keeps the Switch prompts). zbic, the GPL @@ -18,7 +19,9 @@ package that unpacks the game's program files, is not bundled: the AppImage inst ~/.local/share/nx2navi/pydeps the first time it runs, as the Windows launcher downloads it. The compiler and the libraries come from the system (README-BuildKit.md, "Linux"): the AppImage checks for clang, -clang++, lld and the headers before it starts, and names the packages for Arch, Fedora and Debian/Ubuntu. +clang++, lld and the headers before it starts, and names the packages for Arch, Fedora and Debian/Ubuntu. Whether +they are new enough is found out by trying them: tools/make_game.py checks the runtime's sources with that compiler +and links against those libraries before the build's long steps (preflight). """ import argparse import os @@ -72,7 +75,8 @@ ev step deps '"state":"start"' missing=() for tool in clang clang++ ld.lld; do command -v "$tool" >/dev/null || missing+=("$tool"); done -for h in SDL3/SDL.h vulkan/vulkan.h opus/opus.h vpx/vpx_decoder.h dav1d/dav1d.h zstd.h; do +# The Vulkan headers come with this file (the kit's toolchain/vulkan-headers): a distribution's may be too old. +for h in SDL3/SDL.h opus/opus.h vpx/vpx_decoder.h dav1d/dav1d.h zstd.h; do [ -e "/usr/include/$h" ] || [ -e "/usr/local/include/$h" ] || missing+=("$h") done if [ ${#missing[@]} -gt 0 ]; then @@ -226,6 +230,10 @@ def main(): dlss_pin.place(dlss, kit / 'bin', linux=True) (kit / 'toolchain' / 'vma').mkdir(parents=True) shutil.copy2(ROOT / 'toolchain' / 'vma' / 'vk_mem_alloc.h', kit / 'toolchain' / 'vma' / 'vk_mem_alloc.h') + # The Vulkan headers the runtime is written against, ahead of the system's on the player's machine + # (tools/build_game.py vulkan_include finds them in the kit): fetched and checked here when not there yet + import vulkan_pin + shutil.copytree(vulkan_pin.resolve(ROOT / vulkan_pin.FOLDER), kit / vulkan_pin.FOLDER) if a.prompt_art: import prompt_art n, size = prompt_art.copy_art(a.prompt_art, kit / prompt_art.FOLDER) diff --git a/tools/make_game.py b/tools/make_game.py index 49cecb37..542b5b67 100644 --- a/tools/make_game.py +++ b/tools/make_game.py @@ -922,6 +922,81 @@ def compile_all(srcs, cmd_for, cache, jobs, env, progress=None, salt='', stop=No return len(todo), reused, bad +def report_failures(bad): + """The files that did not compile, to stderr: the compiler's errors of the first three, and the names of all of + them, so that a paste of the build's end says which they were.""" + for src, err in bad[:3]: + print(f'--- {src}\n{err}', file=sys.stderr) + print('did not compile: ' + ', '.join(Path(src).name for src, _ in bad), file=sys.stderr, flush=True) + + +def runtime_headers(rt): + """The headers tools/build_shaders.py builds into runtime/ (the runtime's own shaders, the menu font's bytes): in + the repository as source only, so from a source checkout they are not there until they are built, and every + runtime file that includes one would fail to compile. A kit carries them built; a release's kit carries neither + tools/build_shaders.py nor the compilers it calls, and skips this.""" + try: + import build_shaders + except ModuleNotFoundError: + return + stale = build_shaders.stale(rt) + if stale: + say(f' building {len(stale)} of the runtime\'s own headers (runtime/*_spv.h, the menu font)') + build_shaders.build(rt, stale) + + +def preflight(msys, jobs, sources=None): + """Linux, before the long steps of a build whose compile step will run: this machine's compiler, headers and + libraries are tried on the runtime itself. Every runtime source that needs no header of this install is checked + with the compile step's own command (clang -fsyntax-only), and an empty program is linked against the libraries + the game links. The build takes the compiler, the linker and the libraries from the system (only the Vulkan + headers and VMA come with the kit), and a distribution's older compiler, older headers or a missing development + package stopped the compile step only after the RomFS image and the lift, ten minutes and more into the build. + sources: the files to check instead (tests).""" + clang, clangxx, env, vma = toolchain(msys) + version = clang_version(clang, env) + say(f'check: the compiler is {version}') + rt = KIT / 'runtime' + runtime_headers(rt) + inc = ['-I', str(rt), '-I', str(rt / 'third_party' / 'imgui'), '-I', str(vma)] + vulkan = vulkan_headers(clang, env) + cmd_for = runtime_cmd(clang, clangxx, inc, first=vulkan) + if sources is None: + tainted = per_install_names() + sources = [p for p, _ in runtime_sources() if not per_install(p, tainted)] + + def check(src): + cmd = [c for c in cmd_for(src, Path('unused.o')) if c not in ('-c', '-o', 'unused.o')] + p = subprocess.run([cmd[0], '-fsyntax-only', *cmd[1:]], capture_output=True, text=True, env=env, errors='replace') + return src, p.returncode, p.stderr[-1500:] + + t0 = time.time() + with concurrent.futures.ThreadPoolExecutor(max(1, jobs)) as ex: + bad = [(src, err) for src, rc, err in ex.map(check, sources) if rc] + if bad: + report_failures(bad) + lacking = sorted(set(re.findall(r"fatal error: '([^']+)' file not found", ''.join(e for _, e in bad)))) + fail(f'{len(bad)} of the runtime\'s {len(sources)} files do not compile on this machine, so the build would stop ' + f'in its compile step. ' + + (f'These headers are missing: {", ".join(lacking)}. Install the development packages ' + f'README-BuildKit.md lists under "Linux".' if lacking else + f'The compiler ({version}) or a library\'s headers are older than the runtime needs: the releases are ' + f'built with clang 23, and the Vulkan headers come ' + + (f'from {vulkan[1]}' if vulkan else 'from the system') + '. The errors above name what is missing.')) + with tempfile.TemporaryDirectory(prefix='nx2navi-check-') as tmp: + probe = Path(tmp) / 'probe.c' + probe.write_text('int main(void) { return 0; }\n', encoding='ascii') + ld = (['-fuse-ld=lld'] if shutil.which('ld.lld') else []) + ['-no-pie'] + p = subprocess.run([str(clangxx), *ld, '-o', str(Path(tmp) / 'probe'), '-x', 'c', str(probe), '-x', 'none', + *LINK_LIBS], capture_output=True, text=True, env=env, errors='replace') + if p.returncode: + print(p.stderr[-2000:], file=sys.stderr) + fail('the libraries the game links are not all there (the linker\'s lines above name the missing ones): ' + 'install the development packages README-BuildKit.md lists under "Linux". ' + 'On Fedora the static C++ library is its own package, libstdc++-static.') + say(f'check: {len(sources)} runtime files and the libraries pass, {time.time() - t0:.0f} s') + + def runtime_sources(): """(source, object name) of every runtime file this OS's build compiles: the Win32 files on Windows, the SDL window's (*_sdl.*) and platform/linux.c elsewhere (tools/build_game.py's foreign_os).""" @@ -1023,8 +1098,10 @@ def fast_clang(msys, clang, env): return fast if mine and clang_ident(fast, env) == mine else None -def runtime_cmd(clang, clangxx, inc, level=0, fast=None, msys=None): - """fast: fast_clang's compiler for the lifted code, with msys as its sysroot.""" +def runtime_cmd(clang, clangxx, inc, level=0, fast=None, msys=None, first=()): + """fast: fast_clang's compiler for the lifted code, with msys as its sysroot. first: include flags that go + ahead of inc for the runtime's sources only (vulkan_headers); the lifted code includes none of those headers, + and its command, which is part of its objects' keys, stays as it was.""" def cmd_for(src, dst): cpp = src.suffix in ('.cpp', '.cxx') # the lifted code at -O1 (its size makes -O2 slow to build), the runtime at -O2; neither may assume strict @@ -1037,7 +1114,7 @@ def runtime_cmd(clang, clangxx, inc, level=0, fast=None, msys=None): # -fdeclspec: third_party/ffx_api/ffx_api.h declares its entry points __declspec(dllexport) unconditionally declspec = ['-fdeclspec'] if cpp and not WIN else [] return [*cc, '-c', '-std=gnu++17' if cpp else '-std=gnu11', '-O1' if guest else '-O2', - *march, '-fno-strict-aliasing', '-w', *declspec, *inc, '-o', str(dst), str(src)] + *march, '-fno-strict-aliasing', '-w', *declspec, *([] if guest else first), *inc, '-o', str(dst), str(src)] return cmd_for @@ -1056,12 +1133,12 @@ def make_runtime_lib(dest, msys, jobs, obj=None): inc = ['-I', str(rt), '-I', str(rt / 'third_party' / 'imgui'), '-I', str(vma)] cache = ObjectCache(obj / 'objects.json', {'': KIT}) t0 = time.time() - compiled, reused, bad = compile_all(srcs, runtime_cmd(clang, clangxx, inc), cache, jobs, env, + compiled, reused, bad = compile_all(srcs, runtime_cmd(clang, clangxx, inc, first=vulkan_headers(clang, env)), + cache, jobs, env, lambda d, t: (d % 20 == 0 or d == t) and say(f' {d} of {t}'), salt=compiler_id(clang, env)) if bad: - for src, err in bad[:3]: - print(f'--- {src}\n{err}', file=sys.stderr) + report_failures(bad) fail(f'{len(bad)} runtime files did not compile') lib = dest / RUNTIME_LIB lib.unlink(missing_ok=True) @@ -1206,6 +1283,21 @@ def toolchain(msys): return Path(shutil.which('clang')), Path(shutil.which('clang++')), dict(os.environ), vma +def vulkan_headers(clang, env): + """The include flags that put the kit's Vulkan headers ahead of the system's for the runtime's sources: the + pinned release in /toolchain/vulkan-headers (tools/vulkan_pin.py), which the AppImage's kit carries and a + source checkout fetches when the system's headers are older (tools/build_game.py vulkan_include). None on + Windows, where MSYS2 supplies the headers, and none where the system's headers are new enough.""" + if WIN: + return [] + try: + from build_game import vulkan_include + folder = vulkan_include(clang, env) + except ImportError: # a kit without tools/vulkan_pin.py: the system's headers, as before it existed + return [] + return ['-I', str(folder)] if folder else [] + + def xlate_source(): """The folder of the shader translator to copy beside the game: the kit's bin/, else a source checkout's build/xlate/ (tools/build_xlate.py).""" @@ -1261,17 +1353,7 @@ def step_compile(out, cache_dir, msys, jobs, romfs_job=None, update=False): gen.mkdir(parents=True, exist_ok=True) obj.mkdir(parents=True, exist_ok=True) rt = KIT / 'runtime' - # The runtime's own shaders are in the repository as source only: from a source checkout their headers are not - # there until they are built, and every runtime file that includes one would fail to compile. A kit carries them - # built and does not carry tools/build_shaders.py (nor the compilers it calls), so a kit build skips this step. - try: - import build_shaders - except ModuleNotFoundError: - build_shaders = None - stale = build_shaders.stale(rt) if build_shaders else [] - if stale: - say(f' compiling {len(stale)} of the runtime\'s own shaders (runtime/*_spv.h)') - build_shaders.build(rt, stale) + runtime_headers(rt) clang, clangxx, env, vma = toolchain(msys) if not romfs_job: romfs_index(out, gen) @@ -1298,15 +1380,23 @@ def step_compile(out, cache_dir, msys, jobs, romfs_job=None, update=False): fast = fast_clang(msys, clang, env) if fast: say(f' the lifted code compiles with {fast}') - cmd_for = runtime_cmd(clang, clangxx, inc, level, fast, msys) + vulkan = vulkan_headers(clang, env) + if vulkan: + say(f' the Vulkan headers come from {vulkan[1]}') + cmd_for = runtime_cmd(clang, clangxx, inc, level, fast, msys, first=vulkan) # The keys earlier versions gave the same compiles (compile_all alts): 0.6.0's, with the compiler's file time and, - # with the fast compiler, its id and --sysroot; 0.5.0's and earlier's, with their kit's folder in the paths. + # with the fast compiler, its id and --sysroot; 0.5.0's and earlier's, with their kit's folder in the paths; and + # up to 0.7.1 the runtime's without the Vulkan headers' folder ahead of the system's (an object that compiled + # against the system's headers is as good). plain = runtime_cmd(clang, clangxx, inc, level, None, msys) + no_vulkan = runtime_cmd(clang, clangxx, inc, level, fast, msys) fast_old = (cid_old + compiler_ids(fast, env)[1]) if fast else None kits = earlier_kits() def alts(src, dst): keys = [object_key(plain(src, dst), src, dst, cache, cid_old, sysroot=True)] + if vulkan: + keys.append(object_key(no_vulkan(src, dst), src, dst, cache, cid)) if fast: keys.append(object_key(cmd_for(src, dst), src, dst, cache, fast_old, sysroot=True)) keys += [legacy_key(src, dst, cache, cid_old, clang, clangxx, k, lift, gen, vma) for k in kits] @@ -1344,8 +1434,7 @@ def step_compile(out, cache_dir, msys, jobs, romfs_job=None, update=False): compiled, reused = compiled + c2, reused + r2 say(f' {compiled} compiled, {reused} unchanged since the last build') if bad: - for src, err in bad[:3]: - print(f'--- {src}\n{err}', file=sys.stderr) + report_failures(bad) fail(f'{len(bad)} files did not compile') if cache.keep_only({o.name for _, o in srcs}): cache.save() @@ -1891,6 +1980,10 @@ def main(): # the compile step's key does not name the level: without this a finished compile reads as done before state['steps'].pop('compile', None) keep_awake() + # Linux: the compiler, headers and libraries are the system's, so they are tried on the runtime before the long + # steps of a build that will reach the compile step (the first step whose key is not current makes the rest run) + if not WIN and any(state['steps'].get(n) != step_key(n, src.identity, out) for n in STEPS[:STEPS.index('compile') + 1]): + preflight(msys, a.jobs) t0 = time.time() romfs_job = None runs = {'exefs': lambda: step_exefs(src, out), 'romfs': lambda: step_romfs(src, out, min(a.jobs, 8)), diff --git a/tools/make_kit.py b/tools/make_kit.py index 14472176..6c77b9cb 100644 --- a/tools/make_kit.py +++ b/tools/make_kit.py @@ -30,7 +30,8 @@ NAME = 'NX2navi-BuildKit' TOOLS = ('make_game.py', 'nup.py', 'romfs_index.py', 'check_content.py', 'hostmem.py', 'prepare_shaders.py', 'shader_census.py', 'shader_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py', 'romfs_check.py', 'build_game.py', 'check_isa.py', 'build_shaders.py', 'make_dlaa_mask.py', 'make_mvobj_fs.py', - 'pack_shader_cache.py') # make_game.py imports build_game's vma_include on Linux for the VMA header, and + 'pack_shader_cache.py', 'vulkan_pin.py') # make_game.py imports build_game's vma_include and vulkan_include on +# Linux for the VMA header and for Vulkan headers newer than the system's (which build_game takes from vulkan_pin), and # build_game.py imports check_isa and build_shaders, which reads the two make_ scripts' shaders TOP = ('LICENSE', 'LICENSE-ADDITIONS', 'THIRD_PARTY_NOTICES.md', 'VERSION', 'nx2navi.ini.example') SKIP_DIRS = {'__pycache__', '.pytest_cache'} diff --git a/tools/make_release_linux.py b/tools/make_release_linux.py index 2d476b7e..90a50710 100644 --- a/tools/make_release_linux.py +++ b/tools/make_release_linux.py @@ -40,6 +40,7 @@ from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) import build_game # noqa: E402 import make_release as mr # noqa: E402 +import vulkan_pin # noqa: E402 ROOT = mr.ROOT MAIN_ROOT = mr.MAIN_ROOT @@ -69,8 +70,7 @@ PINS = { '5749cbc4e668273514150a80e387a957f933c6ed3f5f11e03fb30955e2bbead6'), 'zstd-src': ('https://github.com/facebook/zstd/releases/download/v1.5.7/zstd-1.5.7.tar.gz', 'eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3'), - 'vulkan-headers-src': ('https://github.com/KhronosGroup/Vulkan-Headers/archive/refs/tags/v1.4.363.tar.gz', - 'cbaf687d3c59b9666fe080e7f8396b6b4f4d344a768ee6b337c59852f4526b68'), + 'vulkan-headers-src': vulkan_pin.SOURCE, # the release a build with the system's headers takes too 'vma-header': build_game.VMA_FETCH, # appimagetool and the type-2 runtime from release tags, not "continuous", whose uploads change under the same # URL and would stop the build at the hash check @@ -169,8 +169,8 @@ def build_deps(out, llvm, sysroot): d = out / 'runtime' / 'deps' (d / 'include' / 'vma').mkdir(parents=True, exist_ok=True) (d / 'lib').mkdir(parents=True, exist_ok=True) - vk = tar_extract('vulkan-headers-src', WORK / 'vulkan-headers', ['Vulkan-Headers-1.4.363/include', - 'Vulkan-Headers-1.4.363/LICENSE.md'], strip=1) + vk = tar_extract('vulkan-headers-src', WORK / 'vulkan-headers', [f'{vulkan_pin.TOP}include', + f'{vulkan_pin.TOP}LICENSE.md'], strip=1) for sub in ('vulkan', 'vk_video'): shutil.copytree(vk / 'include' / sub, d / 'include' / sub) shutil.copy2(mr.fetch('vma-header'), d / 'include' / 'vma' / 'vk_mem_alloc.h') diff --git a/tools/vulkan_pin.py b/tools/vulkan_pin.py new file mode 100644 index 00000000..673eb403 --- /dev/null +++ b/tools/vulkan_pin.py @@ -0,0 +1,109 @@ +"""The Vulkan headers the runtime is compiled against where the system's are older. + +The runtime names structures and constants of Vulkan extensions as they come out (runtime/vk_device.c asks for +VK_KHR_pipeline_binary, VK_KHR_dynamic_rendering_local_read and VK_KHR_shader_maximal_reconvergence when the driver +has them), so it needs headers at least as new as the ones it was written against. On Windows MSYS2 supplies them and +the Linux release (tools/make_release_linux.py) carries them in runtime/deps. A build with the system's own compiler +and headers (tools/make_appimage.py's kit, a source checkout) met distributions whose packaged headers are years +older: every file that names a newer extension then fails to compile. Such a build takes the headers from here +instead: the release the others use, from Khronos's Vulkan-Headers repository, checked against its SHA-256. + +Only the C headers are kept (include/vulkan/*.h and include/vk_video/*.h, which vulkan_core.h includes) with the +repository's licence note. The headers are Apache-2.0 OR MIT (THIRD_PARTY_NOTICES.md). Standard library only: the +build kit's Python runs this. + + python tools/vulkan_pin.py [--dest toolchain/vulkan-headers] fetch and unpack the pinned headers +""" +import argparse +import hashlib +import io +import re +import shutil +import sys +import tarfile +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +VERSION = '1.4.363' +HEADER_VERSION = 363 # VK_HEADER_VERSION of that release; the numbers only grow, across 1.3 and 1.4 alike +# (address, SHA-256): the archive GitHub makes of the release's tag +SOURCE = (f'https://github.com/KhronosGroup/Vulkan-Headers/archive/refs/tags/v{VERSION}.tar.gz', + 'cbaf687d3c59b9666fe080e7f8396b6b4f4d344a768ee6b337c59852f4526b68') +TOP = f'Vulkan-Headers-{VERSION}/' # the archive's top folder +FOLDER = Path('toolchain') / 'vulkan-headers' # below a checkout's or a kit's root: include/ and LICENSE.md +STAMP = 'pinned.txt' # in the folder: the release it holds, written last +KEEP = re.compile(r'include/(vulkan|vk_video)/[A-Za-z0-9_]+\.h|LICENSE\.md') + + +def have(folder): + """True when folder holds the pinned headers, complete (the stamp is written after the last file).""" + try: + return (folder / STAMP).read_text(encoding='ascii').strip() == VERSION and \ + (folder / 'include' / 'vulkan' / 'vulkan_core.h').is_file() + except OSError: + return False + + +def unpack(archive, folder): + """The C headers and the licence note of the release's archive (bytes) into folder, replacing what is there.""" + shutil.rmtree(folder, ignore_errors=True) + n = 0 + with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar: + for m in tar: + rel = m.name[len(TOP):] if m.name.startswith(TOP) else '' + if not m.isfile() or not KEEP.fullmatch(rel): # a name the pattern takes has no '..' and no other folder + continue + dst = folder / rel + dst.parent.mkdir(parents=True, exist_ok=True) + dst.write_bytes(tar.extractfile(m).read()) + n += 1 + if not (folder / 'include' / 'vulkan' / 'vulkan_core.h').is_file() or not (folder / 'LICENSE.md').is_file(): + shutil.rmtree(folder, ignore_errors=True) + sys.exit(f'vulkan_pin: the archive of Vulkan-Headers {VERSION} lacks vulkan_core.h or LICENSE.md') + (folder / STAMP).write_text(VERSION + '\n', encoding='ascii') + return n + + +def resolve(folder=None, archive=None): + """The folder holding the pinned headers (default /toolchain/vulkan-headers), fetched and unpacked there + when it does not hold them yet. archive: the release's archive on disk instead of the download; it must be the + pinned file either way. Returns the folder; its include/ goes on the include path.""" + folder = Path(folder) if folder else ROOT / FOLDER + if have(folder): + return folder + url, want = SOURCE + if archive: + data = Path(archive).read_bytes() + else: + print(f'vulkan_pin: fetching the Vulkan headers {VERSION} into {folder}', flush=True) + try: + data = urllib.request.urlopen(url, timeout=120).read() + except OSError as e: + sys.exit(f'vulkan_pin: fetching {url} failed ({e}). Install Vulkan headers {VERSION} or newer, or put the ' + f'archive on disk and run tools/vulkan_pin.py --archive ') + if hashlib.sha256(data).hexdigest() != want: + sys.exit(f'vulkan_pin: {archive or url} is not the pinned archive of Vulkan-Headers {VERSION} (SHA-256 differs)') + unpack(data, folder) + return folder + + +def header_version(text): + """VK_HEADER_VERSION as vulkan_core.h defines it in text, 0 when it does not.""" + m = re.search(r'^[ \t]*#[ \t]*define[ \t]+VK_HEADER_VERSION[ \t]+(\d+)', text, re.M) + return int(m.group(1)) if m else 0 + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument('--dest', type=Path, default=ROOT / FOLDER, help='where the headers go (default toolchain/vulkan-headers)') + ap.add_argument('--archive', type=Path, help=f'the archive of Vulkan-Headers {VERSION} on disk instead of the download') + a = ap.parse_args() + folder = resolve(a.dest, a.archive) + n = sum(1 for _ in (folder / 'include').rglob('*.h')) + print(f'{folder}: Vulkan headers {VERSION}, {n} files') + return 0 + + +if __name__ == '__main__': + sys.exit(main())