Merge the shader headers branch: the runtime keeps its shaders as source and builds the SPIR-V headers when the game is built.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
chrg6
2026-11-10 03:33:21 -05:00
29 changed files with 176 additions and 6193 deletions

View File

@@ -70,7 +70,7 @@ from pathlib import Path
import check_isa
import romfs_index
from build_shaders import stale as stale_shaders
import build_shaders
from check_content import GENERATED_MARKER
WIN = os.name == 'nt'
@@ -899,9 +899,7 @@ def main():
copy_romfs_patches(args.version, out)
global RUNTIME
RUNTIME = (args.runtime or RUNTIME).resolve()
stale = stale_shaders(RUNTIME)
if stale:
sys.exit(f'{stale[0]}: its SPIR-V header no longer matches; run python tools/build_shaders.py')
build_shaders.build(RUNTIME) # the runtime's own shaders: the headers the sources below include, when missing or stale
skip = {s for s in args.skip_runtime.split(',') if s}
main_out = (ROOT / 'build' / args.version).resolve()
pairs = None

View File

@@ -1,26 +1,41 @@
"""Compile the runtime's own GLSL shaders (runtime/shaders/*.comp, *.vert, *.frag) into C headers of SPIR-V words.
"""Build the runtime's own shaders into C headers of SPIR-V words.
Each runtime/shaders/<name>.comp (or .vert, .frag: one name per shader) becomes runtime/<name>_spv.h, which declares `static const uint32_t <name>_spv[]`
and records the SHA-256 of the source it came from. The headers are committed, so building nx2navi.exe needs no shader
compiler; tools/build_game.py refuses to build when a header no longer matches its source. Run this after editing a
shader. Needs glslc and spirv-val from MSYS2 UCRT64 (package mingw-w64-ucrt-x86_64-shaderc, spirv-tools).
The repository holds the shaders as source only. Each becomes runtime/<name>_spv.h, which declares
`static const uint32_t <name>_spv[]`; the headers are build outputs (ignored by git) and tools/build_game.py builds the
missing and the stale ones before it compiles the runtime. Three kinds of source:
python tools/build_shaders.py [--check]
runtime/shaders/<name>.comp, .vert, .frag GLSL, compiled with glslc (one name per shader) and checked with spirv-val
runtime/shaders/<name>.spvasm SPIR-V assembly, assembled with spirv-as and checked with spirv-val
tools/make_dlaa_mask.py, make_mvobj_fs.py shaders assembled in Python (ASSEMBLED); they need no other tool
A compiled or assembled header records the SHA-256 of its source and is stale when that no longer matches; a Python
one is stale when its text is not what its script gives. glslc, spirv-as and spirv-val come from MSYS2 UCRT64 on
Windows (packages mingw-w64-ucrt-x86_64-shaderc and mingw-w64-ucrt-x86_64-spirv-tools) and from PATH elsewhere
(shaderc, spirv-tools). A folder that carries its headers ready-made (the build kit, tools/make_kit.py) needs none of
them: nothing there is stale.
python tools/build_shaders.py [--check] [--all] [--runtime DIR]
"""
import argparse
import hashlib
import importlib.util
import os
import shutil
import subprocess
import sys
import tempfile
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
BIN = Path(r'C:\msys64\ucrt64\bin') # Windows; elsewhere glslc and spirv-val come from PATH
WIN = os.name == 'nt'
TOOLS = Path(__file__).resolve().parent
ROOT = TOOLS.parent
BIN = Path(r'C:\msys64\ucrt64\bin') # Windows; elsewhere, and without MSYS2 there, the tools come from PATH
TARGET = 'vulkan1.1'
STAGES = {'.comp': 'compute', '.vert': 'vertex', '.frag': 'fragment'}
ASM = '.spvasm'
ASM_TARGET = 'vulkan1.3'
# header name -> the script beside this one that assembles it; the script's HEADER is the header's text
ASSEMBLED = {'dlaa_mask': 'make_dlaa_mask.py', 'mvobj_fs': 'make_mvobj_fs.py'}
def source_hash(src):
@@ -29,7 +44,9 @@ def source_hash(src):
def header_of(src, words):
name = src.stem
lines = [f'// Generated by tools/build_shaders.py from runtime/shaders/{src.name}; do not edit.',
how = (f'Generated from runtime/shaders/{src.name} (spirv-as --target-env {ASM_TARGET})' if src.suffix == ASM
else f'Generated by tools/build_shaders.py from runtime/shaders/{src.name}')
lines = [f'// {how}; do not edit.',
f'// source sha256 {source_hash(src)}',
'#pragma once',
'#include <stdint.h>',
@@ -40,51 +57,104 @@ def header_of(src, words):
return '\n'.join(lines) + '\n'
def header_path(src):
return src.parent.parent / f'{src.stem}_spv.h'
def sources(runtime):
return sorted(p for p in (runtime / 'shaders').iterdir() if p.suffix in STAGES)
"""Every shader source with the header it becomes: the files of runtime/shaders and the scripts of ASSEMBLED that
are there (a folder without them carries the headers ready-made)."""
shaders = runtime / 'shaders'
found = sorted(p for p in shaders.iterdir() if p.suffix in STAGES or p.suffix == ASM) if shaders.is_dir() else []
out = [(p, runtime / f'{p.stem}_spv.h') for p in found]
out += [(TOOLS / script, runtime / f'{name}_spv.h') for name, script in ASSEMBLED.items() if (TOOLS / script).is_file()]
return out
def assembled_text(script):
spec = importlib.util.spec_from_file_location(script.stem, script)
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod.HEADER
def read_text(header):
return header.read_bytes().decode('utf-8').replace('\r\n', '\n') if header.is_file() else None
def stale(runtime):
"""Shader sources under runtime/shaders whose header is missing or records another source hash."""
"""The shader sources whose header is missing or no longer comes from them."""
out = []
for src in sources(runtime):
h = header_path(src)
if not h.exists() or f'// source sha256 {source_hash(src)}' not in h.read_text(encoding='utf-8'):
for src, header in sources(runtime):
text = read_text(header)
if text is None:
out.append(src)
elif src.suffix == '.py':
if text != assembled_text(src):
out.append(src)
elif f'// source sha256 {source_hash(src)}' not in text:
out.append(src)
return out
def compile_one(src):
# "// build_shaders: keep functions" in a source: no optimisation, so its functions stay functions (the HDR tonemap,
# which runtime/vk_hdr.c splices into the game's shader by function)
opt = [] if '// build_shaders: keep functions' in src.read_text(encoding='utf-8') else ['-O']
def tool(name):
"""The path of glslc, spirv-as or spirv-val; ends the run with the packages to install when it is not there."""
exe = BIN / f'{name}.exe' if os.name == 'nt' else None
if exe and exe.is_file():
return str(exe)
found = shutil.which(name)
if found:
return found
where = (f'MSYS2 UCRT64 ({BIN}): pacman -S mingw-w64-ucrt-x86_64-shaderc mingw-w64-ucrt-x86_64-spirv-tools'
if os.name == 'nt' else "PATH: install your distribution's shaderc and spirv-tools packages")
sys.exit(f'build_shaders: {name} not found. The runtime\'s shaders are compiled when the game is built '
f'(docs/building.md); {name} comes from {where}')
def words_of(src):
with tempfile.TemporaryDirectory() as tmp:
spv = Path(tmp) / 'out.spv'
subprocess.run([str(BIN / 'glslc.exe') if WIN else 'glslc', *opt, f'--target-env={TARGET}', f'-fshader-stage={STAGES[src.suffix]}',
'-o', str(spv), str(src)], check=True)
subprocess.run([str(BIN / 'spirv-val.exe') if WIN else 'spirv-val', f'--target-env', TARGET, str(spv)], check=True)
if src.suffix == ASM:
target = ASM_TARGET
subprocess.run([tool('spirv-as'), '--target-env', target, '-o', str(spv), str(src)], check=True)
else:
# "// build_shaders: keep functions" in a source: no optimisation, so its functions stay functions (the HDR
# tonemap, which runtime/vk_hdr.c splices into the game's shader by function)
target = TARGET
opt = [] if '// build_shaders: keep functions' in src.read_text(encoding='utf-8') else ['-O']
subprocess.run([tool('glslc'), *opt, f'--target-env={target}', f'-fshader-stage={STAGES[src.suffix]}',
'-o', str(spv), str(src)], check=True)
subprocess.run([tool('spirv-val'), '--target-env', target, str(spv)], check=True)
data = spv.read_bytes()
words = [int.from_bytes(data[i:i + 4], 'little') for i in range(0, len(data), 4)]
header_path(src).write_text(header_of(src, words), encoding='utf-8', newline='\n')
print(f'{header_path(src).relative_to(ROOT)}: {len(words)} words')
return [int.from_bytes(data[i:i + 4], 'little') for i in range(0, len(data), 4)]
def build(runtime, only=None):
"""Writes the headers of the sources in only (default: the stale ones) and returns how many it wrote."""
todo = stale(runtime) if only is None else list(only)
headers = dict(sources(runtime))
for src in todo:
text = assembled_text(src) if src.suffix == '.py' else header_of(src, words_of(src))
header = headers[src]
part = header.with_name(header.name + '.part') # whole or not at all: another build may be reading the folder
part.write_text(text, encoding='utf-8', newline='\n')
part.replace(header)
print(f'{header.name}: {text.count("0x")} words from {src.name}')
return len(todo)
def main():
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
ap.add_argument('--check', action='store_true', help='only report headers that do not match their source')
ap.add_argument('--check', action='store_true', help='only report headers that are missing or do not match their source')
ap.add_argument('--all', action='store_true', help='build every header, not only the missing and the stale ones')
ap.add_argument('--runtime', type=Path, default=ROOT / 'runtime')
args = ap.parse_args()
runtime = args.runtime.resolve()
if args.check:
bad = stale(args.runtime)
bad = stale(runtime)
headers = dict(sources(runtime))
for s in bad:
print(f'{header_path(s)}: stale, run tools/build_shaders.py')
print(f'{headers[s]}: missing or stale, run tools/build_shaders.py')
return 1 if bad else 0
for src in sources(args.runtime):
compile_one(src)
n = build(runtime, [s for s, _ in sources(runtime)] if args.all else None)
if not n:
print('build_shaders: every header matches its source')
return 0

View File

@@ -1,4 +1,4 @@
"""Refuse files that look like game content, keys or generated game code.
"""Refuse files that look like game content, keys, generated game code or compiled shaders.
Runs in CI on every push and pull request, and locally as the pre-commit hook
(.githooks/pre-commit). With no arguments it checks every tracked file; with
@@ -83,6 +83,14 @@ def problems_for(path: Path, allow_large: bool = False, data: bytes | None = Non
return found
def compiled_shader(path: Path) -> bool:
"""A SPIR-V binary or a header of SPIR-V words. The repository holds its shaders as source (runtime/shaders/, the
tools/make_*.py assemblers) and tools/build_shaders.py builds runtime/<name>_spv.h from them when the game is
built. A build kit or a release folder carries the headers, so make_kit.py and make_release.py, which call
problems_for, are not held to this."""
return path.suffix.lower() == ".spv" or path.name.lower().endswith("_spv.h")
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("--staged", action="store_true", help="check staged files only")
@@ -92,11 +100,14 @@ def main() -> int:
for path in list_files(args.staged):
# --staged checks what the commit takes: a marker staged and then removed from the working copy alone
# passed when the file on disk was read
for problem in problems_for(path, data=staged_bytes(path) if args.staged else None):
problems = problems_for(path, data=staged_bytes(path) if args.staged else None)
if compiled_shader(path):
problems.append("compiled shader: the repository holds the source, tools/build_shaders.py builds this")
for problem in problems:
print(f"{path}: {problem}")
failed = True
if failed:
print("\nGame content, keys and generated game code never go into the repository.")
print("\nGame content, keys, generated game code and compiled shaders never go into the repository.")
print("See CONTRIBUTING.md.")
return 1
return 0

View File

@@ -145,6 +145,9 @@ def main():
shutil.rmtree(app, ignore_errors=True)
kit = app / 'usr' / 'kit'
kit.mkdir(parents=True)
sys.path.insert(0, str(ROOT / 'tools'))
import build_shaders
build_shaders.build(ROOT / 'runtime') # the image carries the runtime's shaders compiled (runtime/*_spv.h)
for d in KIT_DIRS:
copy_tree(ROOT / d, kit / d)
copy_tree(ROOT / 'tools', kit / 'tools', TOOLS_SKIP)

View File

@@ -1,5 +1,6 @@
"""Writes runtime/dlaa_mask_spv.h: the SPIR-V of DLSS's reactive mask pass (vk_dlaa.c, mask_pass), assembled here like
tools/make_mvobj_fs.py (no GLSL compiler on the machine this was written on).
"""The SPIR-V of DLSS's reactive mask pass (vk_dlaa.c, mask_pass) as runtime/dlaa_mask_spv.h, assembled here like
tools/make_mvobj_fs.py. tools/build_shaders.py writes the header from HEADER when the game is built; run on its own,
this script writes it too.
The shader, as GLSL:
@@ -196,8 +197,11 @@ a.op(F, OpFunctionEnd)
w = a.words()
lines = [', '.join(f'0x{x:08x}' for x in w[i:i + 8]) for i in range(0, len(w), 8)]
OUT.write_text(
HEADER = (
'// Generated by tools/make_dlaa_mask.py: DLSS\'s reactive mask pass (vk_dlaa.c mask_pass). Do not edit.\n'
'#pragma once\n#include <stdint.h>\n'
'static const uint32_t dlaa_mask_spv[] = {\n ' + ',\n '.join(lines) + '\n};\n', newline='\n')
print(f'{OUT}: {len(w)} words')
'static const uint32_t dlaa_mask_spv[] = {\n ' + ',\n '.join(lines) + '\n};\n')
if __name__ == '__main__':
OUT.write_text(HEADER, newline='\n')
print(f'{OUT}: {len(w)} words')

View File

@@ -4,7 +4,9 @@
[--dlss nvngx_dlss.dll | --no-dlss] [--dlssg nx2navi_dlssg.dll [--streamline SDK]]
[--prompt-art FOLDER [FOLDER ...]] [--out build/kit] [--zip]
The kit holds this repository's runtime/, recomp/, the build tools, the shader translator built from tools/xlate,
The kit holds this repository's runtime/ with its shaders compiled (tools/build_shaders.py builds the SPIR-V headers
here, so a player's PC needs no shader compiler; their sources go along), recomp/, the build tools, the shader
translator built from tools/xlate,
NVIDIA's DLSS library with its notice (tools/dlss_pin.py: downloaded from NVIDIA's DLSS SDK repository and checked,
or --dlss; not with --no-dlss), DLSS Frame Generation in bin/dlssg/ with --dlssg (nx2navi_dlssg.dll from
tools/build_dlssg.py beside NVIDIA Streamline's runtime, tools/streamline_pin.py: downloaded from NVIDIA's Streamline
@@ -27,8 +29,9 @@ ROOT = Path(__file__).resolve().parents[1]
NAME = 'NX2navi-BuildKit'
TOOLS = ('make_game.py', 'nup.py', 'romfs_index.py', 'check_content.py', 'hostmem.py', 'prepare_shaders.py',
'shader_census.py', 'shader_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py', 'romfs_check.py',
'build_game.py', 'check_isa.py', 'build_shaders.py', 'pack_shader_cache.py') # make_game.py imports build_game's vma_include on
# Linux for the VMA header, and build_game.py imports check_isa and build_shaders
'build_game.py', 'check_isa.py', 'build_shaders.py', 'make_dlaa_mask.py', 'make_mvobj_fs.py',
'pack_shader_cache.py') # make_game.py imports build_game's vma_include on Linux for the VMA header, and
# build_game.py imports check_isa and build_shaders, which reads the two make_ scripts' shaders
TOP = ('LICENSE', 'LICENSE-ADDITIONS', 'THIRD_PARTY_NOTICES.md', 'VERSION', 'nx2navi.ini.example')
SKIP_DIRS = {'__pycache__', '.pytest_cache'}
SKIP_SUFFIXES = {'.o', '.obj', '.pyc', '.exe', '.dll', '.a', '.lib'}
@@ -60,6 +63,7 @@ def main():
ap.add_argument('--zip', action='store_true')
a = ap.parse_args()
sys.path.insert(0, str(ROOT / 'tools'))
import build_shaders
import dlss_pin
import prompt_art
import streamline_pin
@@ -73,6 +77,7 @@ def main():
if kit.exists():
shutil.rmtree(kit)
kit.mkdir(parents=True)
build_shaders.build(ROOT / 'runtime') # the kit carries the shader headers: build the missing and the stale ones
for d in ('runtime', 'recomp'):
copy_tree(ROOT / d, kit / d)
(kit / 'tools').mkdir()

View File

@@ -57,6 +57,7 @@ from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
sys.path.insert(0, str(ROOT / 'tools'))
import build_shaders # noqa: E402
import check_content # noqa: E402
import dlss_pin # noqa: E402
import prompt_art # noqa: E402
@@ -177,6 +178,9 @@ def build_python(py):
# ---------------------------------------------------------------- app/<version>/
def build_app(app, ver, xlate, feed_repo, lock, ffx=None, dlss=None, art=None, dlssg=None):
# The runtime's own shaders are source in the repository and SPIR-V headers in a release: a player's PC compiles
# runtime/ (make_game.py) and has no shader compiler. Build the missing and the stale ones before the copy.
build_shaders.build(ROOT / 'runtime')
copy_tree(ROOT / 'launcher', app / 'launcher')
for d in ('runtime', 'recomp'):
copy_tree(ROOT / d, app / d)

View File

@@ -1,5 +1,6 @@
"""Writes runtime/vk_mvobj_fs.h: the SPIR-V of the per-object motion fragment shader (vk_mvobj.c), assembled here
because it must be patched at run time (the Location of its input follows the vertex shader variant) and is small.
"""The SPIR-V of the per-object motion fragment shader (vk_mvobj.c) as runtime/mvobj_fs_spv.h, assembled here because
it must be patched at run time (the Location of its input follows the vertex shader variant) and is small.
tools/build_shaders.py writes the header from HEADER when the game is built; run on its own, this script writes it too.
The shader, as GLSL:
@@ -24,7 +25,7 @@ import struct
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
OUT = ROOT / 'runtime' / 'vk_mvobj_fs.h'
OUT = ROOT / 'runtime' / 'mvobj_fs_spv.h'
class Asm:
@@ -156,8 +157,11 @@ a.op(F, OpFunctionEnd)
w = a.words()
lines = [', '.join(f'0x{x:08x}' for x in w[i:i + 8]) for i in range(0, len(w), 8)]
OUT.write_text(
HEADER = (
'// Generated by tools/make_mvobj_fs.py: the per-object motion fragment shader (vk_mvobj.c). Do not edit.\n'
f'#define MVOBJ_FS_LOCATION_WORD {a.loc_word} // the input Location literal, patched to the variant\'s\n'
'static const uint32_t mvobj_fs_spv[] = {\n ' + ',\n '.join(lines) + '\n};\n', newline='\n')
print(f'{OUT}: {len(w)} words, Location at word {a.loc_word}')
'static const uint32_t mvobj_fs_spv[] = {\n ' + ',\n '.join(lines) + '\n};\n')
if __name__ == '__main__':
OUT.write_text(HEADER, newline='\n')
print(f'{OUT}: {len(w)} words, Location at word {a.loc_word}')