mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-17 15:49:57 -04:00
Merge the shader headers branch: the runtime keeps its shaders as source and builds the SPIR-V headers when the game is built.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -70,7 +70,7 @@ from pathlib import Path
|
||||
|
||||
import check_isa
|
||||
import romfs_index
|
||||
from build_shaders import stale as stale_shaders
|
||||
import build_shaders
|
||||
from check_content import GENERATED_MARKER
|
||||
|
||||
WIN = os.name == 'nt'
|
||||
@@ -899,9 +899,7 @@ def main():
|
||||
copy_romfs_patches(args.version, out)
|
||||
global RUNTIME
|
||||
RUNTIME = (args.runtime or RUNTIME).resolve()
|
||||
stale = stale_shaders(RUNTIME)
|
||||
if stale:
|
||||
sys.exit(f'{stale[0]}: its SPIR-V header no longer matches; run python tools/build_shaders.py')
|
||||
build_shaders.build(RUNTIME) # the runtime's own shaders: the headers the sources below include, when missing or stale
|
||||
skip = {s for s in args.skip_runtime.split(',') if s}
|
||||
main_out = (ROOT / 'build' / args.version).resolve()
|
||||
pairs = None
|
||||
|
||||
@@ -1,26 +1,41 @@
|
||||
"""Compile the runtime's own GLSL shaders (runtime/shaders/*.comp, *.vert, *.frag) into C headers of SPIR-V words.
|
||||
"""Build the runtime's own shaders into C headers of SPIR-V words.
|
||||
|
||||
Each runtime/shaders/<name>.comp (or .vert, .frag: one name per shader) becomes runtime/<name>_spv.h, which declares `static const uint32_t <name>_spv[]`
|
||||
and records the SHA-256 of the source it came from. The headers are committed, so building nx2navi.exe needs no shader
|
||||
compiler; tools/build_game.py refuses to build when a header no longer matches its source. Run this after editing a
|
||||
shader. Needs glslc and spirv-val from MSYS2 UCRT64 (package mingw-w64-ucrt-x86_64-shaderc, spirv-tools).
|
||||
The repository holds the shaders as source only. Each becomes runtime/<name>_spv.h, which declares
|
||||
`static const uint32_t <name>_spv[]`; the headers are build outputs (ignored by git) and tools/build_game.py builds the
|
||||
missing and the stale ones before it compiles the runtime. Three kinds of source:
|
||||
|
||||
python tools/build_shaders.py [--check]
|
||||
runtime/shaders/<name>.comp, .vert, .frag GLSL, compiled with glslc (one name per shader) and checked with spirv-val
|
||||
runtime/shaders/<name>.spvasm SPIR-V assembly, assembled with spirv-as and checked with spirv-val
|
||||
tools/make_dlaa_mask.py, make_mvobj_fs.py shaders assembled in Python (ASSEMBLED); they need no other tool
|
||||
|
||||
A compiled or assembled header records the SHA-256 of its source and is stale when that no longer matches; a Python
|
||||
one is stale when its text is not what its script gives. glslc, spirv-as and spirv-val come from MSYS2 UCRT64 on
|
||||
Windows (packages mingw-w64-ucrt-x86_64-shaderc and mingw-w64-ucrt-x86_64-spirv-tools) and from PATH elsewhere
|
||||
(shaderc, spirv-tools). A folder that carries its headers ready-made (the build kit, tools/make_kit.py) needs none of
|
||||
them: nothing there is stale.
|
||||
|
||||
python tools/build_shaders.py [--check] [--all] [--runtime DIR]
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
BIN = Path(r'C:\msys64\ucrt64\bin') # Windows; elsewhere glslc and spirv-val come from PATH
|
||||
WIN = os.name == 'nt'
|
||||
TOOLS = Path(__file__).resolve().parent
|
||||
ROOT = TOOLS.parent
|
||||
BIN = Path(r'C:\msys64\ucrt64\bin') # Windows; elsewhere, and without MSYS2 there, the tools come from PATH
|
||||
TARGET = 'vulkan1.1'
|
||||
STAGES = {'.comp': 'compute', '.vert': 'vertex', '.frag': 'fragment'}
|
||||
ASM = '.spvasm'
|
||||
ASM_TARGET = 'vulkan1.3'
|
||||
# header name -> the script beside this one that assembles it; the script's HEADER is the header's text
|
||||
ASSEMBLED = {'dlaa_mask': 'make_dlaa_mask.py', 'mvobj_fs': 'make_mvobj_fs.py'}
|
||||
|
||||
|
||||
def source_hash(src):
|
||||
@@ -29,7 +44,9 @@ def source_hash(src):
|
||||
|
||||
def header_of(src, words):
|
||||
name = src.stem
|
||||
lines = [f'// Generated by tools/build_shaders.py from runtime/shaders/{src.name}; do not edit.',
|
||||
how = (f'Generated from runtime/shaders/{src.name} (spirv-as --target-env {ASM_TARGET})' if src.suffix == ASM
|
||||
else f'Generated by tools/build_shaders.py from runtime/shaders/{src.name}')
|
||||
lines = [f'// {how}; do not edit.',
|
||||
f'// source sha256 {source_hash(src)}',
|
||||
'#pragma once',
|
||||
'#include <stdint.h>',
|
||||
@@ -40,51 +57,104 @@ def header_of(src, words):
|
||||
return '\n'.join(lines) + '\n'
|
||||
|
||||
|
||||
def header_path(src):
|
||||
return src.parent.parent / f'{src.stem}_spv.h'
|
||||
|
||||
|
||||
def sources(runtime):
|
||||
return sorted(p for p in (runtime / 'shaders').iterdir() if p.suffix in STAGES)
|
||||
"""Every shader source with the header it becomes: the files of runtime/shaders and the scripts of ASSEMBLED that
|
||||
are there (a folder without them carries the headers ready-made)."""
|
||||
shaders = runtime / 'shaders'
|
||||
found = sorted(p for p in shaders.iterdir() if p.suffix in STAGES or p.suffix == ASM) if shaders.is_dir() else []
|
||||
out = [(p, runtime / f'{p.stem}_spv.h') for p in found]
|
||||
out += [(TOOLS / script, runtime / f'{name}_spv.h') for name, script in ASSEMBLED.items() if (TOOLS / script).is_file()]
|
||||
return out
|
||||
|
||||
|
||||
def assembled_text(script):
|
||||
spec = importlib.util.spec_from_file_location(script.stem, script)
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
return mod.HEADER
|
||||
|
||||
|
||||
def read_text(header):
|
||||
return header.read_bytes().decode('utf-8').replace('\r\n', '\n') if header.is_file() else None
|
||||
|
||||
|
||||
def stale(runtime):
|
||||
"""Shader sources under runtime/shaders whose header is missing or records another source hash."""
|
||||
"""The shader sources whose header is missing or no longer comes from them."""
|
||||
out = []
|
||||
for src in sources(runtime):
|
||||
h = header_path(src)
|
||||
if not h.exists() or f'// source sha256 {source_hash(src)}' not in h.read_text(encoding='utf-8'):
|
||||
for src, header in sources(runtime):
|
||||
text = read_text(header)
|
||||
if text is None:
|
||||
out.append(src)
|
||||
elif src.suffix == '.py':
|
||||
if text != assembled_text(src):
|
||||
out.append(src)
|
||||
elif f'// source sha256 {source_hash(src)}' not in text:
|
||||
out.append(src)
|
||||
return out
|
||||
|
||||
|
||||
def compile_one(src):
|
||||
# "// build_shaders: keep functions" in a source: no optimisation, so its functions stay functions (the HDR tonemap,
|
||||
# which runtime/vk_hdr.c splices into the game's shader by function)
|
||||
opt = [] if '// build_shaders: keep functions' in src.read_text(encoding='utf-8') else ['-O']
|
||||
def tool(name):
|
||||
"""The path of glslc, spirv-as or spirv-val; ends the run with the packages to install when it is not there."""
|
||||
exe = BIN / f'{name}.exe' if os.name == 'nt' else None
|
||||
if exe and exe.is_file():
|
||||
return str(exe)
|
||||
found = shutil.which(name)
|
||||
if found:
|
||||
return found
|
||||
where = (f'MSYS2 UCRT64 ({BIN}): pacman -S mingw-w64-ucrt-x86_64-shaderc mingw-w64-ucrt-x86_64-spirv-tools'
|
||||
if os.name == 'nt' else "PATH: install your distribution's shaderc and spirv-tools packages")
|
||||
sys.exit(f'build_shaders: {name} not found. The runtime\'s shaders are compiled when the game is built '
|
||||
f'(docs/building.md); {name} comes from {where}')
|
||||
|
||||
|
||||
def words_of(src):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
spv = Path(tmp) / 'out.spv'
|
||||
subprocess.run([str(BIN / 'glslc.exe') if WIN else 'glslc', *opt, f'--target-env={TARGET}', f'-fshader-stage={STAGES[src.suffix]}',
|
||||
'-o', str(spv), str(src)], check=True)
|
||||
subprocess.run([str(BIN / 'spirv-val.exe') if WIN else 'spirv-val', f'--target-env', TARGET, str(spv)], check=True)
|
||||
if src.suffix == ASM:
|
||||
target = ASM_TARGET
|
||||
subprocess.run([tool('spirv-as'), '--target-env', target, '-o', str(spv), str(src)], check=True)
|
||||
else:
|
||||
# "// build_shaders: keep functions" in a source: no optimisation, so its functions stay functions (the HDR
|
||||
# tonemap, which runtime/vk_hdr.c splices into the game's shader by function)
|
||||
target = TARGET
|
||||
opt = [] if '// build_shaders: keep functions' in src.read_text(encoding='utf-8') else ['-O']
|
||||
subprocess.run([tool('glslc'), *opt, f'--target-env={target}', f'-fshader-stage={STAGES[src.suffix]}',
|
||||
'-o', str(spv), str(src)], check=True)
|
||||
subprocess.run([tool('spirv-val'), '--target-env', target, str(spv)], check=True)
|
||||
data = spv.read_bytes()
|
||||
words = [int.from_bytes(data[i:i + 4], 'little') for i in range(0, len(data), 4)]
|
||||
header_path(src).write_text(header_of(src, words), encoding='utf-8', newline='\n')
|
||||
print(f'{header_path(src).relative_to(ROOT)}: {len(words)} words')
|
||||
return [int.from_bytes(data[i:i + 4], 'little') for i in range(0, len(data), 4)]
|
||||
|
||||
|
||||
def build(runtime, only=None):
|
||||
"""Writes the headers of the sources in only (default: the stale ones) and returns how many it wrote."""
|
||||
todo = stale(runtime) if only is None else list(only)
|
||||
headers = dict(sources(runtime))
|
||||
for src in todo:
|
||||
text = assembled_text(src) if src.suffix == '.py' else header_of(src, words_of(src))
|
||||
header = headers[src]
|
||||
part = header.with_name(header.name + '.part') # whole or not at all: another build may be reading the folder
|
||||
part.write_text(text, encoding='utf-8', newline='\n')
|
||||
part.replace(header)
|
||||
print(f'{header.name}: {text.count("0x")} words from {src.name}')
|
||||
return len(todo)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument('--check', action='store_true', help='only report headers that do not match their source')
|
||||
ap.add_argument('--check', action='store_true', help='only report headers that are missing or do not match their source')
|
||||
ap.add_argument('--all', action='store_true', help='build every header, not only the missing and the stale ones')
|
||||
ap.add_argument('--runtime', type=Path, default=ROOT / 'runtime')
|
||||
args = ap.parse_args()
|
||||
runtime = args.runtime.resolve()
|
||||
if args.check:
|
||||
bad = stale(args.runtime)
|
||||
bad = stale(runtime)
|
||||
headers = dict(sources(runtime))
|
||||
for s in bad:
|
||||
print(f'{header_path(s)}: stale, run tools/build_shaders.py')
|
||||
print(f'{headers[s]}: missing or stale, run tools/build_shaders.py')
|
||||
return 1 if bad else 0
|
||||
for src in sources(args.runtime):
|
||||
compile_one(src)
|
||||
n = build(runtime, [s for s, _ in sources(runtime)] if args.all else None)
|
||||
if not n:
|
||||
print('build_shaders: every header matches its source')
|
||||
return 0
|
||||
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Refuse files that look like game content, keys or generated game code.
|
||||
"""Refuse files that look like game content, keys, generated game code or compiled shaders.
|
||||
|
||||
Runs in CI on every push and pull request, and locally as the pre-commit hook
|
||||
(.githooks/pre-commit). With no arguments it checks every tracked file; with
|
||||
@@ -83,6 +83,14 @@ def problems_for(path: Path, allow_large: bool = False, data: bytes | None = Non
|
||||
return found
|
||||
|
||||
|
||||
def compiled_shader(path: Path) -> bool:
|
||||
"""A SPIR-V binary or a header of SPIR-V words. The repository holds its shaders as source (runtime/shaders/, the
|
||||
tools/make_*.py assemblers) and tools/build_shaders.py builds runtime/<name>_spv.h from them when the game is
|
||||
built. A build kit or a release folder carries the headers, so make_kit.py and make_release.py, which call
|
||||
problems_for, are not held to this."""
|
||||
return path.suffix.lower() == ".spv" or path.name.lower().endswith("_spv.h")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
parser.add_argument("--staged", action="store_true", help="check staged files only")
|
||||
@@ -92,11 +100,14 @@ def main() -> int:
|
||||
for path in list_files(args.staged):
|
||||
# --staged checks what the commit takes: a marker staged and then removed from the working copy alone
|
||||
# passed when the file on disk was read
|
||||
for problem in problems_for(path, data=staged_bytes(path) if args.staged else None):
|
||||
problems = problems_for(path, data=staged_bytes(path) if args.staged else None)
|
||||
if compiled_shader(path):
|
||||
problems.append("compiled shader: the repository holds the source, tools/build_shaders.py builds this")
|
||||
for problem in problems:
|
||||
print(f"{path}: {problem}")
|
||||
failed = True
|
||||
if failed:
|
||||
print("\nGame content, keys and generated game code never go into the repository.")
|
||||
print("\nGame content, keys, generated game code and compiled shaders never go into the repository.")
|
||||
print("See CONTRIBUTING.md.")
|
||||
return 1
|
||||
return 0
|
||||
|
||||
@@ -145,6 +145,9 @@ def main():
|
||||
shutil.rmtree(app, ignore_errors=True)
|
||||
kit = app / 'usr' / 'kit'
|
||||
kit.mkdir(parents=True)
|
||||
sys.path.insert(0, str(ROOT / 'tools'))
|
||||
import build_shaders
|
||||
build_shaders.build(ROOT / 'runtime') # the image carries the runtime's shaders compiled (runtime/*_spv.h)
|
||||
for d in KIT_DIRS:
|
||||
copy_tree(ROOT / d, kit / d)
|
||||
copy_tree(ROOT / 'tools', kit / 'tools', TOOLS_SKIP)
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""Writes runtime/dlaa_mask_spv.h: the SPIR-V of DLSS's reactive mask pass (vk_dlaa.c, mask_pass), assembled here like
|
||||
tools/make_mvobj_fs.py (no GLSL compiler on the machine this was written on).
|
||||
"""The SPIR-V of DLSS's reactive mask pass (vk_dlaa.c, mask_pass) as runtime/dlaa_mask_spv.h, assembled here like
|
||||
tools/make_mvobj_fs.py. tools/build_shaders.py writes the header from HEADER when the game is built; run on its own,
|
||||
this script writes it too.
|
||||
|
||||
The shader, as GLSL:
|
||||
|
||||
@@ -196,8 +197,11 @@ a.op(F, OpFunctionEnd)
|
||||
|
||||
w = a.words()
|
||||
lines = [', '.join(f'0x{x:08x}' for x in w[i:i + 8]) for i in range(0, len(w), 8)]
|
||||
OUT.write_text(
|
||||
HEADER = (
|
||||
'// Generated by tools/make_dlaa_mask.py: DLSS\'s reactive mask pass (vk_dlaa.c mask_pass). Do not edit.\n'
|
||||
'#pragma once\n#include <stdint.h>\n'
|
||||
'static const uint32_t dlaa_mask_spv[] = {\n ' + ',\n '.join(lines) + '\n};\n', newline='\n')
|
||||
print(f'{OUT}: {len(w)} words')
|
||||
'static const uint32_t dlaa_mask_spv[] = {\n ' + ',\n '.join(lines) + '\n};\n')
|
||||
|
||||
if __name__ == '__main__':
|
||||
OUT.write_text(HEADER, newline='\n')
|
||||
print(f'{OUT}: {len(w)} words')
|
||||
|
||||
@@ -4,7 +4,9 @@
|
||||
[--dlss nvngx_dlss.dll | --no-dlss] [--dlssg nx2navi_dlssg.dll [--streamline SDK]]
|
||||
[--prompt-art FOLDER [FOLDER ...]] [--out build/kit] [--zip]
|
||||
|
||||
The kit holds this repository's runtime/, recomp/, the build tools, the shader translator built from tools/xlate,
|
||||
The kit holds this repository's runtime/ with its shaders compiled (tools/build_shaders.py builds the SPIR-V headers
|
||||
here, so a player's PC needs no shader compiler; their sources go along), recomp/, the build tools, the shader
|
||||
translator built from tools/xlate,
|
||||
NVIDIA's DLSS library with its notice (tools/dlss_pin.py: downloaded from NVIDIA's DLSS SDK repository and checked,
|
||||
or --dlss; not with --no-dlss), DLSS Frame Generation in bin/dlssg/ with --dlssg (nx2navi_dlssg.dll from
|
||||
tools/build_dlssg.py beside NVIDIA Streamline's runtime, tools/streamline_pin.py: downloaded from NVIDIA's Streamline
|
||||
@@ -27,8 +29,9 @@ ROOT = Path(__file__).resolve().parents[1]
|
||||
NAME = 'NX2navi-BuildKit'
|
||||
TOOLS = ('make_game.py', 'nup.py', 'romfs_index.py', 'check_content.py', 'hostmem.py', 'prepare_shaders.py',
|
||||
'shader_census.py', 'shader_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py', 'romfs_check.py',
|
||||
'build_game.py', 'check_isa.py', 'build_shaders.py', 'pack_shader_cache.py') # make_game.py imports build_game's vma_include on
|
||||
# Linux for the VMA header, and build_game.py imports check_isa and build_shaders
|
||||
'build_game.py', 'check_isa.py', 'build_shaders.py', 'make_dlaa_mask.py', 'make_mvobj_fs.py',
|
||||
'pack_shader_cache.py') # make_game.py imports build_game's vma_include on Linux for the VMA header, and
|
||||
# build_game.py imports check_isa and build_shaders, which reads the two make_ scripts' shaders
|
||||
TOP = ('LICENSE', 'LICENSE-ADDITIONS', 'THIRD_PARTY_NOTICES.md', 'VERSION', 'nx2navi.ini.example')
|
||||
SKIP_DIRS = {'__pycache__', '.pytest_cache'}
|
||||
SKIP_SUFFIXES = {'.o', '.obj', '.pyc', '.exe', '.dll', '.a', '.lib'}
|
||||
@@ -60,6 +63,7 @@ def main():
|
||||
ap.add_argument('--zip', action='store_true')
|
||||
a = ap.parse_args()
|
||||
sys.path.insert(0, str(ROOT / 'tools'))
|
||||
import build_shaders
|
||||
import dlss_pin
|
||||
import prompt_art
|
||||
import streamline_pin
|
||||
@@ -73,6 +77,7 @@ def main():
|
||||
if kit.exists():
|
||||
shutil.rmtree(kit)
|
||||
kit.mkdir(parents=True)
|
||||
build_shaders.build(ROOT / 'runtime') # the kit carries the shader headers: build the missing and the stale ones
|
||||
for d in ('runtime', 'recomp'):
|
||||
copy_tree(ROOT / d, kit / d)
|
||||
(kit / 'tools').mkdir()
|
||||
|
||||
@@ -57,6 +57,7 @@ from pathlib import Path
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
sys.path.insert(0, str(ROOT / 'tools'))
|
||||
import build_shaders # noqa: E402
|
||||
import check_content # noqa: E402
|
||||
import dlss_pin # noqa: E402
|
||||
import prompt_art # noqa: E402
|
||||
@@ -177,6 +178,9 @@ def build_python(py):
|
||||
# ---------------------------------------------------------------- app/<version>/
|
||||
|
||||
def build_app(app, ver, xlate, feed_repo, lock, ffx=None, dlss=None, art=None, dlssg=None):
|
||||
# The runtime's own shaders are source in the repository and SPIR-V headers in a release: a player's PC compiles
|
||||
# runtime/ (make_game.py) and has no shader compiler. Build the missing and the stale ones before the copy.
|
||||
build_shaders.build(ROOT / 'runtime')
|
||||
copy_tree(ROOT / 'launcher', app / 'launcher')
|
||||
for d in ('runtime', 'recomp'):
|
||||
copy_tree(ROOT / d, app / d)
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""Writes runtime/vk_mvobj_fs.h: the SPIR-V of the per-object motion fragment shader (vk_mvobj.c), assembled here
|
||||
because it must be patched at run time (the Location of its input follows the vertex shader variant) and is small.
|
||||
"""The SPIR-V of the per-object motion fragment shader (vk_mvobj.c) as runtime/mvobj_fs_spv.h, assembled here because
|
||||
it must be patched at run time (the Location of its input follows the vertex shader variant) and is small.
|
||||
tools/build_shaders.py writes the header from HEADER when the game is built; run on its own, this script writes it too.
|
||||
|
||||
The shader, as GLSL:
|
||||
|
||||
@@ -24,7 +25,7 @@ import struct
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
OUT = ROOT / 'runtime' / 'vk_mvobj_fs.h'
|
||||
OUT = ROOT / 'runtime' / 'mvobj_fs_spv.h'
|
||||
|
||||
|
||||
class Asm:
|
||||
@@ -156,8 +157,11 @@ a.op(F, OpFunctionEnd)
|
||||
|
||||
w = a.words()
|
||||
lines = [', '.join(f'0x{x:08x}' for x in w[i:i + 8]) for i in range(0, len(w), 8)]
|
||||
OUT.write_text(
|
||||
HEADER = (
|
||||
'// Generated by tools/make_mvobj_fs.py: the per-object motion fragment shader (vk_mvobj.c). Do not edit.\n'
|
||||
f'#define MVOBJ_FS_LOCATION_WORD {a.loc_word} // the input Location literal, patched to the variant\'s\n'
|
||||
'static const uint32_t mvobj_fs_spv[] = {\n ' + ',\n '.join(lines) + '\n};\n', newline='\n')
|
||||
print(f'{OUT}: {len(w)} words, Location at word {a.loc_word}')
|
||||
'static const uint32_t mvobj_fs_spv[] = {\n ' + ',\n '.join(lines) + '\n};\n')
|
||||
|
||||
if __name__ == '__main__':
|
||||
OUT.write_text(HEADER, newline='\n')
|
||||
print(f'{OUT}: {len(w)} words, Location at word {a.loc_word}')
|
||||
|
||||
Reference in New Issue
Block a user