mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-19 10:49:58 -04:00
Merge the fixes line: the stutter, memory, flicker and crash fixes, co-op, the mesh shader fallback and the launcher fixes.
Brought onto the line with FSR 4, its frame generation and the debug menu. Where both sides had added to the same place, both stay: the patch table keeps the pickup count and the free camera, the settings list keeps both sides' names, the menu includes the co-op page and the debug menu, and the launcher builder takes the pipeline list and the FSR 4 folder. Both sides had added the menu's typed-character event; one definition stays, with the window code that passes characters outside the basic plane. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
@@ -5,7 +5,7 @@ recompiled only when it, a header it includes or its flags change.
|
||||
|
||||
On Linux the same steps make build/<version>/nx2navi with the clang, lld and ninja on PATH (or the clang and lld in --toolchain DIR/bin): the
|
||||
runtime's *_sdl.* and platform/linux.c take the place of its *_win.*, platform/win32.c and nx2navi.rc, the libraries are
|
||||
the system's (Vulkan, SDL3, Opus, libvpx, zstd; libstdc++ static), and the executable finds a libSDL3.so.0 beside it
|
||||
the system's (Vulkan, SDL3, Opus, dav1d, libvpx, zstd; libstdc++ static), and the executable finds a libSDL3.so.0 beside it
|
||||
first (rpath $ORIGIN). docs/building.md lists the packages.
|
||||
|
||||
.venv\\Scripts\\python tools\\build_game.py # generated/v121 -> build/v121/nx2navi.exe
|
||||
@@ -45,8 +45,8 @@ of its processes (Windows job object accounting: committed private bytes).
|
||||
|
||||
--ninja PATH names the Ninja to run; without MSYS2 the bin/ninja.exe of the release folder (tools/make_release.py) is used.
|
||||
--toolchain DIR builds with an llvm-mingw tree (DIR/bin/clang.exe) instead of MSYS2 UCRT64 clang. Ninja still comes
|
||||
from MSYS2. The libraries and headers MSYS2 supplies (Vulkan, SDL3, Opus, libvpx, VMA) then come from --deps DIR
|
||||
(default toolchain/deps-mingw: include/, lib/ with the import libraries and libvpx.a, bin/ with the DLLs), and the C++
|
||||
from MSYS2. The libraries and headers MSYS2 supplies (Vulkan, SDL3, Opus, dav1d, libvpx, VMA) then come from --deps DIR
|
||||
(default toolchain/deps-mingw: include/, lib/ with the import libraries, libdav1d.a and libvpx.a, bin/ with the DLLs), and the C++
|
||||
runtime is libc++ from llvm-mingw. Such a build never reuses another build's generated objects.
|
||||
|
||||
Split build. --make-runtime-lib compiles only runtime/ into <out>/libnx2navirt.a (what CI prebuilds). --runtime-lib PATH
|
||||
@@ -61,6 +61,7 @@ import ctypes.wintypes as wt # the job-object structures below; importing it w
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
@@ -145,14 +146,15 @@ if WIN:
|
||||
RT_FLAGS.append('-D_WIN32_WINNT=0x0A00')
|
||||
# -Xlinker passes the map path as one argument; -Wl, would split it at any comma in the build dir path
|
||||
LINK_FLAGS = ['-fuse-ld=lld', '-Wl,--stack,16777216', '-Xlinker', '"--Map=$builddir/nx2navi.map"'] # the map feeds tools/prof_report.py
|
||||
LINK_LIBS = ['-lpsapi', '-lbcrypt', '-lvulkan-1', '-luser32', '-lgdi32', '-ldwmapi', '-limm32', '-lwinmm', '-lole32', '-lshell32', '-luuid', '-lSDL3', '-lopus', '-l:libvpx.a', '-l:libzstd.a', '-l:libstdc++.a', '-l:libwinpthread.a'] # Vulkan back end; libvpx static (movies); libzstd static (resolution.c)
|
||||
LINK_LIBS = ['-lpsapi', '-lbcrypt', '-lws2_32', '-lvulkan-1', '-luser32', '-lgdi32', '-ldwmapi', '-limm32', '-lwinmm', '-lole32', '-lshell32', '-luuid', '-lSDL3', '-lopus', '-l:libdav1d.a', '-l:libvpx.a', '-l:libzstd.a', '-l:libstdc++.a', '-l:libwinpthread.a'] # Vulkan back end; libvpx and libdav1d static (movies); libzstd static (resolution.c)
|
||||
VMA_INCLUDE = UCRT.parent / 'include' / 'vma' # vk_mem_alloc.h; MSYS2 package vulkan-memory-allocator
|
||||
else:
|
||||
# -no-pie: the lifted modules' function tables (hundreds of thousands of pointers) need no load-time relocation.
|
||||
# $ORIGIN: a libSDL3.so.0 next to nx2navi comes before the system's (the release's portable folder).
|
||||
# -Xlinker for the map path, as on Windows above: -Wl, would split it at any comma in the build dir path.
|
||||
LINK_FLAGS = ['-fuse-ld=lld', '-no-pie', '-Wl,-z,noexecstack', "'-Wl,-rpath,$$ORIGIN'",
|
||||
'"-Wl,-Map=$builddir/nx2navi.map"']
|
||||
LINK_LIBS = ['-lvulkan', '-lSDL3', '-lopus', '-lvpx', '-lzstd', '-l:libstdc++.a', '-lm', '-lpthread', '-ldl']
|
||||
'-Xlinker', '"-Map=$builddir/nx2navi.map"']
|
||||
LINK_LIBS = ['-lvulkan', '-lSDL3', '-lopus', '-ldav1d', '-lvpx', '-lzstd', '-l:libstdc++.a', '-lm', '-lpthread', '-ldl']
|
||||
VMA_INCLUDE = Path('/usr/include') # vk_mem_alloc.h; Arch (AUR) vulkan-memory-allocator, Debian libvulkan-memory-allocator-dev
|
||||
# Without a system vk_mem_alloc.h, the header the Windows release pins (tools/make_release.py, VMA 3.4.0) is fetched once
|
||||
# into toolchain/vma: one MIT header (THIRD_PARTY_NOTICES.md), checked against its hash.
|
||||
@@ -180,9 +182,43 @@ def vma_include():
|
||||
return VMA_LOCAL
|
||||
|
||||
|
||||
def system_vulkan_version(clang, env=None):
|
||||
"""VK_HEADER_VERSION of the <vulkan/vulkan_core.h> this compiler finds by itself (its own search path, CPATH
|
||||
included), 0 when it finds none."""
|
||||
probe = '#include <vulkan/vulkan_core.h>\nnx2navi_vulkan_headers VK_HEADER_VERSION\n'
|
||||
try:
|
||||
p = subprocess.run([str(clang), '-E', '-P', '-x', 'c', '-'], input=probe, capture_output=True, text=True,
|
||||
env=env, errors='replace')
|
||||
except OSError:
|
||||
return 0
|
||||
m = re.search(r'nx2navi_vulkan_headers\s+(\d+)', p.stdout) if p.returncode == 0 else None
|
||||
return int(m.group(1)) if m else 0
|
||||
|
||||
|
||||
def vulkan_include(clang=None, env=None):
|
||||
"""A build with the system's own headers (Linux without --deps): the folder to put ahead of them for
|
||||
<vulkan/vulkan.h>, or None when the system's serve. The runtime names Vulkan extensions newer than the headers
|
||||
many distributions package, and every file that does fails to compile there (tools/vulkan_pin.py). So:
|
||||
toolchain/vulkan-headers when it holds the pinned release (a kit carries it; a checkout once it was fetched), else
|
||||
nothing when the system's headers are at least that release, else the pinned release, fetched there once.
|
||||
Windows takes MSYS2's headers and returns None."""
|
||||
if WIN:
|
||||
return None
|
||||
import vulkan_pin
|
||||
local = ROOT / vulkan_pin.FOLDER
|
||||
if vulkan_pin.have(local):
|
||||
return local / 'include'
|
||||
found = system_vulkan_version(clang or CLANG, env)
|
||||
if found >= vulkan_pin.HEADER_VERSION:
|
||||
return None
|
||||
print(f'build_game: the system\'s Vulkan headers are {f"release {found}" if found else "missing"}; the runtime is '
|
||||
f'built against release {vulkan_pin.HEADER_VERSION} ({vulkan_pin.VERSION}) or newer', flush=True)
|
||||
return vulkan_pin.resolve(local) / 'include'
|
||||
|
||||
|
||||
# --toolchain: an llvm-mingw tree. The libraries, headers and DLLs MSYS2 supplies come from the deps directory instead.
|
||||
LLVM_LINK_LIBS = ['-lpsapi', '-lbcrypt', '-luser32', '-lgdi32', '-ldwmapi', '-limm32', '-lwinmm', '-lole32', '-lshell32', '-luuid', '-lvulkan-1', '-lSDL3', '-lopus',
|
||||
'-l:libvpx.a', '-l:libzstd.a', '-l:libc++.a', '-l:libc++abi.a', '-l:libunwind.a', '-l:libwinpthread.a']
|
||||
LLVM_LINK_LIBS = ['-lpsapi', '-lbcrypt', '-lws2_32', '-luser32', '-lgdi32', '-ldwmapi', '-limm32', '-lwinmm', '-lole32', '-lshell32', '-luuid', '-lvulkan-1', '-lSDL3', '-lopus',
|
||||
'-l:libdav1d.a', '-l:libvpx.a', '-l:libzstd.a', '-l:libc++.a', '-l:libc++abi.a', '-l:libunwind.a', '-l:libwinpthread.a']
|
||||
DEFAULT_DEPS = ROOT / 'toolchain' / 'deps-mingw'
|
||||
# Runtime sources compiled per install, not in the library: romfs.c includes romfs_config.h, patches.c and ov_zstd.c
|
||||
# the patches.h of the player's game version (its patch page is the page after that version's main; ov_zstd.c's host
|
||||
@@ -322,7 +358,7 @@ def runtime_sources(skip=()):
|
||||
"""Every runtime/*.c, *.cpp, platform/*.c, *.rc, the in-game menu (ui/*.cpp) and Dear ImGui of this OS except the
|
||||
names in skip, so a task adds a file without editing this script."""
|
||||
pats = ('*.c', '*.cpp', 'platform/*.c', '*.rc', 'ui/*.cpp', 'third_party/imgui/*.cpp',
|
||||
'third_party/imgui/backends/*.cpp')
|
||||
'third_party/imgui/backends/*.cpp', 'third_party/monocypher/*.c')
|
||||
return sorted(p for pat in pats for p in RUNTIME.glob(pat) if p.name not in skip and not other_os(p.name))
|
||||
|
||||
|
||||
@@ -381,6 +417,9 @@ def ninja_file(gen, out, mods, pool_depth, skip=(), prebuilt=False, runtime_lib=
|
||||
vma = f'{cmd_path(DEPS / "include")} -I {cmd_path(DEPS / "include" / "vma")}' if DEPS else cmd_path(vma_include())
|
||||
imgui = cmd_path(RUNTIME / 'third_party' / 'imgui')
|
||||
rt_inc = f'{inc} -I {cmd_path(out)} -I {vma} -I {imgui}' # romfs_config.h, vk_mem_alloc.h (+ deps headers), imgui.h
|
||||
vulkan = None if DEPS else vulkan_include() # deps carries the Vulkan headers itself
|
||||
if vulkan: # first, so that no other folder on the path answers for <vulkan/vulkan.h> (VMA's may be /usr/include)
|
||||
rt_inc = f'-I {cmd_path(vulkan)} {rt_inc}'
|
||||
libs = [f'-L {cmd_path(DEPS / "lib")}'] + (LLVM_LINK_LIBS if WIN else LINK_LIBS) if DEPS else LINK_LIBS
|
||||
# A compiler cache (sccache) may front clang: every lift rewrites the generated chunks with the same contents, so
|
||||
# the cached objects save the compile step on the next build.
|
||||
@@ -857,9 +896,9 @@ def main():
|
||||
DEPS = (args.deps or DEFAULT_DEPS).resolve()
|
||||
dlls = sorted((DEPS / 'bin').glob('*.dll')) if WIN else []
|
||||
if DEPS and not WIN:
|
||||
# A release (--deps): Opus, libvpx and zstd go into the executable, so it needs no more than libc, Vulkan and
|
||||
# the libSDL3.so.0 beside it
|
||||
for lib in ('opus', 'vpx', 'zstd'):
|
||||
# A release (--deps): Opus, dav1d, libvpx and zstd go into the executable, so it needs no more than libc,
|
||||
# Vulkan and the libSDL3.so.0 beside it
|
||||
for lib in ('opus', 'dav1d', 'vpx', 'zstd'):
|
||||
LINK_LIBS[LINK_LIBS.index(f'-l{lib}')] = f'-l:lib{lib}.a'
|
||||
if args.sysroot and not WIN:
|
||||
# Every compile and the link: the game then needs no newer glibc than the sysroot's (the Steam Runtime's)
|
||||
|
||||
@@ -1,13 +1,17 @@
|
||||
"""Build the runtime's own shaders into C headers of SPIR-V words.
|
||||
"""Build the runtime's own shaders into C headers of SPIR-V words, and the files it embeds into headers of bytes.
|
||||
|
||||
The repository holds the shaders as source only. Each becomes runtime/<name>_spv.h, which declares
|
||||
`static const uint32_t <name>_spv[]`; the headers are build outputs (ignored by git) and tools/build_game.py builds the
|
||||
missing and the stale ones before it compiles the runtime. Three kinds of source:
|
||||
|
||||
runtime/shaders/<name>.comp, .vert, .frag GLSL, compiled with glslc (one name per shader) and checked with spirv-val
|
||||
runtime/shaders/<name>.mesh, .task the same for the mesh fallback probe's mesh and task shaders (SPIR-V 1.5)
|
||||
runtime/shaders/<name>.spvasm SPIR-V assembly, assembled with spirv-as and checked with spirv-val
|
||||
tools/make_dlaa_mask.py, make_mvobj_fs.py shaders assembled in Python (ASSEMBLED); they need no other tool
|
||||
|
||||
One more kind of header is no shader: a file the runtime embeds (EMBEDDED: the menu's font) as a list of its bytes,
|
||||
for a compiler without #embed (runtime/ui/menu.cpp includes it there). It needs no tool.
|
||||
|
||||
A compiled or assembled header records the SHA-256 of its source and is stale when that no longer matches; a Python
|
||||
one is stale when its text is not what its script gives. glslc, spirv-as and spirv-val come from MSYS2 UCRT64 on
|
||||
Windows (packages mingw-w64-ucrt-x86_64-shaderc and mingw-w64-ucrt-x86_64-spirv-tools) and from PATH elsewhere
|
||||
@@ -31,15 +35,34 @@ TOOLS = Path(__file__).resolve().parent
|
||||
ROOT = TOOLS.parent
|
||||
BIN = Path(r'C:\msys64\ucrt64\bin') # Windows; elsewhere, and without MSYS2 there, the tools come from PATH
|
||||
TARGET = 'vulkan1.1'
|
||||
STAGES = {'.comp': 'compute', '.vert': 'vertex', '.frag': 'fragment'}
|
||||
STAGES = {'.comp': 'compute', '.vert': 'vertex', '.frag': 'fragment', '.mesh': 'mesh', '.task': 'task'}
|
||||
MESH_TARGET = 'vulkan1.2' # mesh and task shaders need SPIR-V 1.4 or later
|
||||
ASM = '.spvasm'
|
||||
ASM_TARGET = 'vulkan1.3'
|
||||
# header name -> the script beside this one that assembles it; the script's HEADER is the header's text
|
||||
ASSEMBLED = {'dlaa_mask': 'make_dlaa_mask.py', 'mvobj_fs': 'make_mvobj_fs.py'}
|
||||
# a file below runtime/ the runtime embeds -> the header of its bytes, below runtime/ too (git ignores it)
|
||||
EMBEDDED = {'ui/fonts/Roboto-Medium.ttf': 'ui/roboto_ttf.h'}
|
||||
|
||||
|
||||
def embedded(src):
|
||||
"""True for a file of EMBEDDED: a binary file, read as it is."""
|
||||
return src.suffix not in STAGES and src.suffix not in (ASM, '.py')
|
||||
|
||||
|
||||
def source_hash(src):
|
||||
return hashlib.sha256(src.read_bytes().replace(b'\r\n', b'\n')).hexdigest() # the same on a CRLF checkout
|
||||
data = src.read_bytes()
|
||||
return hashlib.sha256(data if embedded(src) else data.replace(b'\r\n', b'\n')).hexdigest() # text: the same on a CRLF checkout
|
||||
|
||||
|
||||
def bytes_header(src, runtime):
|
||||
"""The header of an embedded file: its bytes as decimal numbers, to include inside an array's braces."""
|
||||
data = src.read_bytes()
|
||||
lines = [f'// Generated by tools/build_shaders.py from runtime/{src.relative_to(runtime).as_posix()}; do not edit.',
|
||||
f'// source sha256 {source_hash(src)}',
|
||||
f'// The file\'s {len(data)} bytes, for a compiler without #embed.']
|
||||
lines += [','.join(map(str, data[i:i + 32])) + ',' for i in range(0, len(data), 32)]
|
||||
return '\n'.join(lines) + '\n'
|
||||
|
||||
|
||||
def header_of(src, words):
|
||||
@@ -64,6 +87,7 @@ def sources(runtime):
|
||||
found = sorted(p for p in shaders.iterdir() if p.suffix in STAGES or p.suffix == ASM) if shaders.is_dir() else []
|
||||
out = [(p, runtime / f'{p.stem}_spv.h') for p in found]
|
||||
out += [(TOOLS / script, runtime / f'{name}_spv.h') for name, script in ASSEMBLED.items() if (TOOLS / script).is_file()]
|
||||
out += [(runtime / src, runtime / header) for src, header in EMBEDDED.items() if (runtime / src).is_file()]
|
||||
return out
|
||||
|
||||
|
||||
@@ -116,7 +140,7 @@ def words_of(src):
|
||||
else:
|
||||
# "// build_shaders: keep functions" in a source: no optimisation, so its functions stay functions (the HDR
|
||||
# tonemap, which runtime/vk_hdr.c splices into the game's shader by function)
|
||||
target = TARGET
|
||||
target = MESH_TARGET if src.suffix in ('.mesh', '.task') else TARGET
|
||||
opt = [] if '// build_shaders: keep functions' in src.read_text(encoding='utf-8') else ['-O']
|
||||
subprocess.run([tool('glslc'), *opt, f'--target-env={target}', f'-fshader-stage={STAGES[src.suffix]}',
|
||||
'-o', str(spv), str(src)], check=True)
|
||||
@@ -130,12 +154,14 @@ def build(runtime, only=None):
|
||||
todo = stale(runtime) if only is None else list(only)
|
||||
headers = dict(sources(runtime))
|
||||
for src in todo:
|
||||
text = assembled_text(src) if src.suffix == '.py' else header_of(src, words_of(src))
|
||||
text = (assembled_text(src) if src.suffix == '.py' else bytes_header(src, runtime) if embedded(src)
|
||||
else header_of(src, words_of(src)))
|
||||
header = headers[src]
|
||||
part = header.with_name(header.name + '.part') # whole or not at all: another build may be reading the folder
|
||||
part.write_text(text, encoding='utf-8', newline='\n')
|
||||
part.replace(header)
|
||||
print(f'{header.name}: {text.count("0x")} words from {src.name}')
|
||||
print(f'{header.name}: {src.stat().st_size} bytes of {src.name}' if embedded(src)
|
||||
else f'{header.name}: {text.count("0x")} words from {src.name}')
|
||||
return len(todo)
|
||||
|
||||
|
||||
|
||||
@@ -99,8 +99,8 @@ def prepare_sources() -> Path:
|
||||
for junk in list(src.glob("src/*/obj")) + list(src.glob("src/*/bin")):
|
||||
shutil.rmtree(junk)
|
||||
patches = sorted((XLATE / "patches").glob("*.patch"))
|
||||
if len(patches) != 37:
|
||||
sys.exit(f"expected 37 patches, found {len(patches)}")
|
||||
if len(patches) != 39:
|
||||
sys.exit(f"expected 39 patches, found {len(patches)}")
|
||||
# build/xlate is inside this checkout, and git inside a repository reads a "diff --git" patch's paths from the
|
||||
# repository's top: those hunks fall outside the directory handed to --directory, git skips every file of the patch
|
||||
# and still exits 0. GIT_CEILING_DIRECTORIES stops git from walking up to the checkout, so it reads each patch as it
|
||||
|
||||
@@ -91,6 +91,23 @@ def compiled_shader(path: Path) -> bool:
|
||||
return path.suffix.lower() == ".spv" or path.name.lower().endswith("_spv.h")
|
||||
|
||||
|
||||
# What a game folder records in play and what tools/pipeline_list.py folds from it: the game's shader code, the
|
||||
# answers it was translated for, the pipelines it drew. All of it is derived from game data.
|
||||
RECORD_MAGICS = (b"NX2NSPRG", b"NX2NSANS", b"NX2NPKEY", b"NX2NPLST")
|
||||
|
||||
|
||||
def recorded_in_play(path: Path, data: bytes | None) -> bool:
|
||||
"""A shader-programs.bin, shader-answers.bin, pipeline-keys.bin or a pipeline list, by its first bytes. A build kit
|
||||
or a release folder may carry a pipeline list (data/pipeline_list.bin), so make_kit.py and make_release.py, which
|
||||
call problems_for, are not held to this."""
|
||||
if data is None:
|
||||
if not path.is_file():
|
||||
return False
|
||||
with open(path, "rb") as f:
|
||||
data = f.read(8)
|
||||
return data[:8] in RECORD_MAGICS
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
parser.add_argument("--staged", action="store_true", help="check staged files only")
|
||||
@@ -100,9 +117,12 @@ def main() -> int:
|
||||
for path in list_files(args.staged):
|
||||
# --staged checks what the commit takes: a marker staged and then removed from the working copy alone
|
||||
# passed when the file on disk was read
|
||||
problems = problems_for(path, data=staged_bytes(path) if args.staged else None)
|
||||
data = staged_bytes(path) if args.staged else None
|
||||
problems = problems_for(path, data=data)
|
||||
if compiled_shader(path):
|
||||
problems.append("compiled shader: the repository holds the source, tools/build_shaders.py builds this")
|
||||
if recorded_in_play(path, data):
|
||||
problems.append("recorded from a game folder (shaders, answers, pipeline keys or a pipeline list)")
|
||||
for problem in problems:
|
||||
print(f"{path}: {problem}")
|
||||
failed = True
|
||||
|
||||
@@ -39,6 +39,8 @@ CALLS = [
|
||||
"localtime_s", "strtok_s", "GetModuleHandle(Ex)?[AW]?", "ExitProcess", "SetEnvironmentVariable[AW]?", "_putenv(_s)?",
|
||||
"GetLastError", "MultiByteToWideChar", "WideCharToMultiByte", "HWND", "LARGE_INTEGER", "FILETIME",
|
||||
"MAX_PATH", "GetActiveProcessorCount", "timeBeginPeriod",
|
||||
"WSAStartup", "WSAPoll", "WSAIoctl", "WSAGetLastError", "closesocket", "ioctlsocket", "SOCKET",
|
||||
"sockaddr(_in6?|_storage)?",
|
||||
]
|
||||
CALL_RE = re.compile(r"\b(" + "|".join(CALLS) + r")\b")
|
||||
|
||||
|
||||
868
tools/coop_relay.py
Executable file
868
tools/coop_relay.py
Executable file
@@ -0,0 +1,868 @@
|
||||
#!/usr/bin/env python3
|
||||
"""NX2navi co-op relay: forwards datagrams between the players of a room (up to 8), nothing else. Standard library only.
|
||||
|
||||
Usage: python tools/coop_relay.py [--host 0.0.0.0] [--port 24680] [--log-addrs] [--log-joins] [--max-rooms 1000]
|
||||
|
||||
The game's co-op layer (runtime/coop.c) talks to it over UDP; any player can run one on a machine the others can reach
|
||||
(forward the UDP port on a router) and put its address into the game's coop_server setting. docs/coop.md describes the
|
||||
setup. The relay keeps only who is in which room: it stores nothing on disk, and a restart loses the rooms (the players'
|
||||
games make them again).
|
||||
|
||||
A room is named by a secret the players share among themselves; the relay never sees it. It knows the room by a locator
|
||||
derived from the secret (16 bytes, BLAKE2b), which also keys the tag of every datagram between a client and the relay.
|
||||
What the players send each other (DATA) is sealed with another key derived from the secret: the relay forwards it and
|
||||
can neither read nor change nor make it.
|
||||
|
||||
The wire format is runtime/coop_proto.h's; tests/coop_vectors.txt holds byte vectors both sides are checked against.
|
||||
A datagram is a 24-byte header, little-endian, then a payload:
|
||||
|
||||
0 u16 magic 0x434E 2 u8 version (2; the relay accepts that one) 3 u8 type
|
||||
4 u32 room (0 until joined) 8 u8 src member 1..8 (0 = relay) 9 u8 dst member (0xFF: every other member)
|
||||
10 u16 payload length 12 u32 seq (the sender's datagram counter)
|
||||
16 u8[8] tag = HMAC-SHA256(locator, header[0:16] + payload)[:8]
|
||||
|
||||
Joining is a cookie exchange, so a spoofed source address cannot make the relay send anything larger than it received:
|
||||
a CONNECT is answered with a CHALLENGE holding a cookie of the sender's address and the minute; the CONNECT that carries
|
||||
the cookie joins. Every answer echoes the CONNECT's nonce. JOINED hands the member a rejoin token: a later CONNECT of the
|
||||
same client that carries it gets the same member id back (from the member's own address and in its own session without
|
||||
a cookie, from another address or with another session only with one), one without it is a new member. A session that
|
||||
was in the room before is refused: the players' games bind a member's signed identity to its session.
|
||||
|
||||
Limits: per member (datagrams and bytes, a forwarded datagram counted once for each receiver), per network (/24 for
|
||||
IPv4, /48 for IPv6: challenges, joins, rooms made; at most MAX_NETWORKS of them are tracked), per room (an equal share
|
||||
of the bytes the relay forwards among the rooms that forwarded to someone in the last ACTIVE seconds), and for the
|
||||
whole relay (challenges, with a share set aside for members that rejoin with their token, and bytes sent). A join
|
||||
attempt costs its network one join, whatever the answer; the same attempt sent again (the same client, address and
|
||||
CONNECT sequence) while the answer would be the same refusal is answered from memory without a charge, so a player
|
||||
who waits for a room that is not there yet costs one join per attempt, not one per question.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import hmac
|
||||
import ipaddress
|
||||
import secrets
|
||||
import select
|
||||
import socket
|
||||
import struct
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
# ---- the wire format
|
||||
|
||||
MAGIC = 0x434E
|
||||
PROTO = 2
|
||||
HDR_LEN = 24
|
||||
MAX_DGRAM = 1200
|
||||
DST_ALL = 0xFF
|
||||
ALPHABET = '23456789BCDFGHJKMNPRSTVWXYZ' # no vowels, no 0/1/O/I/L, no Q or E: 27 symbols
|
||||
SECRET_LEN = 18 # 85.6 bits
|
||||
LOCATOR_LEN = 16
|
||||
MAX_MEMBERS = 8
|
||||
NAME_MAX = 24
|
||||
COOKIE_LEN = 16
|
||||
CONNECT_MIN = 64 # a datagram's smallest size for a CONNECT: the CHALLENGE answering it is smaller
|
||||
CONNECT_FIXED = 43
|
||||
ENV_V1, ENV_HDR, ENV_MAC = 1, 13, 16
|
||||
|
||||
T_CONNECT, T_CHALLENGE, T_JOINED, T_ERROR, T_ROSTER, T_PING, T_PONG, T_LEAVE, T_KICK, T_DATA = 1, 2, 3, 5, 6, 7, 8, 9, 11, 0x10
|
||||
INTENT_CREATE, INTENT_JOIN = 1, 2
|
||||
E_NO_ROOM, E_ROOM_FULL, E_PROTO, E_RATE, E_BAD_TAG, E_SERVER_FULL, E_ROOM_EXISTS, E_REMOVED, E_SESSION_USED = 1, 2, 3, 4, 5, 6, 7, 8, 9
|
||||
FLAG_AWAY, FLAG_MAKER = 1, 2
|
||||
|
||||
# ---- timing (seconds)
|
||||
SILENT_AWAY = 8 # a member that sent nothing for this long shows as away in the roster
|
||||
SILENT_DROP = 15 # and is dropped after this; its member id stays reserved for a rejoin
|
||||
REJOIN_KEEP = 60 # a dropped member's id is kept this long after its last datagram
|
||||
ROOM_EMPTY_TTL = 600 # a room with nobody in it is deleted after this; one nobody but its maker ever joined, at REJOIN_KEEP
|
||||
ROSTER_EVERY = 5 # the roster is sent again this often: a lost update repairs itself
|
||||
ROSTER_AGAIN = (0.25, 0.75) # and after a change, again this long after it: three chances against loss, as in coop.c's CONNECT bursts
|
||||
MAX_NETWORKS = 4096 # networks whose challenge or join budget is tracked at once; a budget that is full again is forgotten
|
||||
SESSIONS_KEPT = 4096 # sessions a room remembers having seen, so that none is taken twice (the oldest forgotten first)
|
||||
ACTIVE = 2 # a room that forwarded to someone within this many seconds shares the forwarding budget
|
||||
FAILED_KEPT = 4096 # join attempts whose refusal is remembered, so that the same attempt again costs nothing
|
||||
FAILED_TTL = 120 # and for this long (a cookie lasts one to two minutes)
|
||||
MEMBER_CHALLENGES = (2, 8) # challenges a second, and at once, for CONNECTs with a member's token: from its own network, and from others
|
||||
|
||||
|
||||
def normalize_secret(text):
|
||||
"""The canonical 18-character room secret of text (any case, '-' and spaces ignored), or None when it is not one."""
|
||||
out = []
|
||||
for ch in text:
|
||||
if ch in '- ':
|
||||
continue
|
||||
o = ord(ch)
|
||||
if 97 <= o <= 122:
|
||||
ch = chr(o - 32)
|
||||
out.append(ch)
|
||||
s = ''.join(out)
|
||||
return s if len(s) == SECRET_LEN and all(c in ALPHABET for c in s) else None
|
||||
|
||||
|
||||
def format_secret(s):
|
||||
return '-'.join(s[i:i + 6] for i in range(0, SECRET_LEN, 6))
|
||||
|
||||
|
||||
def random_secret():
|
||||
return ''.join(secrets.choice(ALPHABET) for _ in range(SECRET_LEN))
|
||||
|
||||
|
||||
def room_locator(secret):
|
||||
"""The relay's name of the room, and the key of the tags: BLAKE2b-128 keyed with the secret."""
|
||||
return hashlib.blake2b(b'nx2navi-coop/2 room', key=secret.encode('ascii'), digest_size=LOCATOR_LEN).digest()
|
||||
|
||||
|
||||
def message_key(secret):
|
||||
"""The key that seals the players' messages (the relay never computes it; the tests and vectors do)."""
|
||||
return hashlib.blake2b(b'nx2navi-coop/2 messages', key=secret.encode('ascii'), digest_size=32).digest()
|
||||
|
||||
|
||||
def make_tag(key, hdr16, payload):
|
||||
return hmac.new(key, hdr16 + payload, hashlib.sha256).digest()[:8]
|
||||
|
||||
|
||||
class Dgram:
|
||||
"""A parsed datagram: the header fields, its first 16 bytes (what the tag covers) and the payload."""
|
||||
__slots__ = ('ver', 'type', 'room', 'src', 'dst', 'seq', 'tag', 'payload', 'hdr16')
|
||||
|
||||
def __init__(self, ver, type_, room, src, dst, seq, tag, payload, hdr16):
|
||||
self.ver, self.type, self.room, self.src, self.dst = ver, type_, room, src, dst
|
||||
self.seq, self.tag, self.payload, self.hdr16 = seq, tag, payload, hdr16
|
||||
|
||||
def verify(self, key):
|
||||
return hmac.compare_digest(make_tag(key, self.hdr16, self.payload), self.tag)
|
||||
|
||||
|
||||
def pack(type_, room, src, dst, seq, payload=b'', key=None, ver=PROTO):
|
||||
"""One datagram. key None: the tag is zero (the answer to a client of another protocol version)."""
|
||||
hdr = struct.pack('<HBBIBBHI', MAGIC, ver, type_, room, src, dst, len(payload), seq & 0xFFFFFFFF)
|
||||
return hdr + (make_tag(key, hdr, payload) if key else bytes(8)) + payload
|
||||
|
||||
|
||||
def parse(data):
|
||||
"""The Dgram in data, or None when it is not one: wrong size, magic, or a length that does not match."""
|
||||
if len(data) < HDR_LEN or len(data) > MAX_DGRAM:
|
||||
return None
|
||||
magic, ver, type_, room, src, dst, length, seq = struct.unpack_from('<HBBIBBHI', data)
|
||||
if magic != MAGIC or length != len(data) - HDR_LEN:
|
||||
return None
|
||||
return Dgram(ver, type_, room, src, dst, seq, data[16:24], data[HDR_LEN:], data[:16])
|
||||
|
||||
|
||||
def control_char(c):
|
||||
"""A character a log or a console may act on: C0, DEL, C1 and the bidi controls (as coop_proto.c's)."""
|
||||
o = ord(c)
|
||||
return o < 32 or 127 <= o < 160 or o in (0x061C, 0x200E, 0x200F) or 0x202A <= o <= 0x202E or 0x2066 <= o <= 0x2069
|
||||
|
||||
|
||||
def clean_name(raw):
|
||||
"""A display name from the bytes a client sent: at most NAME_MAX bytes of valid UTF-8 without control characters
|
||||
(each of those, and each invalid byte, becomes '?')."""
|
||||
text = raw.decode('utf-8', errors='replace')
|
||||
text = ''.join('?' if c == '\ufffd' or control_char(c) else c for c in text)
|
||||
out = b''
|
||||
for c in text:
|
||||
enc = c.encode('utf-8')
|
||||
if len(out) + len(enc) > NAME_MAX:
|
||||
break
|
||||
out += enc
|
||||
return out
|
||||
|
||||
|
||||
def pack_roster(members):
|
||||
"""members: (id, flags, session, name bytes) in id order."""
|
||||
out = bytes([len(members)])
|
||||
for mid, flags, session, name in members:
|
||||
out += struct.pack('<BBQB', mid, flags, session, len(name)) + name
|
||||
return out
|
||||
|
||||
|
||||
def parse_roster(payload, offset=0):
|
||||
"""((id, flags, session, name bytes), ...) from payload[offset:], or None when it is cut short or malformed."""
|
||||
if len(payload) <= offset:
|
||||
return None
|
||||
n, at, out = payload[offset], offset + 1, []
|
||||
if n > MAX_MEMBERS:
|
||||
return None
|
||||
for _ in range(n):
|
||||
if at + 11 > len(payload):
|
||||
return None
|
||||
mid, flags, session, nlen = struct.unpack_from('<BBQB', payload, at)
|
||||
at += 11
|
||||
if nlen > NAME_MAX or at + nlen > len(payload):
|
||||
return None
|
||||
out.append((mid, flags, session, payload[at:at + nlen]))
|
||||
at += nlen
|
||||
return tuple(out)
|
||||
|
||||
|
||||
def pack_connect(intent, flags, locator, client_id, session, nonce, name, cookie=bytes(COOKIE_LEN), token=bytes(COOKIE_LEN)):
|
||||
"""The CONNECT payload, zero-padded so that the datagram is CONNECT_MIN bytes at least."""
|
||||
body = struct.pack('<BB16sQQQB', intent, flags, locator, client_id, session, nonce, len(name)) + name + cookie + token
|
||||
return body + bytes(max(0, CONNECT_MIN - HDR_LEN - len(body)))
|
||||
|
||||
|
||||
def parse_connect(payload):
|
||||
"""(intent, flags, locator, client_id, session, nonce, name bytes, cookie, token) or None."""
|
||||
if len(payload) < CONNECT_FIXED:
|
||||
return None
|
||||
intent, flags, locator, cid, session, nonce, nlen = struct.unpack_from('<BB16sQQQB', payload)
|
||||
if nlen > NAME_MAX or CONNECT_FIXED + nlen + 2 * COOKIE_LEN > len(payload):
|
||||
return None
|
||||
at = CONNECT_FIXED + nlen
|
||||
return (intent, flags, locator, cid, session, nonce, payload[CONNECT_FIXED:at], payload[at:at + COOKIE_LEN],
|
||||
payload[at + COOKIE_LEN:at + 2 * COOKIE_LEN])
|
||||
|
||||
|
||||
def pack_challenge(nonce, cookie):
|
||||
return struct.pack('<Q', nonce) + cookie
|
||||
|
||||
|
||||
def parse_challenge(payload):
|
||||
if len(payload) != 8 + COOKIE_LEN:
|
||||
return None
|
||||
return struct.unpack_from('<Q', payload)[0], payload[8:]
|
||||
|
||||
|
||||
def pack_joined(member, nonce, token, roster):
|
||||
return struct.pack('<BQ', member, nonce) + token + roster
|
||||
|
||||
|
||||
def parse_joined(payload):
|
||||
"""(member, nonce, token, roster) or None."""
|
||||
if len(payload) < 9 + COOKIE_LEN:
|
||||
return None
|
||||
member, nonce = struct.unpack_from('<BQ', payload)
|
||||
roster = parse_roster(payload, 9 + COOKIE_LEN)
|
||||
return None if roster is None else (member, nonce, payload[9:9 + COOKIE_LEN], roster)
|
||||
|
||||
|
||||
def pack_error(reason, nonce, text):
|
||||
t = text.encode('utf-8')[:60]
|
||||
return struct.pack('<BQB', reason, nonce, len(t)) + t
|
||||
|
||||
|
||||
def parse_error(payload):
|
||||
"""(reason, nonce, text) or None."""
|
||||
if len(payload) < 10 or 10 + payload[9] > len(payload):
|
||||
return None
|
||||
reason, nonce, tlen = struct.unpack_from('<BQB', payload)
|
||||
return reason, nonce, payload[10:10 + tlen].decode('utf-8', errors='replace')
|
||||
|
||||
|
||||
def pack_kick(member, session):
|
||||
return struct.pack('<BQ', member, session)
|
||||
|
||||
|
||||
def parse_kick(payload):
|
||||
return struct.unpack('<BQ', payload) if len(payload) == 9 else None
|
||||
|
||||
|
||||
def prefix_of(addr):
|
||||
"""The network an address belongs to for the limits: its /24 for IPv4 (and IPv4 in IPv6), its /48 for IPv6 (the
|
||||
smallest block a provider usually hands one customer, who may hold every /64 in it)."""
|
||||
try:
|
||||
ip = ipaddress.ip_address(addr[0].split('%')[0])
|
||||
except ValueError:
|
||||
return addr[0]
|
||||
if ip.version == 6 and ip.ipv4_mapped:
|
||||
ip = ip.ipv4_mapped
|
||||
bits = 24 if ip.version == 4 else 48
|
||||
return str(ipaddress.ip_network(f'{ip}/{bits}', strict=False))
|
||||
|
||||
|
||||
# ---- the relay
|
||||
|
||||
class Bucket:
|
||||
"""A token bucket: rate tokens per second up to burst."""
|
||||
__slots__ = ('rate', 'burst', 'tokens', 'stamp')
|
||||
|
||||
def __init__(self, rate, burst, now):
|
||||
self.rate, self.burst, self.tokens, self.stamp = rate, burst, burst, now
|
||||
|
||||
def refill(self, now):
|
||||
self.tokens = min(self.burst, self.tokens + max(0.0, now - self.stamp) * self.rate)
|
||||
self.stamp = max(self.stamp, now)
|
||||
|
||||
def take(self, n, now):
|
||||
self.refill(now)
|
||||
if self.tokens < n:
|
||||
return False
|
||||
self.tokens -= n
|
||||
return True
|
||||
|
||||
def full(self, now):
|
||||
"""Refilled to its burst: the same as a new bucket, so it need not be kept."""
|
||||
self.refill(now)
|
||||
return self.tokens >= self.burst
|
||||
|
||||
|
||||
class Member:
|
||||
__slots__ = ('id', 'cid', 'session', 'name', 'addr', 'last_seen', 'away', 'live', 'connect_seq', 'pkts', 'bytes',
|
||||
'token', 'maker', 'home_challenges', 'away_challenges', 'home_rejoins')
|
||||
|
||||
def __init__(self, mid, cid, session, name, addr, now, connect_seq, pkts, bytes_, maker):
|
||||
self.id, self.cid, self.session, self.name, self.addr = mid, cid, session, name, addr
|
||||
self.last_seen, self.away, self.live, self.connect_seq = now, False, True, connect_seq
|
||||
self.pkts, self.bytes = pkts, bytes_
|
||||
self.token = secrets.token_bytes(COOKIE_LEN)
|
||||
self.maker = maker
|
||||
# its own challenges, apart from its packets: anyone who saw its join has its token, and a flood of CONNECTs with
|
||||
# it from another network takes neither the member's messages nor the challenge of its own rejoin from its network
|
||||
self.home_challenges, self.away_challenges = Bucket(*MEMBER_CHALLENGES, now), Bucket(*MEMBER_CHALLENGES, now)
|
||||
self.home_rejoins = None # its share of the challenges set aside for rejoins from its own network (Relay.take_rejoin)
|
||||
|
||||
|
||||
class Room:
|
||||
__slots__ = ('id', 'locator', 'members', 'empty_since', 'next_roster', 'again', 'banned', 'prefix', 'shared', 'egress',
|
||||
'sessions', 'forwarded_at', 'rejoins')
|
||||
|
||||
def __init__(self, rid, locator, prefix, now):
|
||||
self.id, self.locator, self.prefix = rid, locator, prefix
|
||||
self.members = {} # member id -> Member, a dropped one (live False) keeps its id for a rejoin
|
||||
self.empty_since = now
|
||||
self.next_roster = now + ROSTER_EVERY
|
||||
self.again = [] # when the roster goes out once more after a change
|
||||
self.banned = set() # client ids and addresses the maker removed
|
||||
self.shared = False # someone besides its maker joined it once
|
||||
self.egress = None # its share of the bytes the relay forwards (Relay.share_egress)
|
||||
self.sessions = {} # sessions its members used (insertion order: the oldest first), never taken again
|
||||
self.forwarded_at = None # when it last forwarded: a room that did recently shares the budget
|
||||
self.rejoins = None # its share of the challenges set aside for its members' rejoins from other networks (Relay.take_rejoin)
|
||||
|
||||
def use_session(self, session):
|
||||
self.sessions[session] = True
|
||||
if len(self.sessions) > SESSIONS_KEPT:
|
||||
del self.sessions[next(iter(self.sessions))]
|
||||
|
||||
@property
|
||||
def key(self):
|
||||
return self.locator
|
||||
|
||||
@property
|
||||
def label(self):
|
||||
return f'{self.id:08x}'
|
||||
|
||||
def live(self):
|
||||
return [m for m in self.members.values() if m.live]
|
||||
|
||||
|
||||
class Relay:
|
||||
"""The relay's state machine. handle() takes one received datagram and tick() the passage of time; both return the
|
||||
datagrams to send as (address, bytes). now is a monotonic clock in seconds that the caller supplies, so a test
|
||||
can run the timeouts without waiting. An address is the tuple socket.recvfrom gives."""
|
||||
|
||||
def __init__(self, max_rooms=1000, log=None, log_addrs=False, secret=None,
|
||||
pkt_rate=200, pkt_burst=400, byte_rate=65536, byte_burst=131072, join_rate=1 / 3, join_burst=20,
|
||||
challenge_rate=5000, prefix_challenge_rate=20, prefix_challenge_burst=40, rooms_per_prefix=8,
|
||||
egress_rate=8 << 20, rejoin_challenge_rate=1000, log_joins=False):
|
||||
self.max_rooms = max_rooms
|
||||
self.log_fn = log
|
||||
self.log_addrs = log_addrs
|
||||
self.log_joins = log_joins
|
||||
self.secret = secret or secrets.token_bytes(32)
|
||||
self.limits = (pkt_rate, pkt_burst, byte_rate, byte_burst)
|
||||
self.join_limit = (join_rate, join_burst)
|
||||
self.challenge_rate = challenge_rate
|
||||
self.rejoin_challenge_rate = rejoin_challenge_rate
|
||||
self.prefix_challenge = (prefix_challenge_rate, prefix_challenge_burst)
|
||||
self.rooms_per_prefix = rooms_per_prefix
|
||||
self.egress_rate = egress_rate
|
||||
self.rooms = {} # room id -> Room
|
||||
self.locators = {} # locator -> Room
|
||||
self.by_addr = {} # address -> (Room, Member)
|
||||
self.join_buckets = {} # network prefix -> Bucket
|
||||
self.challenge_buckets = {} # network prefix -> Bucket
|
||||
self.prefix_rooms = {} # network prefix -> rooms it made that are open
|
||||
self.failed = {} # (client id, address, CONNECT seq, locator) -> (reason, when): refusals of an attempt (oldest first)
|
||||
self.challenges = None
|
||||
self.next_prune = 0
|
||||
self.egress = None
|
||||
self.rate_said = -10.0 # when the log last told of a join refused for the network's budget
|
||||
self.seq = 0
|
||||
self.stats = {k: 0 for k in ('rx', 'malformed', 'bad_tag', 'no_member', 'wrong_addr', 'rate_limited',
|
||||
'challenges', 'joins', 'rejoins', 'leaves', 'drops', 'forwarded', 'rooms_made',
|
||||
'kicks', 'egress_limited', 'repeats', 'join_charges')}
|
||||
# -- helpers
|
||||
|
||||
def log(self, text):
|
||||
if self.log_fn:
|
||||
self.log_fn(text)
|
||||
|
||||
def where(self, addr):
|
||||
return f' ({addr[0]}:{addr[1]})' if self.log_addrs else ''
|
||||
|
||||
def next_seq(self):
|
||||
self.seq = (self.seq + 1) & 0xFFFFFFFF
|
||||
return self.seq
|
||||
|
||||
def cookie(self, addr, minute):
|
||||
return hmac.new(self.secret, f'{addr[0]}|{addr[1]}|{minute}'.encode(), hashlib.sha256).digest()[:COOKIE_LEN]
|
||||
|
||||
def cookie_ok(self, addr, cookie, now):
|
||||
minute = int(now // 60)
|
||||
return any(hmac.compare_digest(cookie, self.cookie(addr, m)) for m in (minute, minute - 1))
|
||||
|
||||
def to_member(self, room, member, type_, payload=b''):
|
||||
return member.addr, pack(type_, room.id, 0, member.id, self.next_seq(), payload, room.key)
|
||||
|
||||
def error(self, addr, reason, nonce, text, key):
|
||||
return [(addr, pack(T_ERROR, 0, 0, 0, self.next_seq(), pack_error(reason, nonce, text), key))]
|
||||
|
||||
def roster_payload(self, room):
|
||||
return pack_roster([(m.id, (FLAG_AWAY if m.away else 0) | (FLAG_MAKER if m.maker else 0), m.session, m.name)
|
||||
for m in sorted(room.live(), key=lambda m: m.id)])
|
||||
|
||||
def roster_to_all(self, room, skip=None, changed_at=None):
|
||||
"""The roster to every live member but skip. changed_at (the time): the roster changed, so it goes out again soon
|
||||
(ROSTER_AGAIN) for a member that lost this one."""
|
||||
if changed_at is not None:
|
||||
room.again = [changed_at + t for t in ROSTER_AGAIN]
|
||||
payload = self.roster_payload(room)
|
||||
return [self.to_member(room, m, T_ROSTER, payload) for m in room.live() if m is not skip]
|
||||
|
||||
def joined(self, room, member, nonce):
|
||||
payload = pack_joined(member.id, nonce, member.token, self.roster_payload(room))
|
||||
return [self.to_member(room, member, T_JOINED, payload)]
|
||||
|
||||
def new_member_limits(self, now):
|
||||
pkt_rate, pkt_burst, byte_rate, byte_burst = self.limits
|
||||
return Bucket(pkt_rate, pkt_burst, now), Bucket(byte_rate, byte_burst, now)
|
||||
|
||||
def forget(self, room, member):
|
||||
"""Takes member out of the address index; the caller drops it from the room."""
|
||||
if self.by_addr.get(member.addr[:2], (None, None))[1] is member:
|
||||
del self.by_addr[member.addr[:2]]
|
||||
|
||||
def prune(self, now):
|
||||
"""Forgets the networks whose budgets are full again (as good as new): the tables hold only networks that sent
|
||||
something in the last seconds. And the refusals older than FAILED_TTL."""
|
||||
for table in (self.join_buckets, self.challenge_buckets):
|
||||
for k in [k for k, b in table.items() if b.full(now)]:
|
||||
del table[k]
|
||||
for k in [k for k, (_, when) in self.failed.items() if now - when >= FAILED_TTL]:
|
||||
del self.failed[k]
|
||||
|
||||
def note_join(self, net, bucket, what):
|
||||
"""With --log-joins, where a network's join budget went (the network itself only with --log-addrs)."""
|
||||
if self.log_joins:
|
||||
who = net if self.log_addrs else 'a network'
|
||||
self.log(f'joins of {who}: {what}, {bucket.tokens:.1f} left' if bucket else f'joins of {who}: {what}')
|
||||
|
||||
def bucket(self, table, key, rate, burst, now):
|
||||
"""The budget of network key, or None when MAX_NETWORKS networks are tracked and none of them is done."""
|
||||
b = table.get(key)
|
||||
if b is None:
|
||||
if len(self.join_buckets) + len(self.challenge_buckets) >= MAX_NETWORKS:
|
||||
self.prune(now)
|
||||
if len(self.join_buckets) + len(self.challenge_buckets) >= MAX_NETWORKS:
|
||||
return None
|
||||
b = table[key] = Bucket(rate, burst, now)
|
||||
return b
|
||||
|
||||
def take_challenge(self, addr, now, member=None, room=None):
|
||||
"""A challenge (or an answer to another protocol version) for addr: the relay's budget first (a flood it refuses
|
||||
leaves no state behind), then its network's. A CONNECT that carries a member's valid token (a rejoin) takes from
|
||||
that member's own challenges for its network or for the others (MEMBER_CHALLENGES), never from its packets, and
|
||||
then from the share set aside for rejoins (take_rejoin), which a flood without a token never reaches."""
|
||||
if member is not None:
|
||||
home = prefix_of(addr) == prefix_of(member.addr)
|
||||
mine = member.home_challenges if home else member.away_challenges
|
||||
return mine.take(1, now) and self.take_rejoin(room, member, home, now)
|
||||
if self.challenges is None:
|
||||
self.challenges = Bucket(self.challenge_rate, self.challenge_rate, now)
|
||||
self.challenges.refill(now)
|
||||
if self.challenges.tokens < 1:
|
||||
return False
|
||||
mine = self.bucket(self.challenge_buckets, prefix_of(addr), *self.prefix_challenge, now)
|
||||
return mine is not None and mine.take(1, now) and self.challenges.take(1, now)
|
||||
|
||||
def take_rejoin(self, room, member, home, now):
|
||||
"""A challenge from the share set aside for rejoins: rejoin_challenge_rate a second split equally among the open
|
||||
rooms, for CONNECTs from other networks than the member's (a member that moves, or someone with a token it saw),
|
||||
and as much again, split among the rooms and then among the eight member ids of each, for CONNECTs from the
|
||||
member's own network. Each holds MAX_MEMBERS at once at least. So the tokens of rooms someone made to flood with
|
||||
take only those rooms' shares, and nobody without the member's own network (a stranger with a token it saw, or
|
||||
another member of its room) takes the share of the member's own rejoin."""
|
||||
share = self.rejoin_challenge_rate / max(1, len(self.rooms))
|
||||
if home:
|
||||
share /= MAX_MEMBERS
|
||||
burst = max(share, MAX_MEMBERS)
|
||||
b = member.home_rejoins if home else room.rejoins
|
||||
if b is None:
|
||||
b = Bucket(share, burst, now)
|
||||
if home:
|
||||
member.home_rejoins = b
|
||||
else:
|
||||
room.rejoins = b
|
||||
else:
|
||||
b.refill(now)
|
||||
b.rate, b.burst = share, burst
|
||||
b.tokens = min(b.tokens, burst)
|
||||
return b.take(1, now)
|
||||
|
||||
def share_egress(self, now):
|
||||
"""Each room's share of the forwarding budget: equal among the rooms that forwarded to someone in the last ACTIVE
|
||||
seconds, so a busy room cannot take what another needs and rooms that send nothing (or to nobody) take nothing."""
|
||||
active = sum(1 for r in self.rooms.values() if r.forwarded_at is not None and now - r.forwarded_at < ACTIVE)
|
||||
share = self.egress_rate / max(1, active)
|
||||
for room in self.rooms.values():
|
||||
if room.egress is None:
|
||||
room.egress = Bucket(share, share, now)
|
||||
else:
|
||||
room.egress.refill(now)
|
||||
room.egress.rate, room.egress.burst = share, share
|
||||
room.egress.tokens = min(room.egress.tokens, share)
|
||||
|
||||
def take_egress(self, room, n, now):
|
||||
if self.egress is None:
|
||||
self.egress = Bucket(self.egress_rate, self.egress_rate, now)
|
||||
idle = room.forwarded_at is None or now - room.forwarded_at >= ACTIVE
|
||||
room.forwarded_at = now
|
||||
if room.egress is None or idle: # it counts among the rooms that share from now on
|
||||
self.share_egress(now)
|
||||
return room.egress.take(n, now) and self.egress.take(n, now)
|
||||
|
||||
def delete_room(self, room, why):
|
||||
self.rooms.pop(room.id, None)
|
||||
self.locators.pop(room.locator, None)
|
||||
left = self.prefix_rooms.get(room.prefix, 1) - 1
|
||||
if left > 0:
|
||||
self.prefix_rooms[room.prefix] = left
|
||||
else:
|
||||
self.prefix_rooms.pop(room.prefix, None)
|
||||
for m in list(room.members.values()):
|
||||
self.forget(room, m)
|
||||
self.log(f'room {room.label} closed, {why} ({len(self.rooms)} rooms)')
|
||||
|
||||
# -- datagrams
|
||||
|
||||
def handle(self, data, addr, now):
|
||||
self.stats['rx'] += 1
|
||||
dg = parse(data)
|
||||
if dg is None:
|
||||
self.stats['malformed'] += 1
|
||||
return []
|
||||
if dg.ver != PROTO:
|
||||
# a client of another protocol major: tell it so, if the answer is no larger than what it sent (a spoofed
|
||||
# source gains nothing) and the network's challenge budget allows
|
||||
out = self.error(addr, E_PROTO, 0, f'relay speaks protocol {PROTO}', None)
|
||||
return out if len(out[0][1]) <= len(data) and self.take_challenge(addr, now) else []
|
||||
if dg.type == T_CONNECT:
|
||||
return self.connect(dg, data, addr, now)
|
||||
room = self.rooms.get(dg.room)
|
||||
member = room.members.get(dg.src) if room else None
|
||||
if member is None or not member.live and dg.type != T_LEAVE:
|
||||
self.stats['no_member'] += 1
|
||||
return []
|
||||
if member.addr != addr:
|
||||
self.stats['wrong_addr'] += 1
|
||||
return []
|
||||
if not (member.pkts.take(1, now) and member.bytes.take(len(data), now)):
|
||||
self.stats['rate_limited'] += 1
|
||||
return []
|
||||
if not dg.verify(room.key):
|
||||
self.stats['bad_tag'] += 1
|
||||
return []
|
||||
out = []
|
||||
if not member.live: # dropped, its id reserved: a LEAVE gives the id up, anything else is ignored
|
||||
del room.members[member.id]
|
||||
return out
|
||||
member.last_seen = now
|
||||
if member.away:
|
||||
member.away = False
|
||||
out += self.roster_to_all(room, changed_at=now)
|
||||
if dg.type == T_PING and len(dg.payload) >= 4:
|
||||
out.append(self.to_member(room, member, T_PONG, dg.payload[:4]))
|
||||
elif dg.type == T_LEAVE:
|
||||
out += self.leave(room, member, now, 'left')
|
||||
elif dg.type == T_KICK:
|
||||
out += self.kick(room, member, dg, now)
|
||||
elif dg.type == T_DATA:
|
||||
out += self.forward(room, member, dg, data, now)
|
||||
return out
|
||||
|
||||
def forward(self, room, member, dg, data, now):
|
||||
if dg.dst == DST_ALL:
|
||||
targets = [m for m in room.live() if m is not member]
|
||||
else:
|
||||
target = room.members.get(dg.dst)
|
||||
targets = [target] if target and target.live and target is not member else []
|
||||
# each copy counts against the sender (its byte budget already paid one) and against the relay's
|
||||
if not targets: # to nobody: nothing to forward, and the room does not count among those that share the budget
|
||||
return []
|
||||
extra = len(data) * max(0, len(targets) - 1)
|
||||
if extra and not member.bytes.take(extra, now):
|
||||
self.stats['rate_limited'] += 1
|
||||
return []
|
||||
if not self.take_egress(room, len(data) * len(targets), now):
|
||||
self.stats['egress_limited'] += 1
|
||||
return []
|
||||
self.stats['forwarded'] += len(targets)
|
||||
return [(m.addr, data) for m in targets]
|
||||
|
||||
def kick(self, room, member, dg, now):
|
||||
k = parse_kick(dg.payload)
|
||||
if k is None or not member.maker:
|
||||
return []
|
||||
target = room.members.get(k[0])
|
||||
if target is None or target is member or target.session != k[1]:
|
||||
return [] # an earlier session of that id, or nobody: a KICK sent again later removes no one else
|
||||
room.banned.update((target.cid, target.addr))
|
||||
self.stats['kicks'] += 1
|
||||
out = []
|
||||
if target.live:
|
||||
out.append((target.addr, pack(T_ERROR, room.id, 0, target.id, self.next_seq(),
|
||||
pack_error(E_REMOVED, 0, 'removed by the room\'s maker'), room.key)))
|
||||
return out + self.leave(room, target, now, 'was removed by the maker')
|
||||
|
||||
def connect(self, dg, data, addr, now):
|
||||
c = parse_connect(dg.payload)
|
||||
if c is None:
|
||||
self.stats['malformed'] += 1
|
||||
return []
|
||||
intent, _flags, locator, cid, session, nonce, raw_name, cookie, token = c
|
||||
if intent not in (INTENT_CREATE, INTENT_JOIN):
|
||||
self.stats['malformed'] += 1
|
||||
return []
|
||||
key = locator # the tag proves the sender knows the locator, which an observer of a CONNECT does too
|
||||
room = self.locators.get(locator)
|
||||
rejoin = None
|
||||
if room is not None and any(token): # a member of the room again: its token is its proof
|
||||
rejoin = next((m for m in room.members.values() if m.cid == cid and hmac.compare_digest(m.token, token)), None)
|
||||
# no cookie needed only from the member's own address and in its own session: from another address the token
|
||||
# would aim the room's traffic at an address that never showed it receives there, and with another session
|
||||
# someone who saw the member's CONNECT on the way would take its place
|
||||
if (rejoin is None or rejoin.addr != addr or rejoin.session != session) and not self.cookie_ok(addr, cookie, now):
|
||||
# not yet proven to receive at its address: only a challenge, and only if that is no larger than the request
|
||||
if len(data) < CONNECT_MIN or not self.take_challenge(addr, now, rejoin, room):
|
||||
return []
|
||||
self.stats['challenges'] += 1
|
||||
return [(addr, pack(T_CHALLENGE, 0, 0, 0, self.next_seq(), pack_challenge(nonce, self.cookie(addr, int(now // 60))), key))]
|
||||
if not dg.verify(key):
|
||||
self.stats['bad_tag'] += 1
|
||||
return self.error(addr, E_BAD_TAG, nonce, 'bad tag', key)
|
||||
known = self.by_addr.get(addr[:2])
|
||||
if known and known[1].cid == cid and known[1].connect_seq == dg.seq and known[1].live and known[0] is room:
|
||||
known[1].last_seen = now # a CONNECT sent again because its answer was lost: the answer again
|
||||
return self.joined(known[0], known[1], nonce)
|
||||
net = prefix_of(addr)
|
||||
refusal = self.refusal(intent, room, rejoin, cid, addr, session, net)
|
||||
attempt = (cid, addr[:2], dg.seq, locator)
|
||||
if refusal is not None and self.failed.get(attempt, (None,))[0] == refusal[0]:
|
||||
# the same attempt again (an answer was lost, or it asks again for a room that is not there yet): the same
|
||||
# answer, without a charge, so that waiting does not use up the network's joins
|
||||
self.stats['repeats'] += 1
|
||||
return self.error(addr, refusal[0], nonce, refusal[1], key)
|
||||
bucket = self.bucket(self.join_buckets, net, *self.join_limit, now)
|
||||
if bucket is None or not bucket.take(1, now):
|
||||
self.stats['rate_limited'] += 1
|
||||
if self.log_joins or now - self.rate_said >= 10: # a line every 10 s at most: a flood does not fill the log
|
||||
self.rate_said = now
|
||||
self.log(f'a join refused: too many joins from {net if self.log_addrs else "one network"} ({self.stats["rate_limited"]} so far)')
|
||||
return self.error(addr, E_RATE, nonce, 'too many joins, try again later', key)
|
||||
self.stats['join_charges'] += 1
|
||||
if refusal is not None:
|
||||
self.failed.pop(attempt, None)
|
||||
self.failed[attempt] = (refusal[0], now)
|
||||
if len(self.failed) > FAILED_KEPT:
|
||||
del self.failed[next(iter(self.failed))]
|
||||
self.note_join(net, bucket, f'charged 1 for a refused attempt ({refusal[1]})')
|
||||
return self.error(addr, refusal[0], nonce, refusal[1], key)
|
||||
self.note_join(net, bucket, 'charged 1 for a ' + ('rejoin' if rejoin else 'join' if room else 'new room'))
|
||||
name = clean_name(raw_name) or b'Player'
|
||||
out = []
|
||||
if room is None:
|
||||
room = self.make_room(locator, net, now)
|
||||
room.use_session(session)
|
||||
if known and known[0] is not room:
|
||||
out += self.leave(known[0], known[1], now, 'moved to another room') # same address, new room: it left the old
|
||||
member = rejoin
|
||||
if member is None:
|
||||
mid = next((i for i in range(1, MAX_MEMBERS + 1) if i not in room.members), None)
|
||||
if mid is None: # every id is taken: a dropped member's id goes to the newcomer, the longest gone first
|
||||
gone = sorted((m for m in room.members.values() if not m.live), key=lambda m: m.last_seen)
|
||||
if not gone:
|
||||
return out + self.error(addr, E_ROOM_FULL, nonce, 'room full', key)
|
||||
mid = gone[0].id
|
||||
del room.members[mid]
|
||||
pkts, byts = self.new_member_limits(now)
|
||||
member = Member(mid, cid, session, name, addr, now, dg.seq, pkts, byts, maker=not room.members and intent == INTENT_CREATE)
|
||||
room.members[mid] = member
|
||||
if not member.maker:
|
||||
room.shared = True
|
||||
self.stats['joins'] += 1
|
||||
self.log(f'room {room.label}: {name.decode()} joined as member {mid} ({len(room.live())}/{MAX_MEMBERS}){self.where(addr)}')
|
||||
else: # the same member again (after the relay was silent, from a new address, under a new name or session)
|
||||
if member.addr != addr:
|
||||
self.forget(room, member)
|
||||
if not member.live:
|
||||
self.log(f'room {room.label}: {name.decode()} rejoined as member {member.id}{self.where(addr)}')
|
||||
self.stats['rejoins'] += 1
|
||||
member.addr, member.name, member.connect_seq, member.session = addr, name, dg.seq, session
|
||||
member.last_seen, member.away, member.live = now, False, True
|
||||
old = self.by_addr.get(addr[:2])
|
||||
if old and old[1] is not member: # another member of this room on the same address: that one is gone
|
||||
self.forget(old[0], old[1])
|
||||
old[0].members.pop(old[1].id, None)
|
||||
self.by_addr[addr[:2]] = (room, member)
|
||||
room.empty_since = None
|
||||
return out + self.joined(room, member, nonce) + self.roster_to_all(room, skip=member, changed_at=now)
|
||||
|
||||
def refusal(self, intent, room, rejoin, cid, addr, session, net):
|
||||
"""(reason, text) when this attempt is refused, None when it joins or makes the room; nothing is charged here."""
|
||||
if intent == INTENT_JOIN and room is None:
|
||||
return E_NO_ROOM, 'no such room'
|
||||
if intent == INTENT_CREATE and room is not None and rejoin is None:
|
||||
return E_ROOM_EXISTS, 'a room with this secret is open'
|
||||
if room is None:
|
||||
if len(self.rooms) >= self.max_rooms:
|
||||
return E_SERVER_FULL, 'server full'
|
||||
if self.prefix_rooms.get(net, 0) >= self.rooms_per_prefix:
|
||||
return E_RATE, 'too many rooms from this network'
|
||||
return None
|
||||
if cid in room.banned or addr in room.banned:
|
||||
return E_REMOVED, 'removed by the room\'s maker'
|
||||
if session in room.sessions and (rejoin is None or session != rejoin.session):
|
||||
# a session is the member's identity to the others (their games bind its signature to it): one that was in
|
||||
# the room before is someone else's, replayed
|
||||
return E_SESSION_USED, 'a session used in this room before'
|
||||
return None
|
||||
|
||||
def make_room(self, locator, prefix, now):
|
||||
rid = 0
|
||||
while not rid or rid in self.rooms:
|
||||
rid = secrets.randbits(32)
|
||||
room = Room(rid, locator, prefix, now)
|
||||
self.rooms[rid] = room
|
||||
self.locators[locator] = room
|
||||
self.prefix_rooms[prefix] = self.prefix_rooms.get(prefix, 0) + 1
|
||||
self.share_egress(now)
|
||||
self.stats['rooms_made'] += 1
|
||||
self.log(f'room {room.label} created ({len(self.rooms)} rooms)')
|
||||
return room
|
||||
|
||||
def leave(self, room, member, now, why):
|
||||
self.forget(room, member)
|
||||
room.members.pop(member.id, None)
|
||||
self.stats['leaves'] += 1
|
||||
self.log(f'room {room.label}: {member.name.decode()} (member {member.id}) {why} ({len(room.live())}/{MAX_MEMBERS})')
|
||||
if not room.live():
|
||||
room.empty_since = now
|
||||
if not room.shared: # nobody but its maker was ever in it: nothing to keep it for
|
||||
self.delete_room(room, 'its maker left before anyone joined')
|
||||
return []
|
||||
return self.roster_to_all(room, changed_at=now)
|
||||
|
||||
# -- time
|
||||
|
||||
def tick(self, now):
|
||||
out = []
|
||||
for room in list(self.rooms.values()):
|
||||
changed = False
|
||||
for m in list(room.members.values()):
|
||||
silent = now - m.last_seen
|
||||
if m.live and silent >= SILENT_DROP:
|
||||
m.live = False
|
||||
self.forget(room, m)
|
||||
self.stats['drops'] += 1
|
||||
self.log(f'room {room.label}: {m.name.decode()} (member {m.id}) dropped, silent {SILENT_DROP} s '
|
||||
f'({len(room.live())}/{MAX_MEMBERS})')
|
||||
changed = True
|
||||
elif m.live and not m.away and silent >= SILENT_AWAY:
|
||||
m.away = True
|
||||
changed = True
|
||||
elif not m.live and silent >= REJOIN_KEEP:
|
||||
del room.members[m.id]
|
||||
if not room.live() and room.empty_since is None:
|
||||
room.empty_since = now
|
||||
ttl = ROOM_EMPTY_TTL if room.shared else REJOIN_KEEP
|
||||
if room.empty_since is not None and now - room.empty_since >= ttl:
|
||||
self.delete_room(room, f'empty for {ttl} s')
|
||||
continue
|
||||
again = any(t <= now for t in room.again)
|
||||
if again:
|
||||
room.again = [t for t in room.again if t > now]
|
||||
if changed or again or now >= room.next_roster:
|
||||
out += self.roster_to_all(room, changed_at=now if changed else None)
|
||||
room.next_roster = now + ROSTER_EVERY
|
||||
if now >= self.next_prune:
|
||||
self.next_prune = now + 1
|
||||
self.prune(now)
|
||||
self.share_egress(now)
|
||||
return out
|
||||
|
||||
|
||||
# ---- the socket loop
|
||||
|
||||
def open_socket(host, port):
|
||||
family, _, _, _, sockaddr = socket.getaddrinfo(host, port, type=socket.SOCK_DGRAM, flags=socket.AI_PASSIVE)[0]
|
||||
s = socket.socket(family, socket.SOCK_DGRAM)
|
||||
if family == socket.AF_INET6:
|
||||
s.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0)
|
||||
s.bind(sockaddr)
|
||||
s.setblocking(False)
|
||||
return s
|
||||
|
||||
|
||||
def serve(relay, sock, stop=None, clock=time.monotonic):
|
||||
"""Runs relay on sock until stop (a threading.Event) is set."""
|
||||
|
||||
def send_all(datagrams):
|
||||
for to, out in datagrams:
|
||||
try:
|
||||
sock.sendto(out, to)
|
||||
except OSError:
|
||||
pass # a datagram that cannot go is a lost datagram
|
||||
|
||||
last_tick = clock()
|
||||
while not (stop and stop.is_set()):
|
||||
if select.select([sock], [], [], 0.1)[0]:
|
||||
for _ in range(512):
|
||||
try:
|
||||
data, addr = sock.recvfrom(2048)
|
||||
except OSError: # nothing more to read (or an ICMP error reported on Windows)
|
||||
break
|
||||
send_all(relay.handle(data, addr, clock()))
|
||||
now = clock()
|
||||
if now - last_tick >= 0.25:
|
||||
last_tick = now
|
||||
send_all(relay.tick(now))
|
||||
|
||||
|
||||
def run_in_thread(relay, host='127.0.0.1', port=0):
|
||||
"""Starts serve() on a new socket in a thread (tests): (thread, stop event, bound port)."""
|
||||
sock = open_socket(host, port)
|
||||
stop = threading.Event()
|
||||
t = threading.Thread(target=lambda: (serve(relay, sock, stop), sock.close()), daemon=True)
|
||||
t.start()
|
||||
return t, stop, sock.getsockname()[1]
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description='NX2navi co-op relay (see docs/coop.md)')
|
||||
ap.add_argument('--host', default='0.0.0.0', help='address to listen on (default 0.0.0.0; :: for IPv6 and IPv4)')
|
||||
ap.add_argument('--port', type=int, default=24680, help='UDP port (default 24680; 0 picks a free one)')
|
||||
ap.add_argument('--log-addrs', action='store_true', help="write the players' addresses to the log (off by default)")
|
||||
ap.add_argument('--max-rooms', type=int, default=1000, help='rooms at most (default 1000)')
|
||||
ap.add_argument('--rooms-per-network', type=int, default=8, help='open rooms one /24 or /48 may make (default 8)')
|
||||
ap.add_argument('--log-joins', action='store_true', help="write where each network's join budget goes to the log")
|
||||
a = ap.parse_args()
|
||||
|
||||
def log(text):
|
||||
print(time.strftime('%H:%M:%S ') + text, flush=True)
|
||||
|
||||
relay = Relay(max_rooms=a.max_rooms, log=log, log_addrs=a.log_addrs, rooms_per_prefix=a.rooms_per_network, log_joins=a.log_joins)
|
||||
sock = open_socket(a.host, a.port)
|
||||
log(f'coop_relay: listening on {a.host}:{sock.getsockname()[1]}')
|
||||
try:
|
||||
serve(relay, sock)
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -8,9 +8,10 @@ with --events and reads its progress lines).
|
||||
|
||||
The AppImage holds no game files: the kit's tools/, recomp/, runtime/ and docs, the shader translator built by
|
||||
tools/build_xlate.py (bin/nx2navi_xlate), NVIDIA's DLSS library with its notice (bin/, tools/dlss_pin.py: downloaded
|
||||
from NVIDIA's DLSS SDK repository and checked, or --dlss; not with --no-dlss), the pinned Vulkan Memory Allocator
|
||||
header (an AppImage made without the library downloads it after a build on machines with NVIDIA's driver: AppRun's
|
||||
get_dlss), and a relocatable CPython 3.14 (python-build-standalone, as `uv python install 3.14` puts in
|
||||
from NVIDIA's DLSS SDK repository and checked, or --dlss; not with --no-dlss: an AppImage made without the library
|
||||
downloads it after a build on machines with NVIDIA's driver, AppRun's get_dlss), the pinned Vulkan Memory Allocator
|
||||
header, the pinned Vulkan headers (toolchain/vulkan-headers, tools/vulkan_pin.py: the runtime names Vulkan extensions
|
||||
newer than the headers many distributions package), and a relocatable CPython 3.14 (python-build-standalone, as `uv python install 3.14` puts in
|
||||
~/.local/share/uv/python) with numpy, lz4, zstandard, fonttools and pillow, and with --prompt-art the button art
|
||||
make_game.py's prompts step draws from (as make_kit.py --prompt-art: the pictures tools/prompt_art.py art_files names,
|
||||
into the kit's prompt-art folder; without it that step is skipped and the game keeps the Switch prompts). zbic, the GPL
|
||||
@@ -18,7 +19,9 @@ package that unpacks the game's program files, is not bundled: the AppImage inst
|
||||
~/.local/share/nx2navi/pydeps the first time it runs, as the Windows launcher downloads it.
|
||||
|
||||
The compiler and the libraries come from the system (README-BuildKit.md, "Linux"): the AppImage checks for clang,
|
||||
clang++, lld and the headers before it starts, and names the packages for Arch, Fedora and Debian/Ubuntu.
|
||||
clang++, lld and the headers before it starts, and names the packages for Arch, Fedora and Debian/Ubuntu. Whether
|
||||
they are new enough is found out by trying them: tools/make_game.py checks the runtime's sources with that compiler
|
||||
and links against those libraries before the build's long steps (preflight).
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
@@ -72,7 +75,8 @@ ev step deps '"state":"start"'
|
||||
|
||||
missing=()
|
||||
for tool in clang clang++ ld.lld; do command -v "$tool" >/dev/null || missing+=("$tool"); done
|
||||
for h in SDL3/SDL.h vulkan/vulkan.h opus/opus.h vpx/vpx_decoder.h dav1d/dav1d.h zstd.h; do
|
||||
# The Vulkan headers come with this file (the kit's toolchain/vulkan-headers): a distribution's may be too old.
|
||||
for h in SDL3/SDL.h opus/opus.h vpx/vpx_decoder.h dav1d/dav1d.h zstd.h; do
|
||||
[ -e "/usr/include/$h" ] || [ -e "/usr/local/include/$h" ] || missing+=("$h")
|
||||
done
|
||||
if [ ${#missing[@]} -gt 0 ]; then
|
||||
@@ -91,7 +95,7 @@ EOF
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! PYTHONPATH="$PYDEPS" "$PY" -c 'import zbic' 2>/dev/null; then
|
||||
if ! "$PY" -I -c 'import sys; sys.path[:0] = [sys.argv[1]]; import zbic' "$PYDEPS" 2>/dev/null; then
|
||||
echo "Installing zbic (GPL-2.0, unpacks the game's program files) into $PYDEPS ..."
|
||||
"$PY" -m pip install --quiet --disable-pip-version-check --target "$PYDEPS" zbic || {
|
||||
echo "error: pip could not install zbic (it needs the internet once)" >&2
|
||||
@@ -226,6 +230,10 @@ def main():
|
||||
dlss_pin.place(dlss, kit / 'bin', linux=True)
|
||||
(kit / 'toolchain' / 'vma').mkdir(parents=True)
|
||||
shutil.copy2(ROOT / 'toolchain' / 'vma' / 'vk_mem_alloc.h', kit / 'toolchain' / 'vma' / 'vk_mem_alloc.h')
|
||||
# The Vulkan headers the runtime is written against, ahead of the system's on the player's machine
|
||||
# (tools/build_game.py vulkan_include finds them in the kit): fetched and checked here when not there yet
|
||||
import vulkan_pin
|
||||
shutil.copytree(vulkan_pin.resolve(ROOT / vulkan_pin.FOLDER), kit / vulkan_pin.FOLDER)
|
||||
if a.prompt_art:
|
||||
import prompt_art
|
||||
n, size = prompt_art.copy_art(a.prompt_art, kit / prompt_art.FOLDER)
|
||||
|
||||
@@ -26,7 +26,8 @@ Steps, each skipped when a run before finished it for the same inputs (state in
|
||||
tools, those lines), so it runs again only when one of them changed or a folder is gone, and running
|
||||
it makes no other step run again. A failure here leaves the game and its shaders complete.
|
||||
8. scan tools/romfs_check.py decompresses every file of the game data once, writes <out>/romfs-check.json and
|
||||
makes stand-ins for damaged textures in <out>/mods. Its key names the game files and the tool, and it
|
||||
makes stand-ins for damaged textures in <out>/mods. Its key names the game files and the check's
|
||||
version (SCAN_TOOL, changed by hand: an edit of the tool alone does not run it again), and it
|
||||
is last: nothing of it runs while the game does.
|
||||
Without art the step is skipped and the game shows its own prompts, as before
|
||||
|
||||
@@ -35,10 +36,12 @@ therefore lifts and compiles again but keeps the unpacked game files, and --rebu
|
||||
compile step keeps every object with the dependency list clang wrote for it (build-cache/obj/objects.json) and
|
||||
compiles a file again only when it, a header it includes, the flags or the compiler changed; the kit's own folder
|
||||
is not part of that key, so a kit in another folder (the launcher keeps each version in its own) reuses the objects.
|
||||
The list goes to disk every 25 objects while the compile runs, so a compile that is cut short keeps what it finished.
|
||||
A kit with lib/libnx2navirt.a built by --make-runtime-lib for the same runtime/ and the same compiler links the
|
||||
runtime from it and compiles only the runtime files that include headers written per install. The new ocarina.exe
|
||||
replaces the old one only once it links (with lld when the toolchain has it), so a failed build leaves the last good
|
||||
one in place. Compiles run on every logical processor, the largest files first. The lifted code is built for this
|
||||
one in place. Compiles run on every logical processor, or on as many as the free memory allows (default_jobs; --jobs
|
||||
sets the number), the largest files first. The lifted code is built for this
|
||||
PC's x86-64 level (cpu_level), with MSYS2 CLANG64's clang when it sits beside the UCRT64 folder and reports the same
|
||||
version (fast_clang: the same objects, about a fifth faster).
|
||||
|
||||
@@ -63,6 +66,7 @@ by a JSON object (launcher/build.py reads them): {"ev": "step", "step": ..., "st
|
||||
"""
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import errno
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
@@ -74,6 +78,7 @@ import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
|
||||
KIT = Path(__file__).resolve().parents[1]
|
||||
@@ -91,11 +96,28 @@ STEPS = ('exefs', 'romfs', 'lift', 'compile', 'shaders', 'prompts', 'scan')
|
||||
# Steps whose key names every input they read rather than the step before them: a step before them that runs does
|
||||
# not make them run again, and they make no step after them run again.
|
||||
OWN_KEY_STEPS = ('prompts', 'scan')
|
||||
# (a step that runs, a later step that reads nothing it writes): the later step is not made to run again. The lift
|
||||
# reads only the unpacked ExeFS (step_key), so the RomFS step leaves it alone: an install that was cut short while the
|
||||
# image was still being packed, the lift beside it already done, packs the image again but does not lift again.
|
||||
UNAFFECTED = {('romfs', 'lift')}
|
||||
# The Python packages tools/pad_prompts.py imports (pip name: import name), checked before a build with button art.
|
||||
PROMPT_PACKAGES = {'zstandard': 'zstandard', 'fonttools': 'fontTools', 'pillow': 'PIL'}
|
||||
# The line tools/pad_prompts.py prints for each overlay folder it wrote.
|
||||
PROMPTS_WRITTEN = re.compile(r'^.+: \d+ files, the font grew by -?\d+ bytes$')
|
||||
SCAN_CHECKED = re.compile(r'^checked (\d+) of (\d+)$') # tools/romfs_check.py's progress lines
|
||||
# What stands for tools/romfs_check.py in the scan step's key (step_key). The key held the sha256 of the script's
|
||||
# bytes, so any edit of the script, a comment's too, made every install check its 26 GB of game files again, four
|
||||
# minutes. It is this constant now, changed by hand when a change to the script can change what the check finds
|
||||
# or writes, and only then. Its value is the sha256 the script had in 0.7.0 and 0.7.1, the releases that brought the
|
||||
# check, so the key of every install that passed it stays what it is. tests/launcher_test.py
|
||||
# test_the_file_checks_logic_is_the_recorded_one fails when the script's logic changes, to ask this question.
|
||||
SCAN_TOOL = '55ea0c150aaf9dac93a2e6fd3b4ca11e1c572b5eee8b6bd74960535964384634'
|
||||
# The same for the tools that draw the button prompts (tools/pad_prompts.py, prompt_art.py, button_icons.py) in the
|
||||
# prompts step's key, which held the digest of the three scripts' bytes: two of four releases changed a script, and
|
||||
# each time every install drew its prompts again, a minute. Changed by hand when a change to the scripts can change
|
||||
# what they write. Its value is their digest in 0.7.0 and 0.7.1, so the key of an install made by those stays what it
|
||||
# is. tests/launcher_test.py test_the_prompt_tools_logic_is_the_recorded_one asks the question when the logic changes.
|
||||
PROMPT_TOOLS = 'ad59f2c4533f6ea2630b916ef5f3695f72daffe9b2b7663eea4168a4f08e4249'
|
||||
|
||||
# Bring-up fixes in the lifted SDK, each anchored on exact generated text that must occur once.
|
||||
LIFT_FIXES = [
|
||||
@@ -114,7 +136,7 @@ LIFT_FIXES = [
|
||||
]
|
||||
WIN = os.name == 'nt'
|
||||
if WIN:
|
||||
LINK_LIBS = ['-lvulkan-1', '-lSDL3', '-lopus', '-lvpx', '-l:libdav1d.a', '-lzstd', '-l:libatomic.a', '-lbcrypt',
|
||||
LINK_LIBS = ['-lvulkan-1', '-lSDL3', '-lopus', '-lvpx', '-l:libdav1d.a', '-lzstd', '-l:libatomic.a', '-lbcrypt', '-lws2_32',
|
||||
'-limm32', '-luuid', '-lole32', '-luser32', '-lshell32', '-lgdi32', '-ladvapi32', '-lwinmm']
|
||||
else: # tools/build_game.py's Linux libraries, plus dav1d (the AV1 movies) and libatomic
|
||||
LINK_LIBS = ['-lvulkan', '-lSDL3', '-lopus', '-lvpx', '-ldav1d', '-lzstd', '-l:libstdc++.a', '-latomic',
|
||||
@@ -129,7 +151,7 @@ RUNTIME_LIB = 'libnx2navirt.a'
|
||||
FFX4_DLLS = ('amd_fidelityfx_loader_vk.dll', 'amd_fidelityfx_upscaler_vk.dll') # AMD's FSR SDK 2.4, for FSR 4
|
||||
FFX4_FG_DLL = 'amd_fidelityfx_framegeneration_vk.dll' # ... and its frame generation, with the two above
|
||||
RUNTIME_PATTERNS = ('*.c', '*.cpp', 'platform/*.c', 'ui/*.cpp', 'third_party/imgui/*.cpp',
|
||||
'third_party/imgui/backends/*.cpp')
|
||||
'third_party/imgui/backends/*.cpp', 'third_party/monocypher/*.c')
|
||||
INCLUDE = re.compile(r'^[ \t]*#[ \t]*include[ \t]+"([^"]+)"', re.M)
|
||||
|
||||
# Runs a script the way `python script.py` does, with the script's folder first on sys.path. The launcher's
|
||||
@@ -162,6 +184,16 @@ def fail(msg):
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def error_text(e):
|
||||
"""An exception in a line a player can read: what the system said and about which file for a file error, the
|
||||
full disk named as such, else the exception's kind and message."""
|
||||
if isinstance(e, OSError) and e.strerror:
|
||||
if e.errno == errno.ENOSPC:
|
||||
return f'the disk is full ({e.filename})' if e.filename else 'the disk is full'
|
||||
return e.strerror + (f': {e.filename}' if e.filename else '')
|
||||
return f'{type(e).__name__}: {e}' if str(e) else type(e).__name__
|
||||
|
||||
|
||||
class Background:
|
||||
"""One step on a thread of its own. join() waits for it and raises its failure (fail's SystemExit included) in
|
||||
the caller; failed() tells the caller early, so it can stop work that would be thrown away."""
|
||||
@@ -370,6 +402,19 @@ def exefs_digest(out):
|
||||
ROMFS_READ_AHEAD = 64 << 20 # total file bytes requested from workers in one group
|
||||
|
||||
|
||||
def unreadable(rel, err):
|
||||
"""Ends the build on a game file that could not be read while the RomFS image was packed, naming the file. A
|
||||
file that does not match the dump's own checksum (nup.NupError) means the dump is damaged: a copy that went
|
||||
wrong or a dump that was cut short, which only a new copy or a new dump mends."""
|
||||
if isinstance(err, nup.NupError):
|
||||
why = str(err).split(': ', 1)[-1] # without nup.py's "file <number>: ", which says nothing to a player
|
||||
if why.startswith('damaged (') and why.endswith(')'):
|
||||
why = why[len('damaged ('):-1]
|
||||
fail(f'the dump is damaged: {rel}: {why}. Copy the dump to this PC again, or dump the game again, and then '
|
||||
'try again')
|
||||
fail(f'cannot read {rel} from the game files: {error_text(err)}')
|
||||
|
||||
|
||||
def romfs_read_groups(files):
|
||||
group, buffered = [], 0
|
||||
for entry in files:
|
||||
@@ -389,38 +434,71 @@ def step_romfs(src, out, jobs):
|
||||
dst.mkdir(parents=True, exist_ok=True)
|
||||
files = sorted(((rel, ref, src.romfs_size(ref)) for rel, ref in src.romfs), key=lambda e: e[0].encode())
|
||||
total = sum(size for _, _, size in files)
|
||||
# What a run that was cut short (Cancel, a full disk, a power cut) left of the image goes first: this run
|
||||
# writes the image anew, and counted as used space its up to 26 GB made a retry on a nearly full disk fail the
|
||||
# check below although the retry needs no more room than the first try.
|
||||
part = dst / 'romfs.bin.part'
|
||||
try:
|
||||
part.unlink(missing_ok=True)
|
||||
except OSError as e:
|
||||
fail(f'cannot remove {part}, left by an earlier run ({e.strerror or e}): close whatever has it open')
|
||||
free = shutil.disk_usage(dst).free
|
||||
if free < total + (1 << 30):
|
||||
fail(f'{dst} needs {total / 1e9:.1f} GB free for the RomFS image, it has {free / 1e9:.1f} GB')
|
||||
say(f' {len(files)} files, {total / 1e9:.1f} GB')
|
||||
entries, h, pos, done, t0 = [], hashlib.sha256(), 0, 0, time.time()
|
||||
part = dst / 'romfs.bin.part'
|
||||
emit('progress', step='romfs', done=0, total=total, unit='bytes')
|
||||
|
||||
def load(e): # small files are read whole on a worker; big ones are streamed by the writer
|
||||
_, ref, size = e
|
||||
return None if size > ROMFS_READ_AHEAD else b''.join(src.romfs_chunks(ref))
|
||||
if size > ROMFS_READ_AHEAD:
|
||||
return None
|
||||
try:
|
||||
return b''.join(src.romfs_chunks(ref))
|
||||
except (nup.NupError, OSError) as err: # handed to the writer, which knows the file's name
|
||||
return err
|
||||
|
||||
with open(part, 'wb', buffering=16 << 20) as img, concurrent.futures.ThreadPoolExecutor(jobs) as ex:
|
||||
for batch, group in enumerate(romfs_read_groups(files)):
|
||||
for (rel, ref, size), data in zip(group, ex.map(load, group)):
|
||||
pad = -pos % ALIGN
|
||||
if pad:
|
||||
img.write(b'\0' * pad)
|
||||
h.update(b'\0' * pad)
|
||||
pos += pad
|
||||
offset = pos
|
||||
for chunk in ((data,) if data is not None else src.romfs_chunks(ref)):
|
||||
img.write(chunk)
|
||||
h.update(chunk)
|
||||
pos += len(chunk)
|
||||
if pos - offset != size:
|
||||
fail(f'{rel} changed size while packing the RomFS: expected {size} bytes, read {pos - offset}')
|
||||
entries.append({'path': rel, 'offset': offset, 'size': size})
|
||||
done += size
|
||||
emit('progress', step='romfs', done=done, total=total, unit='bytes')
|
||||
if batch % 40 == 0:
|
||||
say(f' RomFS {done / 1e9:5.1f} of {total / 1e9:.1f} GB, {time.time() - t0:.0f} s')
|
||||
def pieces(rel, ref, data):
|
||||
"""The file's bytes; a file of the game files that cannot be read, or does not match its checksum in the
|
||||
dump, ends the step with its name (unreadable)."""
|
||||
if isinstance(data, Exception):
|
||||
unreadable(rel, data)
|
||||
if data is not None:
|
||||
yield data
|
||||
return
|
||||
try:
|
||||
yield from src.romfs_chunks(ref)
|
||||
except (nup.NupError, OSError) as err:
|
||||
unreadable(rel, err)
|
||||
|
||||
try:
|
||||
with open(part, 'wb', buffering=16 << 20) as img, concurrent.futures.ThreadPoolExecutor(jobs) as ex:
|
||||
for batch, group in enumerate(romfs_read_groups(files)):
|
||||
for (rel, ref, size), data in zip(group, ex.map(load, group)):
|
||||
pad = -pos % ALIGN
|
||||
if pad:
|
||||
img.write(b'\0' * pad)
|
||||
h.update(b'\0' * pad)
|
||||
pos += pad
|
||||
offset = pos
|
||||
for chunk in pieces(rel, ref, data):
|
||||
img.write(chunk)
|
||||
h.update(chunk)
|
||||
pos += len(chunk)
|
||||
if pos - offset != size:
|
||||
fail(f'{rel} changed size while packing the RomFS: expected {size} bytes, read {pos - offset}')
|
||||
entries.append({'path': rel, 'offset': offset, 'size': size})
|
||||
done += size
|
||||
emit('progress', step='romfs', done=done, total=total, unit='bytes')
|
||||
if batch % 40 == 0:
|
||||
say(f' RomFS {done / 1e9:5.1f} of {total / 1e9:.1f} GB, {time.time() - t0:.0f} s')
|
||||
except BaseException:
|
||||
# a failed image is of no use, the next run writes it anew: up to 26 GB of it would only fill the disk
|
||||
try:
|
||||
part.unlink(missing_ok=True)
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
part.replace(dst / 'romfs.bin')
|
||||
index = {'status': 'pass', 'romfs_size': pos, 'romfs_sha256': h.hexdigest(), 'files': entries}
|
||||
(dst / 'index.json').write_text(json.dumps(index), encoding='utf-8')
|
||||
@@ -465,6 +543,15 @@ def step_lift(out, cache):
|
||||
|
||||
# ---------------------------------------------------------------- compile
|
||||
|
||||
def c_string(text):
|
||||
"""text as a C string literal for a source file in UTF-8, which modules.c is: JSON's escapes for the quote, the
|
||||
backslash and the control characters are C's too, and every other character stands as itself. With JSON's
|
||||
default, which writes each character outside ASCII as \\uXXXX, a character outside the Basic Multilingual Plane
|
||||
came out as two surrogate halves (\\ud83e\\uddda), which C refuses ("invalid universal character"): a game
|
||||
folder with an emoji or a rare Han character in its path failed at the end of the compile."""
|
||||
return json.dumps(text, ensure_ascii=False)
|
||||
|
||||
|
||||
def modules_c(lift, out, gen):
|
||||
layout = json.loads((lift / 'layout.json').read_text(encoding='utf-8'))
|
||||
L = ['#include "dispatch.h"', '#include "host.h"', '#include "layout.h"', '']
|
||||
@@ -483,13 +570,13 @@ def modules_c(lift, out, gen):
|
||||
L += [' return n;', '}', '', 'const RtFileHash rt_exefs_sha256[] = {']
|
||||
L += [f' {{"{name}", "{sha256_file(out / "exefs" / name)}"}},' for name in EXEFS_FILES]
|
||||
L += ['};', f'const int rt_exefs_sha256_count = {len(EXEFS_FILES)};', '',
|
||||
f'const char rt_default_exefs[] = {json.dumps(str(out / "exefs"))};',
|
||||
f'const char rt_default_romfs[] = {json.dumps(str(out / "romfs"))};',
|
||||
f'const char rt_default_profile[] = {json.dumps(str(out / "profile"))};',
|
||||
f'const char rt_default_exefs[] = {c_string(str(out / "exefs"))};',
|
||||
f'const char rt_default_romfs[] = {c_string(str(out / "romfs"))};',
|
||||
f'const char rt_default_profile[] = {c_string(str(out / "profile"))};',
|
||||
'const int rt_game_version = 111;',
|
||||
'const unsigned long long rt_app_pool_bytes = 9809ull << 20; /* the Switch 2 application pool */',
|
||||
'const int rt_nvn_api_version[2] = {231, 40}; /* NVN2: main asks for major 231, minor 40 or later */',
|
||||
f'const char rt_port_version[] = {json.dumps(kit_version())};']
|
||||
f'const char rt_port_version[] = {c_string(kit_version())};']
|
||||
p = gen / 'modules.c'
|
||||
write_if_changed(p, '\n'.join(L) + '\n')
|
||||
if WIN: # on Linux ui/menu_sdl.cpp defines it
|
||||
@@ -536,7 +623,9 @@ def pe_imports(path):
|
||||
|
||||
def copy_if_changed(src, dst):
|
||||
"""Atomically replace dst unless its contents match, even when source modification times are normalized.
|
||||
A same-directory temporary copy preserves modes, replaces read-only files on POSIX and leaves dst intact on copy failure."""
|
||||
A same-directory temporary copy preserves modes, replaces read-only files on POSIX and leaves dst intact on copy
|
||||
failure. The copy is a temporary file, not a file in a temporary folder: such a folder is made for its owner alone,
|
||||
and some network shares refuse that while they take ordinary files."""
|
||||
dst = Path(dst)
|
||||
s = os.stat(src)
|
||||
try:
|
||||
@@ -545,13 +634,21 @@ def copy_if_changed(src, dst):
|
||||
return False
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
replacement = None
|
||||
try:
|
||||
with tempfile.TemporaryDirectory(dir=dst.parent, prefix='.nx2navi-copy-') as tmp:
|
||||
replacement = Path(tmp) / dst.name
|
||||
shutil.copy2(src, replacement)
|
||||
os.replace(replacement, dst)
|
||||
fd, name = tempfile.mkstemp(dir=dst.parent, prefix='.nx2navi-copy-')
|
||||
os.close(fd)
|
||||
replacement = Path(name)
|
||||
shutil.copy2(src, replacement)
|
||||
os.replace(replacement, dst)
|
||||
except PermissionError:
|
||||
fail(f'cannot replace {dst}: close the game and check output-directory permissions')
|
||||
finally:
|
||||
if replacement is not None: # still there only when the copy or the replace failed
|
||||
try:
|
||||
replacement.unlink(missing_ok=True)
|
||||
except OSError:
|
||||
pass
|
||||
return True
|
||||
|
||||
|
||||
@@ -773,6 +870,8 @@ class ObjectCache:
|
||||
return 1
|
||||
|
||||
def save(self):
|
||||
"""Whole or not at all: written beside the file and moved over it, so a process that ends in the middle
|
||||
(Cancel) leaves the last complete objects.json."""
|
||||
tmp = self.path.with_suffix('.tmp')
|
||||
tmp.write_text(json.dumps(self.entries), encoding='utf-8')
|
||||
tmp.replace(self.path)
|
||||
@@ -834,6 +933,65 @@ def earlier_kits():
|
||||
return sorted(kits)
|
||||
|
||||
|
||||
# Peak memory of one compile (tools/build_game.py JOB_MEM: 120 to 180 MiB for a chunk of lifted code, 568 MiB for the
|
||||
# largest file), and the share of the free memory the compiles may take in all; the rest stays with the system, the
|
||||
# RomFS step beside them and whatever else the player runs.
|
||||
JOB_MEM = 768 << 20
|
||||
JOB_MEM_SHARE = 0.8
|
||||
|
||||
|
||||
def default_jobs(cpus=None, free=None):
|
||||
"""(compiles at a time, why) without --jobs: one per logical processor, but no more than JOB_MEM_SHARE of the
|
||||
free memory holds at JOB_MEM each, and at least one. One per processor alone took no account of memory: on a
|
||||
PC with many threads and little free memory the compiles filled it, the largest files first as they start
|
||||
together. cpus, free: the processor count and the free bytes instead of this machine's (tests); without a
|
||||
reading of the free memory (tools/hostmem.py) the processor count alone decides, as before."""
|
||||
cpus = cpus or os.cpu_count() or 4
|
||||
if free is None:
|
||||
try:
|
||||
import hostmem
|
||||
free = hostmem.available_ram()
|
||||
except (ImportError, OSError, AttributeError):
|
||||
free = None
|
||||
if free is None:
|
||||
return cpus, f'{cpus} processors, free memory unknown'
|
||||
by_mem = max(1, int(free * JOB_MEM_SHARE) // JOB_MEM)
|
||||
return min(cpus, by_mem), f'{cpus} processors, {free / 2 ** 30:.1f} GiB of free memory allow {by_mem}'
|
||||
|
||||
|
||||
SOURCE_WAIT, SOURCE_PAUSE = 60, 2 # seconds source_size waits for a drive in all, and between two tries
|
||||
|
||||
|
||||
def source_size(src):
|
||||
"""The size of a source file about to be compiled. The sources sit in the build cache beside the game, which a
|
||||
player may keep on a network share, and a share that loses its connection under the build's load answers 'not there'
|
||||
until it is back: while the source's folder does not answer either, the question is asked again for up to
|
||||
SOURCE_WAIT seconds. A folder that answers without the file means the file is gone, which no wait brings back."""
|
||||
src = Path(src)
|
||||
end = time.monotonic() + SOURCE_WAIT
|
||||
told = False
|
||||
while True:
|
||||
try:
|
||||
return src.stat().st_size
|
||||
except OSError as e:
|
||||
away = not src.parent.is_dir()
|
||||
if not away or time.monotonic() >= end:
|
||||
fail(f'cannot read {src} ({e.strerror or e}): '
|
||||
+ ('the drive or network share with the game folder stopped answering; check it and run the '
|
||||
'build again' if away else
|
||||
'the file is gone from the build cache; delete the lift folder in the build cache and run '
|
||||
'the build again, which makes it anew'))
|
||||
if not told:
|
||||
say(f' {src.parent} does not answer; waiting up to {SOURCE_WAIT} s for the drive to come back')
|
||||
told = True
|
||||
time.sleep(SOURCE_PAUSE)
|
||||
|
||||
|
||||
# compile_all writes the object cache to disk after this many more objects, and at most this often (seconds): the
|
||||
# cache of a whole game is some megabytes, and writing it after every object would take longer than the compiles.
|
||||
CACHE_SAVE_EVERY, CACHE_SAVE_PAUSE = 25, 2.0
|
||||
|
||||
|
||||
def compile_all(srcs, cmd_for, cache, jobs, env, progress=None, salt='', stop=None, alts=None):
|
||||
"""Compile each (source, object) whose object is not current in cache. cmd_for(src, dst) -> argv without the
|
||||
dependency-file options; salt identifies the compiler. Returns (compiled, reused, failures) with failures as
|
||||
@@ -841,7 +999,10 @@ def compile_all(srcs, cmd_for, cache, jobs, env, progress=None, salt='', stop=No
|
||||
compile time grows with size, and a big file that started last would run on alone after the others finished.
|
||||
stop() true makes the compiles not yet started end at once, unrecorded (a step beside this one failed).
|
||||
alts(src, dst) -> (keys, kits): the keys earlier versions gave the same compile, and the folders of their kits; an
|
||||
object recorded under one of them is taken over (ObjectCache.adopt) instead of compiled again."""
|
||||
object recorded under one of them is taken over (ObjectCache.adopt) instead of compiled again.
|
||||
Each object is recorded when its compile ends, whichever started first, and the cache goes to disk every
|
||||
CACHE_SAVE_EVERY objects: a compile that is cut short (Cancel ends the process, a crash, a power cut) keeps the
|
||||
objects finished until then, where it used to keep none and the next run compiled the whole game again."""
|
||||
todo, keys = [], {}
|
||||
adopted = 0
|
||||
for src, dst in srcs:
|
||||
@@ -855,7 +1016,7 @@ def compile_all(srcs, cmd_for, cache, jobs, env, progress=None, salt='', stop=No
|
||||
reused = len(srcs) - len(todo)
|
||||
if adopted:
|
||||
say(f' {adopted} objects an earlier version compiled are kept: the same compile of the same sources')
|
||||
todo.sort(key=lambda job: job[0].stat().st_size, reverse=True)
|
||||
todo.sort(key=lambda job: source_size(job[0]), reverse=True)
|
||||
|
||||
def cc(job):
|
||||
src, dst = job
|
||||
@@ -868,10 +1029,14 @@ def compile_all(srcs, cmd_for, cache, jobs, env, progress=None, salt='', stop=No
|
||||
return src, dst, dep, p.returncode, p.stderr[-1500:]
|
||||
|
||||
done, bad = reused, []
|
||||
unsaved, saved_at = 0, time.monotonic()
|
||||
if progress:
|
||||
progress(done, len(srcs))
|
||||
with concurrent.futures.ThreadPoolExecutor(max(1, jobs)) as ex:
|
||||
for src, dst, dep, rc, err in ex.map(cc, todo):
|
||||
# submitted largest first, taken as they end: in submission order every result would wait for the largest
|
||||
# file's, and nothing would be recorded during the minutes that one compiles
|
||||
for result in concurrent.futures.as_completed([ex.submit(cc, job) for job in todo]):
|
||||
src, dst, dep, rc, err = result.result()
|
||||
done += 1
|
||||
if rc is None: # not started (stop)
|
||||
continue
|
||||
@@ -880,12 +1045,210 @@ def compile_all(srcs, cmd_for, cache, jobs, env, progress=None, salt='', stop=No
|
||||
cache.forget(dst)
|
||||
else:
|
||||
cache.record(dst, keys[dst], dep, src)
|
||||
unsaved += 1
|
||||
if unsaved >= CACHE_SAVE_EVERY and time.monotonic() - saved_at >= CACHE_SAVE_PAUSE:
|
||||
try:
|
||||
cache.save()
|
||||
except OSError: # the save at the end still reports a disk that cannot be written
|
||||
pass
|
||||
unsaved, saved_at = 0, time.monotonic()
|
||||
if progress:
|
||||
progress(done, len(srcs))
|
||||
cache.save()
|
||||
return len(todo), reused, bad
|
||||
|
||||
|
||||
def report_failures(bad):
|
||||
"""The files that did not compile, to stderr: the compiler's errors of the first three, and the names of all of
|
||||
them, so that a paste of the build's end says which they were."""
|
||||
for src, err in bad[:3]:
|
||||
print(f'--- {src}\n{err}', file=sys.stderr)
|
||||
print('did not compile: ' + ', '.join(Path(src).name for src, _ in bad), file=sys.stderr, flush=True)
|
||||
|
||||
|
||||
def runtime_headers(rt):
|
||||
"""The headers tools/build_shaders.py builds into runtime/ (the runtime's own shaders, the menu font's bytes): in
|
||||
the repository as source only, so from a source checkout they are not there until they are built, and every
|
||||
runtime file that includes one would fail to compile. A kit carries them built; a release's kit carries neither
|
||||
tools/build_shaders.py nor the compilers it calls, and skips this."""
|
||||
try:
|
||||
import build_shaders
|
||||
except ModuleNotFoundError:
|
||||
return
|
||||
stale = build_shaders.stale(rt)
|
||||
if stale:
|
||||
say(f' building {len(stale)} of the runtime\'s own headers (runtime/*_spv.h, the menu font)')
|
||||
build_shaders.build(rt, stale)
|
||||
|
||||
|
||||
def preflight(msys, jobs, sources=None):
|
||||
"""Linux, before the long steps of a build whose compile step will run: this machine's compiler, headers and
|
||||
libraries are tried on the runtime itself. Every runtime source that needs no header of this install is checked
|
||||
with the compile step's own command (clang -fsyntax-only), and an empty program is linked against the libraries
|
||||
the game links. The build takes the compiler, the linker and the libraries from the system (only the Vulkan
|
||||
headers and VMA come with the kit), and a distribution's older compiler, older headers or a missing development
|
||||
package stopped the compile step only after the RomFS image and the lift, ten minutes and more into the build.
|
||||
sources: the files to check instead (tests)."""
|
||||
clang, clangxx, env, vma = toolchain(msys)
|
||||
version = clang_version(clang, env)
|
||||
say(f'check: the compiler is {version}')
|
||||
rt = KIT / 'runtime'
|
||||
runtime_headers(rt)
|
||||
inc = ['-I', str(rt), '-I', str(rt / 'third_party' / 'imgui'), '-I', str(vma)]
|
||||
vulkan = vulkan_headers(clang, env)
|
||||
cmd_for = runtime_cmd(clang, clangxx, inc, first=vulkan)
|
||||
if sources is None:
|
||||
tainted = per_install_names()
|
||||
sources = [p for p, _ in runtime_sources() if not per_install(p, tainted)]
|
||||
|
||||
def check(src):
|
||||
cmd = [c for c in cmd_for(src, Path('unused.o')) if c not in ('-c', '-o', 'unused.o')]
|
||||
p = subprocess.run([cmd[0], '-fsyntax-only', *cmd[1:]], capture_output=True, text=True, env=env, errors='replace')
|
||||
return src, p.returncode, p.stderr[-1500:]
|
||||
|
||||
t0 = time.time()
|
||||
with concurrent.futures.ThreadPoolExecutor(max(1, jobs)) as ex:
|
||||
bad = [(src, err) for src, rc, err in ex.map(check, sources) if rc]
|
||||
if bad:
|
||||
report_failures(bad)
|
||||
lacking = sorted(set(re.findall(r"fatal error: '([^']+)' file not found", ''.join(e for _, e in bad))))
|
||||
fail(f'{len(bad)} of the runtime\'s {len(sources)} files do not compile on this machine, so the build would stop '
|
||||
f'in its compile step. ' +
|
||||
(f'These headers are missing: {", ".join(lacking)}. Install the development packages '
|
||||
f'README-BuildKit.md lists under "Linux".' if lacking else
|
||||
f'The compiler ({version}) or a library\'s headers are older than the runtime needs: the releases are '
|
||||
f'built with clang 23, and the Vulkan headers come ' +
|
||||
(f'from {vulkan[1]}' if vulkan else 'from the system') + '. The errors above name what is missing.'))
|
||||
with tempfile.TemporaryDirectory(prefix='nx2navi-check-') as tmp:
|
||||
probe = Path(tmp) / 'probe.c'
|
||||
probe.write_text('int main(void) { return 0; }\n', encoding='ascii')
|
||||
ld = (['-fuse-ld=lld'] if shutil.which('ld.lld') else []) + ['-no-pie']
|
||||
p = subprocess.run([str(clangxx), *ld, '-o', str(Path(tmp) / 'probe'), '-x', 'c', str(probe), '-x', 'none',
|
||||
*LINK_LIBS], capture_output=True, text=True, env=env, errors='replace')
|
||||
if p.returncode:
|
||||
print(p.stderr[-2000:], file=sys.stderr)
|
||||
fail('the libraries the game links are not all there (the linker\'s lines above name the missing ones): '
|
||||
'install the development packages README-BuildKit.md lists under "Linux". '
|
||||
'On Fedora the static C++ library is its own package, libstdc++-static.')
|
||||
say(f'check: {len(sources)} runtime files and the libraries pass, {time.time() - t0:.0f} s')
|
||||
|
||||
|
||||
def steps_to_run(state, identity, out, until=None):
|
||||
"""The steps from exefs to shaders this build will run, as main's loop decides it: a step whose key is not the
|
||||
recorded one, and every step after one that runs but those that read nothing of it (UNAFFECTED). until: the
|
||||
step the build stops after (--until). For the checks made before the long steps (space_needs, probe_toolchain)."""
|
||||
runs = []
|
||||
for name in STEPS[:STEPS.index('shaders') + 1]:
|
||||
if (any((earlier, name) not in UNAFFECTED for earlier in runs)
|
||||
or state['steps'].get(name) != step_key(name, identity, out)):
|
||||
runs.append(name)
|
||||
if name == until:
|
||||
break
|
||||
return runs
|
||||
|
||||
|
||||
GiB = 1 << 30
|
||||
# Disk space the steps after the RomFS image take in the game folder, the least each is known to need: a build that
|
||||
# has less is sure to stop, half an hour in when it is the shaders. Measured on a 0.7.1 game folder: the lifted C
|
||||
# 1.16 GiB, its objects 0.34 GiB and the new exe beside the old one 0.3 GiB; with the lifted C already there only
|
||||
# the changed files and the exe. A full shader preparation writes every translation as a loose file (a player's
|
||||
# pack of two translators' translations held 8,290 MiB, 327,560 files) and the game's shader programs and pipeline
|
||||
# cache, 0.84 GiB. Folding the translations into their pack takes as much again, but a pack that cannot be written
|
||||
# does not stop the build (pack_shader_cache), so it is not asked for here.
|
||||
SPACE_BUILD_CACHE, SPACE_REBUILD, SPACE_SHADERS = 2 * GiB, 1 * GiB, 4 * GiB
|
||||
SPACE_IMAGE_MARGIN = 1 * GiB # beside the RomFS image, as step_romfs asks
|
||||
|
||||
|
||||
def space_needs(runs, out, image_bytes=0, carried=False):
|
||||
"""[(what, bytes)] this build will write into the game folder, for the steps it runs (steps_to_run).
|
||||
image_bytes: the size of the RomFS image when that step runs; carried: the shaders step keeps the shaders
|
||||
prepared before (step_shaders update) and writes none. A game folder that already holds translations (a
|
||||
preparation that was cut short, or one repeated with the same translator) is not asked for the shaders' space: the
|
||||
preparation then writes only what is missing, which may be nothing, and asking for all of it would stop a build
|
||||
that has the room it needs."""
|
||||
needs = []
|
||||
if 'romfs' in runs:
|
||||
needs.append(('the RomFS image', image_bytes + SPACE_IMAGE_MARGIN))
|
||||
if 'compile' in runs:
|
||||
lifted = (out / 'build-cache' / 'lift' / 'layout.json').is_file()
|
||||
needs.append(('the build cache (the translated code and its compiled files)',
|
||||
SPACE_REBUILD if lifted else SPACE_BUILD_CACHE))
|
||||
if 'shaders' in runs and not carried and not shaders_begun(out):
|
||||
needs.append(('the prepared shaders', SPACE_SHADERS))
|
||||
return needs
|
||||
|
||||
|
||||
def shaders_begun(out):
|
||||
"""True when the game's shader cache holds a translation already: its pack, or a loose file."""
|
||||
cache = out / 'shader-cache'
|
||||
if (cache / 'shader-cache.pack').is_file():
|
||||
return True
|
||||
try:
|
||||
with os.scandir(cache) as it:
|
||||
return any(e.name.endswith('.spv') for e in it)
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def check_space(out, needs):
|
||||
"""Stops the build before its first long step when the game folder's disk cannot hold what the build will
|
||||
write (space_needs), naming each part and how much is missing. Only the RomFS image's space was looked at,
|
||||
and only when that step began: a disk that held the image but not the build cache or the shaders stopped the
|
||||
build in the compile or, after half an hour, in the shader preparation. What an earlier run left of the image
|
||||
counts as free: the RomFS step removes it (step_romfs)."""
|
||||
need = sum(n for _, n in needs)
|
||||
if not need:
|
||||
return
|
||||
free = shutil.disk_usage(out).free
|
||||
try:
|
||||
free += (out / 'romfs' / 'romfs.bin.part').stat().st_size
|
||||
except OSError:
|
||||
pass
|
||||
if free >= need:
|
||||
return
|
||||
parts = ', '.join(f'{n / 1e9:.1f} GB for {what}' for what, n in needs)
|
||||
fail(f'{out} has {free / 1e9:.1f} GB free and this build needs {need / 1e9:.1f} GB there: {parts}. Free '
|
||||
f'{(need - free) / 1e9:.1f} GB more on that disk, or build into a folder on another one')
|
||||
|
||||
|
||||
def probe_toolchain(msys, work):
|
||||
"""Windows, before the long steps of a build whose compile step will run: an empty program is compiled and
|
||||
linked with the installed toolchain, against the libraries the game links and with the linker the compile step
|
||||
uses, in the folder work. Build tools that are incomplete, or that security software keeps from starting or
|
||||
from writing an exe, stopped the build in its compile step, after the RomFS image and the lift, ten minutes and
|
||||
more in; this takes a second. (Linux tries its compiler on the runtime's own sources: preflight.)"""
|
||||
clang, clangxx, env, _ = toolchain(msys)
|
||||
repair = (f'If the launcher installed these tools, delete the folder {msys.parent} and try again: it unpacks '
|
||||
f'them anew. If you installed MSYS2 yourself, install the packages README-BuildKit.md lists')
|
||||
shutil.rmtree(work, ignore_errors=True)
|
||||
work.mkdir(parents=True)
|
||||
try:
|
||||
src, obj, exe = work / 'probe.c', work / 'probe.o', work / ('probe.exe' if WIN else 'probe')
|
||||
src.write_text('int main(void) { return 0; }\n', encoding='ascii')
|
||||
ld = ['-fuse-ld=lld'] if (msys / 'bin' / 'ld.lld.exe').is_file() else []
|
||||
steps = (('compile', clang, [str(clang), '-c', '-std=gnu11', '-O1', '-o', str(obj), str(src)]),
|
||||
('link', clangxx, [str(clangxx), *ld, '-o', str(exe), str(obj), '-L', str(msys / 'lib'), *LINK_LIBS]))
|
||||
for what, tool, cmd in steps:
|
||||
try:
|
||||
p = subprocess.run(cmd, capture_output=True, text=True, env=env, errors='replace')
|
||||
except OSError as e:
|
||||
fail(f'{tool} does not start ({error_text(e)}): the build tools are damaged, or security software '
|
||||
f'blocks them. {repair}')
|
||||
if not p.returncode:
|
||||
continue
|
||||
said = (p.stderr.strip().splitlines() or [f'exit code {p.returncode}'])[-1][-300:]
|
||||
if what == 'compile':
|
||||
fail(f'{tool} cannot compile a test program ({said}): the build tools are incomplete, or security '
|
||||
f'software blocks them. {repair}')
|
||||
lacking = sorted(set(re.findall(r'unable to find library (-l\S+)', p.stderr)))
|
||||
fail('the build tools cannot link the game: '
|
||||
+ (f'{msys / "lib"} lacks {", ".join(lacking)}' if lacking else f'the linker says: {said}')
|
||||
+ f'. {repair}')
|
||||
finally:
|
||||
shutil.rmtree(work, ignore_errors=True)
|
||||
say('check: the build tools compile and link a test program')
|
||||
|
||||
|
||||
def runtime_sources():
|
||||
"""(source, object name) of every runtime file this OS's build compiles: the Win32 files on Windows, the SDL
|
||||
window's (*_sdl.*) and platform/linux.c elsewhere (tools/build_game.py's foreign_os)."""
|
||||
@@ -987,8 +1350,10 @@ def fast_clang(msys, clang, env):
|
||||
return fast if mine and clang_ident(fast, env) == mine else None
|
||||
|
||||
|
||||
def runtime_cmd(clang, clangxx, inc, level=0, fast=None, msys=None):
|
||||
"""fast: fast_clang's compiler for the lifted code, with msys as its sysroot."""
|
||||
def runtime_cmd(clang, clangxx, inc, level=0, fast=None, msys=None, first=()):
|
||||
"""fast: fast_clang's compiler for the lifted code, with msys as its sysroot. first: include flags that go
|
||||
ahead of inc for the runtime's sources only (vulkan_headers); the lifted code includes none of those headers,
|
||||
and its command, which is part of its objects' keys, stays as it was."""
|
||||
def cmd_for(src, dst):
|
||||
cpp = src.suffix in ('.cpp', '.cxx')
|
||||
# the lifted code at -O1 (its size makes -O2 slow to build), the runtime at -O2; neither may assume strict
|
||||
@@ -1001,7 +1366,7 @@ def runtime_cmd(clang, clangxx, inc, level=0, fast=None, msys=None):
|
||||
# -fdeclspec: third_party/ffx_api/ffx_api.h declares its entry points __declspec(dllexport) unconditionally
|
||||
declspec = ['-fdeclspec'] if cpp and not WIN else []
|
||||
return [*cc, '-c', '-std=gnu++17' if cpp else '-std=gnu11', '-O1' if guest else '-O2',
|
||||
*march, '-fno-strict-aliasing', '-w', *declspec, *inc, '-o', str(dst), str(src)]
|
||||
*march, '-fno-strict-aliasing', '-w', *declspec, *([] if guest else first), *inc, '-o', str(dst), str(src)]
|
||||
return cmd_for
|
||||
|
||||
|
||||
@@ -1020,12 +1385,12 @@ def make_runtime_lib(dest, msys, jobs, obj=None):
|
||||
inc = ['-I', str(rt), '-I', str(rt / 'third_party' / 'imgui'), '-I', str(vma)]
|
||||
cache = ObjectCache(obj / 'objects.json', {'<kit>': KIT})
|
||||
t0 = time.time()
|
||||
compiled, reused, bad = compile_all(srcs, runtime_cmd(clang, clangxx, inc), cache, jobs, env,
|
||||
compiled, reused, bad = compile_all(srcs, runtime_cmd(clang, clangxx, inc, first=vulkan_headers(clang, env)),
|
||||
cache, jobs, env,
|
||||
lambda d, t: (d % 20 == 0 or d == t) and say(f' {d} of {t}'),
|
||||
salt=compiler_id(clang, env))
|
||||
if bad:
|
||||
for src, err in bad[:3]:
|
||||
print(f'--- {src}\n{err}', file=sys.stderr)
|
||||
report_failures(bad)
|
||||
fail(f'{len(bad)} runtime files did not compile')
|
||||
lib = dest / RUNTIME_LIB
|
||||
lib.unlink(missing_ok=True)
|
||||
@@ -1170,6 +1535,21 @@ def toolchain(msys):
|
||||
return Path(shutil.which('clang')), Path(shutil.which('clang++')), dict(os.environ), vma
|
||||
|
||||
|
||||
def vulkan_headers(clang, env):
|
||||
"""The include flags that put the kit's Vulkan headers ahead of the system's for the runtime's sources: the
|
||||
pinned release in <kit>/toolchain/vulkan-headers (tools/vulkan_pin.py), which the AppImage's kit carries and a
|
||||
source checkout fetches when the system's headers are older (tools/build_game.py vulkan_include). None on
|
||||
Windows, where MSYS2 supplies the headers, and none where the system's headers are new enough."""
|
||||
if WIN:
|
||||
return []
|
||||
try:
|
||||
from build_game import vulkan_include
|
||||
folder = vulkan_include(clang, env)
|
||||
except ImportError: # a kit without tools/vulkan_pin.py: the system's headers, as before it existed
|
||||
return []
|
||||
return ['-I', str(folder)] if folder else []
|
||||
|
||||
|
||||
def xlate_source():
|
||||
"""The folder of the shader translator to copy beside the game: the kit's bin/, else a source checkout's
|
||||
build/xlate/ (tools/build_xlate.py)."""
|
||||
@@ -1225,17 +1605,7 @@ def step_compile(out, cache_dir, msys, jobs, romfs_job=None, update=False):
|
||||
gen.mkdir(parents=True, exist_ok=True)
|
||||
obj.mkdir(parents=True, exist_ok=True)
|
||||
rt = KIT / 'runtime'
|
||||
# The runtime's own shaders are in the repository as source only: from a source checkout their headers are not
|
||||
# there until they are built, and every runtime file that includes one would fail to compile. A kit carries them
|
||||
# built and does not carry tools/build_shaders.py (nor the compilers it calls), so a kit build skips this step.
|
||||
try:
|
||||
import build_shaders
|
||||
except ModuleNotFoundError:
|
||||
build_shaders = None
|
||||
stale = build_shaders.stale(rt) if build_shaders else []
|
||||
if stale:
|
||||
say(f' compiling {len(stale)} of the runtime\'s own shaders (runtime/*_spv.h)')
|
||||
build_shaders.build(rt, stale)
|
||||
runtime_headers(rt)
|
||||
clang, clangxx, env, vma = toolchain(msys)
|
||||
if not romfs_job:
|
||||
romfs_index(out, gen)
|
||||
@@ -1262,15 +1632,23 @@ def step_compile(out, cache_dir, msys, jobs, romfs_job=None, update=False):
|
||||
fast = fast_clang(msys, clang, env)
|
||||
if fast:
|
||||
say(f' the lifted code compiles with {fast}')
|
||||
cmd_for = runtime_cmd(clang, clangxx, inc, level, fast, msys)
|
||||
vulkan = vulkan_headers(clang, env)
|
||||
if vulkan:
|
||||
say(f' the Vulkan headers come from {vulkan[1]}')
|
||||
cmd_for = runtime_cmd(clang, clangxx, inc, level, fast, msys, first=vulkan)
|
||||
# The keys earlier versions gave the same compiles (compile_all alts): 0.6.0's, with the compiler's file time and,
|
||||
# with the fast compiler, its id and --sysroot; 0.5.0's and earlier's, with their kit's folder in the paths.
|
||||
# with the fast compiler, its id and --sysroot; 0.5.0's and earlier's, with their kit's folder in the paths; and
|
||||
# up to 0.7.1 the runtime's without the Vulkan headers' folder ahead of the system's (an object that compiled
|
||||
# against the system's headers is as good).
|
||||
plain = runtime_cmd(clang, clangxx, inc, level, None, msys)
|
||||
no_vulkan = runtime_cmd(clang, clangxx, inc, level, fast, msys)
|
||||
fast_old = (cid_old + compiler_ids(fast, env)[1]) if fast else None
|
||||
kits = earlier_kits()
|
||||
|
||||
def alts(src, dst):
|
||||
keys = [object_key(plain(src, dst), src, dst, cache, cid_old, sysroot=True)]
|
||||
if vulkan:
|
||||
keys.append(object_key(no_vulkan(src, dst), src, dst, cache, cid))
|
||||
if fast:
|
||||
keys.append(object_key(cmd_for(src, dst), src, dst, cache, fast_old, sysroot=True))
|
||||
keys += [legacy_key(src, dst, cache, cid_old, clang, clangxx, k, lift, gen, vma) for k in kits]
|
||||
@@ -1308,8 +1686,7 @@ def step_compile(out, cache_dir, msys, jobs, romfs_job=None, update=False):
|
||||
compiled, reused = compiled + c2, reused + r2
|
||||
say(f' {compiled} compiled, {reused} unchanged since the last build')
|
||||
if bad:
|
||||
for src, err in bad[:3]:
|
||||
print(f'--- {src}\n{err}', file=sys.stderr)
|
||||
report_failures(bad)
|
||||
fail(f'{len(bad)} files did not compile')
|
||||
if cache.keep_only({o.name for _, o in srcs}):
|
||||
cache.save()
|
||||
@@ -1326,8 +1703,9 @@ def step_compile(out, cache_dir, msys, jobs, romfs_job=None, update=False):
|
||||
ld = (['-fuse-ld=lld'] if (msys / 'bin' / 'ld.lld.exe').is_file() else []) + ['-Wl,--stack,16777216']
|
||||
else: # tools/build_game.py's Linux flags; the main thread's stack is the process's (ulimit -s), not the linker's
|
||||
ld = (['-fuse-ld=lld'] if shutil.which('ld.lld') else []) + ['-no-pie', '-Wl,-z,noexecstack', '-Wl,-rpath,$ORIGIN']
|
||||
# the link map feeds tools/prof_report.py (symbols for an NX2NAVI_PROFILE sampling profile); both linkers write one
|
||||
ld.append('-Wl,--Map=' + (cache_dir / 'ocarina.map').as_posix())
|
||||
# the link map feeds tools/prof_report.py (symbols for an NX2NAVI_PROFILE sampling profile); both linkers write one.
|
||||
# -Xlinker passes the path as one argument; -Wl, would cut it at every comma of the game folder's path (issue #470)
|
||||
ld += ['-Xlinker', '--Map=' + (cache_dir / 'ocarina.map').as_posix()]
|
||||
lib = ['-Wl,--whole-archive', str(prebuilt[0]), '-Wl,--no-whole-archive'] if prebuilt else []
|
||||
stamp = [[o.name, o.stat().st_size, o.stat().st_mtime_ns] for o in objs]
|
||||
if prebuilt:
|
||||
@@ -1382,23 +1760,28 @@ SHADER_STALL_S = 180 # no count moved for this long: the launcher says so (the
|
||||
SHADER_BEAT_S = 10 # a progress event at least this often while the step runs
|
||||
|
||||
|
||||
def shader_progress(phase, done, total, quiet_s, elapsed_s):
|
||||
def shader_progress(phase, done, total, quiet_s, elapsed_s, build_only=False):
|
||||
"""The shaders step's progress event fields: the translation fills the first half of the bar, the build into the
|
||||
driver's cache the second (nx2navi.exe prints both counts, runtime/gfx_shader.c prepare_files and
|
||||
runtime/vk_pipeline.c lb_warm_prepared); before either count, reading the game's shaders. quiet_s: seconds since a
|
||||
count last moved, said once it passes SHADER_STALL_S, so a slow step never looks frozen without a word. The
|
||||
detail ends without a full stop: the launcher adds one (launcher/ui/app.py)."""
|
||||
detail ends without a full stop: the launcher adds one (launcher/ui/app.py).
|
||||
build_only: a run that only builds (warm_driver_cache), whose count fills the whole bar: starting at one half,
|
||||
the bar of a 20 minute build read 50% in its first second, and the launcher's time left, which follows the bar's
|
||||
pace (launcher/jobs.py Job.eta), had half the bar to go by."""
|
||||
clock = f'{int(elapsed_s) // 60}:{int(elapsed_s) % 60:02d}'
|
||||
if phase is None:
|
||||
fraction, detail = 0.0, f'Reading the game\'s shaders ({clock} so far)'
|
||||
elif phase == 'translate':
|
||||
fraction, detail = 0.5 * done / max(total, 1), f'Translated {done:,} of {total:,} shaders'
|
||||
else:
|
||||
fraction, detail = 0.5 + 0.5 * done / max(total, 1), f'Built {done:,} of {total:,} shaders for your graphics card'
|
||||
part = done / max(total, 1)
|
||||
fraction = part if build_only else 0.5 + 0.5 * part
|
||||
detail = f'Built {done:,} of {total:,} shaders for your graphics card'
|
||||
if quiet_s >= SHADER_STALL_S:
|
||||
detail += (f'. No change for {int(quiet_s) // 60} min, but it is still running: one large shader can take '
|
||||
'a while')
|
||||
return {'step': 'shaders', 'fraction': round(fraction, 4), 'detail': detail}
|
||||
return {'step': 'shaders', 'fraction': fraction, 'detail': detail}
|
||||
|
||||
|
||||
# In the game's shader-cache/: the translator identities its translations are of, newest first (runtime/gfx_shader.c,
|
||||
@@ -1454,10 +1837,51 @@ def shader_carry(out):
|
||||
return (out / 'shader-prepared.bin').is_file() and bool(cache_identities(out))
|
||||
|
||||
|
||||
def shader_run(cmd, cwd=None):
|
||||
# The head of shader-prepared.bin (runtime/prp_file.h): magic, version, record size, and from version 2 on what the
|
||||
# list was prepared from: sha256 of seed-programs.bin, sha256 of seed-answers.bin, the translator identity.
|
||||
PREPARED_HEAD = struct.Struct('<8sII')
|
||||
SEED_FILES = ('seed-programs.bin', 'seed-answers.bin')
|
||||
|
||||
|
||||
def prepared_from(out):
|
||||
"""What the game folder's prepared shader list says it was prepared from: {'programs', 'answers', 'translator'}
|
||||
as hex digests; {} for a list that does not say (written before the list recorded it); None without a list."""
|
||||
try:
|
||||
with open(out / 'shader-prepared.bin', 'rb') as f:
|
||||
data = f.read(PREPARED_HEAD.size + 96)
|
||||
except OSError:
|
||||
return None
|
||||
if len(data) < PREPARED_HEAD.size:
|
||||
return {}
|
||||
magic, version, _rec = PREPARED_HEAD.unpack_from(data)
|
||||
if magic != b'NX2NSPRP' or version != 2 or len(data) < PREPARED_HEAD.size + 96:
|
||||
return {}
|
||||
body = data[PREPARED_HEAD.size:]
|
||||
return {'programs': body[:32].hex(), 'answers': body[32:64].hex(), 'translator': body[64:96].hex()}
|
||||
|
||||
|
||||
def seed_changed(out):
|
||||
"""Why the shaders prepared before cannot stay in use, given the seed just written into the game folder
|
||||
(SEED_FILES): a sentence, or '' when the list was prepared from this very seed. A list that does not say what it
|
||||
was prepared from counts as changed: nothing tells whether it holds what this seed names.
|
||||
The translator is not compared: input ids do not depend on it and the game carries the translations of the one
|
||||
before over (shader_carry)."""
|
||||
was = prepared_from(out)
|
||||
if not was:
|
||||
return 'the shaders prepared before do not say which shader seed they were prepared from'
|
||||
try:
|
||||
now = {'programs': sha256_file(out / SEED_FILES[0]), 'answers': sha256_file(out / SEED_FILES[1])}
|
||||
except OSError:
|
||||
return '' # no seed to compare with: as before the list recorded it
|
||||
if (was['programs'], was['answers']) != (now['programs'], now['answers']):
|
||||
return 'the shader seed changed since the shaders were prepared'
|
||||
return ''
|
||||
|
||||
|
||||
def shader_run(cmd, cwd=None, build_only=False):
|
||||
"""Runs a shader preparation command and returns its exit code. Its output is read as it comes: the counts
|
||||
nx2navi.exe prints become progress events, the other lines go on to the log, and while nothing moves an event
|
||||
every few seconds keeps saying the step is running and for how long."""
|
||||
every few seconds keeps saying the step is running and for how long. build_only: shader_progress."""
|
||||
proc = subprocess.Popen(cmd, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,
|
||||
encoding='utf-8', errors='replace', bufsize=1)
|
||||
t0 = time.monotonic()
|
||||
@@ -1468,7 +1892,7 @@ def shader_run(cmd, cwd=None):
|
||||
def send():
|
||||
now = time.monotonic()
|
||||
emit('progress', **shader_progress(state['phase'], state['done'], state['total'], now - state['moved'],
|
||||
now - t0))
|
||||
now - t0, build_only))
|
||||
|
||||
def heartbeat():
|
||||
while not stop.wait(SHADER_BEAT_S):
|
||||
@@ -1502,6 +1926,20 @@ def shader_run(cmd, cwd=None):
|
||||
return rc
|
||||
|
||||
|
||||
def refresh_pipeline_seed(out):
|
||||
"""The kit's pipeline list (tools/pipeline_list.py) as the game folder's seed-pipeline-keys.bin, so that an update
|
||||
which carries the shaders over builds a newer list's pipelines too: `--warm-shaders` builds from that file. The
|
||||
list's shader variants are not translated on this path, so a record whose variant this folder has neither prepared
|
||||
nor met is left out by the game, which counts those in its line."""
|
||||
lst = KIT / 'tools' / 'data' / 'pipeline_list.bin'
|
||||
if not lst.is_file():
|
||||
return
|
||||
import pipeline_list
|
||||
got = pipeline_list.unpack(lst.read_bytes())
|
||||
if got:
|
||||
(out / 'seed-pipeline-keys.bin').write_bytes(got[1])
|
||||
|
||||
|
||||
def warm_driver_cache(out):
|
||||
"""After an update that carries the shaders over: every prepared stage library is built into the graphics driver's
|
||||
cache again (`nx2navi.exe --warm-shaders`). It takes seconds while the driver still has them. It takes minutes when
|
||||
@@ -1513,10 +1951,11 @@ def warm_driver_cache(out):
|
||||
exe = out / EXE_NAME
|
||||
if not exe.is_file() or not (out / 'shader-prepared.bin').is_file():
|
||||
return
|
||||
refresh_pipeline_seed(out)
|
||||
say(" the prepared shaders are built into the graphics driver's cache again")
|
||||
t0 = time.monotonic()
|
||||
try:
|
||||
rc = shader_run([str(exe), '--warm-shaders'], cwd=str(out))
|
||||
rc = shader_run([str(exe), '--warm-shaders'], cwd=str(out), build_only=True)
|
||||
except OSError as e:
|
||||
rc = str(e)
|
||||
if rc:
|
||||
@@ -1530,7 +1969,8 @@ def step_shaders(out, update=False):
|
||||
(shader_run reports its progress).
|
||||
update: after a kit update the shaders prepared before stay in use and the game translates them again in the
|
||||
background (shader_carry), so the update does not wait the minutes a full preparation takes; the driver's cache
|
||||
gets their stage libraries again (warm_driver_cache)."""
|
||||
gets their stage libraries again (warm_driver_cache). Only a list prepared from the seed this kit writes stays
|
||||
(seed_changed): with another seed, or a list that does not say, everything is prepared again as at the install."""
|
||||
# The game translates with the copy beside it. The compile step refreshes that copy, but a kit that changes only
|
||||
# the translator runs this step again alone (step_key), which would then prepare with the old one. Only a kit that
|
||||
# has one: the compile step, which always runs first, stops a build without it.
|
||||
@@ -1540,19 +1980,35 @@ def step_shaders(out, update=False):
|
||||
for name in (XLATE_NAME, 'nx2navi_xlate.id'):
|
||||
copy_if_changed(d / name, out / name)
|
||||
break
|
||||
if update and shader_carry(out):
|
||||
cmd = [sys.executable, '-u', str(KIT / 'tools' / 'prepare_shaders.py'), str(out), '--romfs', str(out / 'romfs'),
|
||||
'--exefs', str(out / 'exefs'), '--no-list', '--exe', EXE_NAME]
|
||||
why = ''
|
||||
carry = update and shader_carry(out)
|
||||
if carry and prepared_from(out) is not None:
|
||||
# The seed this kit makes from the game files, written without preparing anything (seconds): the list
|
||||
# prepared before stays in use only when it was prepared from this seed. Another seed names other shader
|
||||
# variants (a newer pipeline list, a change to which variants are prepared), and those the old list lacks
|
||||
# would be translated when play first draws them. A seed that cannot be written leaves things as they were.
|
||||
if shader_run(cmd + ['--no-run']) == 0:
|
||||
why = seed_changed(out)
|
||||
if why:
|
||||
say(f' {why}: preparing them again')
|
||||
if carry and not why:
|
||||
for seed in SEED_FILES:
|
||||
(out / seed).unlink(missing_ok=True)
|
||||
say(' the shaders prepared before this update stay in use; the game translates them again in the background')
|
||||
if loose_shaders(out) >= PACK_LOOSE_MIN:
|
||||
# minutes without a count (4 on a player's 328,000 files): the row says what runs meanwhile
|
||||
emit('progress', step='shaders', fraction=0.0, detail='Folding the shader cache into one file')
|
||||
pack_shader_cache(out)
|
||||
warm_driver_cache(out)
|
||||
return
|
||||
cmd = [sys.executable, '-u', str(KIT / 'tools' / 'prepare_shaders.py'), str(out), '--romfs', str(out / 'romfs'),
|
||||
'--exefs', str(out / 'exefs'), '--no-list', '--exe', EXE_NAME]
|
||||
rc = shader_run(cmd)
|
||||
if rc:
|
||||
fail('preparing the shaders failed (the lines above say why); the game still runs, compiling shaders as it '
|
||||
'meets them. Run the build again to retry this step')
|
||||
for seed in ('seed-programs.bin', 'seed-answers.bin'): # translated now: the shader cache holds them
|
||||
# translated now: the shader cache holds them. seed-pipeline-keys.bin stays: --warm-shaders builds from it again
|
||||
for seed in SEED_FILES:
|
||||
(out / seed).unlink(missing_ok=True)
|
||||
pack_shader_cache(out)
|
||||
|
||||
@@ -1754,19 +2210,18 @@ def step_scan(out):
|
||||
def step_key(step, identity, out=None):
|
||||
"""What a step's output depends on. exefs and romfs: the game files only; lift: the unpacked ExeFS and the
|
||||
lifter; compile: the lift, the runtime sources, this script and the processor (cpu_tag); prompts: the game files,
|
||||
the kit's button art, the tools that draw it, the bind_ lines of nx2navi.ini, and whether every folder it writes
|
||||
is still there."""
|
||||
the kit's button art, the version of the tools that draw it (PROMPT_TOOLS), the bind_ lines of nx2navi.ini, and
|
||||
whether every folder it writes is still there; scan: the game files, the check's version (SCAN_TOOL) and whether its report is still there."""
|
||||
if step in ('exefs', 'romfs'):
|
||||
return f'{step}-1|{identity}'
|
||||
if step == 'scan':
|
||||
written = 'written' if (out / 'romfs-check.json').is_file() else 'missing'
|
||||
tool = KIT / 'tools' / 'romfs_check.py'
|
||||
return f'scan-1|{identity}|{sha256_file(tool) if tool.is_file() else "no tool"}|{written}'
|
||||
return f'scan-1|{identity}|{SCAN_TOOL if tool.is_file() else "no tool"}|{written}'
|
||||
if step == 'prompts':
|
||||
tools = tree_digest(KIT / 'tools', ('pad_prompts.py', 'prompt_art.py', 'button_icons.py'))
|
||||
binds = sha256_bytes(json.dumps(bind_lines(out)).encode())
|
||||
written = all((out / f / 'manifest.tsv').is_file() for f in prompt_art.OVERLAYS)
|
||||
return (f'prompts-1|{identity}|{tree_digest(prompt_art_dir(), ("**/*.png",))}|{tools}|{binds}'
|
||||
return (f'prompts-1|{identity}|{tree_digest(prompt_art_dir(), ("**/*.png",))}|{PROMPT_TOOLS}|{binds}'
|
||||
f'|{"written" if written else "missing"}')
|
||||
lift = f'lift|{exefs_digest(out)}|{lift_digest()}'
|
||||
if step == 'lift':
|
||||
@@ -1781,7 +2236,28 @@ def step_key(step, identity, out=None):
|
||||
f'|{sha256_file(xlate_source() / "nx2navi_xlate.id")}')
|
||||
|
||||
|
||||
def out_path(arg):
|
||||
"""--out as an absolute path. On Windows a mapped drive letter stays as given: resolve() would name the folder by
|
||||
its server and share instead (a UNC path), which is a second connection to the same share, and some servers drop
|
||||
that one under the compilers' load while the mapped drive stays up."""
|
||||
return Path(os.path.abspath(arg)) if WIN else Path(arg).resolve()
|
||||
|
||||
|
||||
def main():
|
||||
"""The build (build_main), with every error it does not report itself reported as fail reports one: the launcher
|
||||
shows the message of the error event (launcher/build.py Run), and an exception that only printed its traceback
|
||||
reached the player as "the build stopped (exit code 1)". The traceback still goes to the log."""
|
||||
try:
|
||||
build_main()
|
||||
except nup.NupError as e:
|
||||
fail(f'the dump is damaged: {e}. Copy the dump to this PC again, or dump the game again, and then try again')
|
||||
except Exception as e: # noqa: BLE001 - whatever it is, the player gets its message
|
||||
with OUT_LOCK:
|
||||
traceback.print_exc()
|
||||
fail(f'the build stopped: {error_text(e)}')
|
||||
|
||||
|
||||
def build_main():
|
||||
global EVENTS
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument('--nup', help='the game as one .nup file')
|
||||
@@ -1796,13 +2272,17 @@ def main():
|
||||
ap.add_argument('--check', action='store_true', help='only check the game\'s version; builds nothing')
|
||||
ap.add_argument('--msys', default=r'C:\msys64\ucrt64',
|
||||
help='MSYS2 UCRT64 folder (default C:\\msys64\\ucrt64; Windows only: Linux takes clang and lld from PATH)')
|
||||
ap.add_argument('--jobs', type=int, default=os.cpu_count() or 4, help='parallel compiles')
|
||||
ap.add_argument('--jobs', type=int,
|
||||
help='parallel compiles (default: one per processor, fewer when the free memory is short)')
|
||||
ap.add_argument('--events', action='store_true', help='machine-readable progress lines on stdout ("@@" + JSON)')
|
||||
ap.add_argument('--make-runtime-lib', metavar='DIR', help='build the prebuilt runtime library of a release into DIR')
|
||||
ap.add_argument('--obj', help='with --make-runtime-lib: the folder for its objects')
|
||||
ap.add_argument('--until', choices=STEPS, help=argparse.SUPPRESS) # tests: stop after this step
|
||||
a = ap.parse_args()
|
||||
EVENTS = a.events
|
||||
jobs_why = '--jobs'
|
||||
if not a.jobs or a.jobs < 1:
|
||||
a.jobs, jobs_why = default_jobs()
|
||||
if a.make_runtime_lib:
|
||||
make_runtime_lib(a.make_runtime_lib, Path(a.msys), a.jobs, a.obj)
|
||||
return
|
||||
@@ -1811,7 +2291,7 @@ def main():
|
||||
ap.error('give --nup, both --exefs and --romfs, or --rebuild')
|
||||
if a.rebuild and not a.out:
|
||||
ap.error('--rebuild needs --out')
|
||||
out = Path(a.out).resolve() if a.out else None
|
||||
out = out_path(a.out) if a.out else None
|
||||
cache = out / 'build-cache' if out else None
|
||||
state_path = cache / 'state.json' if cache else None
|
||||
state = load_state(state_path) if state_path else {}
|
||||
@@ -1853,6 +2333,18 @@ def main():
|
||||
# the compile step's key does not name the level: without this a finished compile reads as done before
|
||||
state['steps'].pop('compile', None)
|
||||
keep_awake()
|
||||
# What can be known to fail later is looked at now: the disk space of every step that will run, and on Windows
|
||||
# that the build tools compile and link (not for a build the tests stop before its compile step, --until)
|
||||
ahead = steps_to_run(state, src.identity, out, a.until)
|
||||
check_space(out, space_needs(ahead, out, sum(src.romfs_size(ref) for _, ref in src.romfs) if 'romfs' in ahead else 0,
|
||||
a.rebuild and shader_carry(out)))
|
||||
if WIN and 'compile' in ahead:
|
||||
probe_toolchain(msys, cache / 'toolchain-check')
|
||||
# Linux: the compiler, headers and libraries are the system's, so they are tried on the runtime before the long
|
||||
# steps of a build that will reach the compile step (the first step whose key is not current makes the rest run)
|
||||
if not WIN and any(state['steps'].get(n) != step_key(n, src.identity, out) for n in STEPS[:STEPS.index('compile') + 1]):
|
||||
preflight(msys, a.jobs)
|
||||
say(f'compiles at a time: {a.jobs} ({jobs_why})')
|
||||
t0 = time.time()
|
||||
romfs_job = None
|
||||
runs = {'exefs': lambda: step_exefs(src, out), 'romfs': lambda: step_romfs(src, out, min(a.jobs, 8)),
|
||||
@@ -1900,10 +2392,10 @@ def main():
|
||||
emit('step', step=name, state='skip')
|
||||
else:
|
||||
# a step that runs makes every step after it run too: their keys hash its output, not its key (but
|
||||
# for OWN_KEY_STEPS, whose keys name their inputs)
|
||||
# for OWN_KEY_STEPS, whose keys name their inputs, and a step that reads nothing of it, UNAFFECTED)
|
||||
with state_lock:
|
||||
for later in STEPS[STEPS.index(name):]:
|
||||
if later == name or not {name, later} & set(OWN_KEY_STEPS):
|
||||
if later == name or not ({name, later} & set(OWN_KEY_STEPS) or (name, later) in UNAFFECTED):
|
||||
state['steps'].pop(later, None)
|
||||
say(f'{name}:')
|
||||
emit('step', step=name, state='start')
|
||||
|
||||
@@ -29,9 +29,10 @@ from pathlib import Path
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
NAME = 'NX2navi-BuildKit'
|
||||
TOOLS = ('make_game.py', 'nup.py', 'romfs_index.py', 'check_content.py', 'hostmem.py', 'prepare_shaders.py',
|
||||
'shader_census.py', 'shader_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py', 'romfs_check.py',
|
||||
'shader_census.py', 'shader_list.py', 'pipeline_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py', 'romfs_check.py',
|
||||
'build_game.py', 'check_isa.py', 'build_shaders.py', 'make_dlaa_mask.py', 'make_mvobj_fs.py',
|
||||
'pack_shader_cache.py') # make_game.py imports build_game's vma_include on Linux for the VMA header, and
|
||||
'pack_shader_cache.py', 'vulkan_pin.py') # make_game.py imports build_game's vma_include and vulkan_include on
|
||||
# Linux for the VMA header and for Vulkan headers newer than the system's (which build_game takes from vulkan_pin), and
|
||||
# build_game.py imports check_isa and build_shaders, which reads the two make_ scripts' shaders
|
||||
TOP = ('LICENSE', 'LICENSE-ADDITIONS', 'THIRD_PARTY_NOTICES.md', 'VERSION', 'nx2navi.ini.example')
|
||||
SKIP_DIRS = {'__pycache__', '.pytest_cache'}
|
||||
@@ -63,6 +64,8 @@ def main():
|
||||
help='the Streamline SDK folder instead of downloading the SDK release with --dlssg')
|
||||
ap.add_argument('--prompt-art', type=Path, nargs='+', metavar='FOLDER',
|
||||
help='the button art: folders holding XGamepad, P5Gamepad and Keyboard_Mouse, or those folders')
|
||||
ap.add_argument('--pipeline-list', type=Path, metavar='FILE',
|
||||
help='the shared pipeline list to carry (tools/pipeline_list.py merge writes it; it is derived from game data and stays out of the repository)')
|
||||
ap.add_argument('--out', type=Path, default=ROOT / 'build' / 'kit')
|
||||
ap.add_argument('--zip', action='store_true')
|
||||
a = ap.parse_args()
|
||||
@@ -108,6 +111,12 @@ def main():
|
||||
if a.prompt_art: # make_game.py draws the button prompts from it (its prompts step)
|
||||
n, size = prompt_art.copy_art(a.prompt_art, kit / prompt_art.FOLDER)
|
||||
print(f'button art: {n} pictures, {size / 1e6:.1f} MB')
|
||||
if a.pipeline_list: # prepare_shaders.py reads it there (BUNDLED_PIPELINES)
|
||||
import pipeline_list
|
||||
if pipeline_list.unpack(a.pipeline_list.read_bytes()) is None:
|
||||
sys.exit(f'{a.pipeline_list} is not a pipeline list (tools/pipeline_list.py merge writes one)')
|
||||
(kit / 'tools' / 'data').mkdir(exist_ok=True)
|
||||
shutil.copy2(a.pipeline_list, kit / 'tools' / 'data' / 'pipeline_list.bin')
|
||||
import check_content
|
||||
bad = [f'{p.relative_to(kit)}: {m}' for p in sorted(kit.rglob('*')) if p.is_file()
|
||||
for m in check_content.problems_for(p, allow_large=p.suffix in ('.exe', '.dll'))] # the translator, the DLLs
|
||||
|
||||
@@ -79,7 +79,7 @@ PINS = {
|
||||
'ac87609db3f715404dd41a989c2f1706ea1512f3d86217f938d11c63af8eb505'),
|
||||
}
|
||||
TOOLS = ('make_game.py', 'nup.py', 'romfs_index.py', 'check_content.py', 'hostmem.py', 'prepare_shaders.py',
|
||||
'shader_census.py', 'shader_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py', 'romfs_check.py',
|
||||
'shader_census.py', 'shader_list.py', 'pipeline_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py', 'romfs_check.py',
|
||||
'pack_shader_cache.py')
|
||||
TOP = ('LICENSE', 'LICENSE-ADDITIONS', 'THIRD_PARTY_NOTICES.md', 'nx2navi.ini.example')
|
||||
SKIP_DIRS = {'__pycache__', '.pytest_cache', 'native'}
|
||||
@@ -179,7 +179,7 @@ def build_python(py):
|
||||
|
||||
# ---------------------------------------------------------------- app/<version>/
|
||||
|
||||
def build_app(app, ver, xlate, feed_repo, lock, ffx=None, dlss=None, art=None, dlssg=None, ffx4=None):
|
||||
def build_app(app, ver, xlate, feed_repo, lock, ffx=None, dlss=None, art=None, dlssg=None, pipelines=None, ffx4=None):
|
||||
# The runtime's own shaders are source in the repository and SPIR-V headers in a release: a player's PC compiles
|
||||
# runtime/ (make_game.py) and has no shader compiler. Build the missing and the stale ones before the copy.
|
||||
build_shaders.build(ROOT / 'runtime')
|
||||
@@ -212,6 +212,10 @@ def build_app(app, ver, xlate, feed_repo, lock, ffx=None, dlss=None, art=None, d
|
||||
if art: # make_game.py draws the button prompts from it (its prompts step)
|
||||
n, size = prompt_art.copy_art(art, app / prompt_art.FOLDER)
|
||||
say(f'button art: {n} pictures, {size / 1e6:.1f} MB')
|
||||
if pipelines: # prepare_shaders.py reads it there (BUNDLED_PIPELINES)
|
||||
(app / 'tools' / 'data').mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(pipelines, app / 'tools' / 'data' / 'pipeline_list.bin')
|
||||
say(f'pipeline list: {pipelines.stat().st_size / 1e6:.1f} MB')
|
||||
(app / 'toolchain.lock.json').write_text(json.dumps(lock, indent=1), encoding='utf-8')
|
||||
(app / 'feed.json').write_text(json.dumps({'type': 'github', 'repo': feed_repo}, indent=1), encoding='utf-8')
|
||||
|
||||
@@ -334,6 +338,8 @@ def main():
|
||||
'nvngx_dlssg.dll files, instead of downloading the SDK release with --dlssg')
|
||||
ap.add_argument('--prompt-art', type=Path, nargs='+', metavar='FOLDER',
|
||||
help='the button art: folders holding XGamepad, P5Gamepad and Keyboard_Mouse, or those folders')
|
||||
ap.add_argument('--pipeline-list', type=Path, metavar='FILE',
|
||||
help='the shared pipeline list to carry (tools/pipeline_list.py merge writes it; it is derived from game data and stays out of the repository)')
|
||||
ap.add_argument('--out', type=Path, default=MAIN_ROOT / 'build' / 'launcher-release')
|
||||
ap.add_argument('--feed', default=DEFAULT_FEED, help='the GitHub repository the launcher looks for releases in')
|
||||
ap.add_argument('--notes', type=Path, help='release notes (markdown) for the manifest and the GitHub release')
|
||||
@@ -366,7 +372,11 @@ def main():
|
||||
dlssg = (a.dlssg, streamline_pin.resolve(a.streamline, DL)) if a.dlssg else None
|
||||
say(f'DLSS Frame Generation: {a.dlssg}, Streamline {streamline_pin.VERSION}: {dlssg[1]}' if dlssg
|
||||
else 'DLSS Frame Generation left out (no --dlssg)')
|
||||
build_app(folder / 'app' / ver, ver, a.xlate, a.feed, lock, a.ffx, dlss, a.prompt_art, dlssg, a.ffx4)
|
||||
if a.pipeline_list:
|
||||
import pipeline_list
|
||||
if pipeline_list.unpack(a.pipeline_list.read_bytes()) is None:
|
||||
sys.exit(f'{a.pipeline_list} is not a pipeline list (tools/pipeline_list.py merge writes one)')
|
||||
build_app(folder / 'app' / ver, ver, a.xlate, a.feed, lock, a.ffx, dlss, a.prompt_art, dlssg, a.pipeline_list, a.ffx4)
|
||||
(folder / 'app' / 'current.txt').write_text(ver + '\n', encoding='utf-8')
|
||||
build_python(folder / 'python')
|
||||
cc, windres = native_compiler(lock)
|
||||
|
||||
@@ -113,7 +113,7 @@ LLVM_SKIP_SUFFIXES = {'.idl', '.tlb'}
|
||||
# tools/ files the installer steps run (and what they import), copied from the repo.
|
||||
TOOLS_FILES = ['build_game.py', 'build_shaders.py', 'button_icons.py', 'check_content.py', 'check_isa.py', 'extract.py',
|
||||
'hostmem.py', 'make_romfs_patches.py', 'ncz.py', 'pack_shader_cache.py', 'prepare_shaders.py',
|
||||
'romfs_index.py', 'shader_census.py', 'shader_list.py', 'ultrawide_layout.py', 'cheats.py',
|
||||
'romfs_index.py', 'shader_census.py', 'shader_list.py', 'pipeline_list.py', 'ultrawide_layout.py', 'cheats.py',
|
||||
'exefs_patches.py']
|
||||
TOOLS_DIRS = ['vendor/macnativegames', 'data'] # data/: expected hashes
|
||||
RUNTIME_SUFFIXES = {'.h', '.inc', '.ico'} # .ico: the installer window icon
|
||||
@@ -427,7 +427,7 @@ def runtime_lib(out, llvm, pgo):
|
||||
shutil.copy2(ROOT / 'runtime' / name, rt / name)
|
||||
|
||||
|
||||
def build_sources(out, pgo):
|
||||
def build_sources(out, pgo, pipelines=None):
|
||||
copy_tree(ROOT / 'recomp', out / 'recomp', lambda r: not r.endswith(('.py', '.txt')) or '__pycache__' in r)
|
||||
for f in TOOLS_FILES:
|
||||
(out / 'tools').mkdir(exist_ok=True)
|
||||
@@ -437,6 +437,8 @@ def build_sources(out, pgo):
|
||||
(out / 'data').mkdir(exist_ok=True)
|
||||
text = shader_list.history(MAIN_ROOT / 'refs' / 'MacNativeGames', MNG_COMMIT)
|
||||
(out / 'data' / 'shader_list.tsv').write_text(text, encoding='utf-8', newline='\n')
|
||||
if pipelines: # prepare_shaders.py reads it there (BUNDLED_PIPELINES)
|
||||
shutil.copy2(pipelines, out / 'data' / 'pipeline_list.bin')
|
||||
for p in pgo.values():
|
||||
for f in (p, p.with_suffix('.json')):
|
||||
if f.is_file():
|
||||
@@ -586,6 +588,8 @@ def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument('--out', type=Path, required=True, help='release folder (replaced)')
|
||||
ap.add_argument('--zip', type=Path, help='also write the folder as this zip')
|
||||
ap.add_argument('--pipeline-list', type=Path, metavar='FILE',
|
||||
help='the shared pipeline list to carry (tools/pipeline_list.py merge writes it; it is derived from game data and stays out of the repository)')
|
||||
ap.add_argument('--prove', action='store_true', help='only run the compile-and-probe check on an existing folder')
|
||||
ap.add_argument('--cpu', choices=('v2', 'v3'), default='v3',
|
||||
help='--prove: build and run the check for this CPU level (default v3)')
|
||||
@@ -604,7 +608,7 @@ def main():
|
||||
build_deps(out)
|
||||
pgo = profiles()
|
||||
runtime_lib(out, llvm, pgo)
|
||||
build_sources(out, pgo)
|
||||
build_sources(out, pgo, args.pipeline_list)
|
||||
build_launcher(out, llvm)
|
||||
collect_licenses(out)
|
||||
check_folder(out)
|
||||
|
||||
@@ -40,6 +40,7 @@ from pathlib import Path
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import build_game # noqa: E402
|
||||
import make_release as mr # noqa: E402
|
||||
import vulkan_pin # noqa: E402
|
||||
|
||||
ROOT = mr.ROOT
|
||||
MAIN_ROOT = mr.MAIN_ROOT
|
||||
@@ -69,8 +70,7 @@ PINS = {
|
||||
'5749cbc4e668273514150a80e387a957f933c6ed3f5f11e03fb30955e2bbead6'),
|
||||
'zstd-src': ('https://github.com/facebook/zstd/releases/download/v1.5.7/zstd-1.5.7.tar.gz',
|
||||
'eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3'),
|
||||
'vulkan-headers-src': ('https://github.com/KhronosGroup/Vulkan-Headers/archive/refs/tags/v1.4.363.tar.gz',
|
||||
'cbaf687d3c59b9666fe080e7f8396b6b4f4d344a768ee6b337c59852f4526b68'),
|
||||
'vulkan-headers-src': vulkan_pin.SOURCE, # the release a build with the system's headers takes too
|
||||
'vma-header': build_game.VMA_FETCH,
|
||||
# appimagetool and the type-2 runtime from release tags, not "continuous", whose uploads change under the same
|
||||
# URL and would stop the build at the hash check
|
||||
@@ -169,8 +169,8 @@ def build_deps(out, llvm, sysroot):
|
||||
d = out / 'runtime' / 'deps'
|
||||
(d / 'include' / 'vma').mkdir(parents=True, exist_ok=True)
|
||||
(d / 'lib').mkdir(parents=True, exist_ok=True)
|
||||
vk = tar_extract('vulkan-headers-src', WORK / 'vulkan-headers', ['Vulkan-Headers-1.4.363/include',
|
||||
'Vulkan-Headers-1.4.363/LICENSE.md'], strip=1)
|
||||
vk = tar_extract('vulkan-headers-src', WORK / 'vulkan-headers', [f'{vulkan_pin.TOP}include',
|
||||
f'{vulkan_pin.TOP}LICENSE.md'], strip=1)
|
||||
for sub in ('vulkan', 'vk_video'):
|
||||
shutil.copytree(vk / 'include' / sub, d / 'include' / sub)
|
||||
shutil.copy2(mr.fetch('vma-header'), d / 'include' / 'vma' / 'vk_mem_alloc.h')
|
||||
|
||||
@@ -25,8 +25,8 @@ EXE = '.exe' if os.name == 'nt' else ''
|
||||
DLLS = ('SDL3.dll', 'libopus-0.dll', 'libiconv-2.dll') if os.name == 'nt' else ()
|
||||
# Prepared state next to the exe: translator answers, programs, the prepared list, pipeline cache and its keys, and the
|
||||
# variant lists `nx2navi.exe --prepare-shaders` reads (the driver keys its library cache by exe path, so each copy runs it).
|
||||
STATE = ('romfs-index.bin', 'pipeline-cache.bin', 'pipeline-keys.bin', 'shader-answers.bin', 'shader-programs.bin',
|
||||
'shader-prepared.bin', 'seed-answers.bin', 'seed-programs.bin')
|
||||
STATE = ('romfs-index.bin', 'pipeline-cache.bin', 'pipeline-cache-play.bin', 'pipeline-keys.bin', 'shader-answers.bin',
|
||||
'shader-programs.bin', 'shader-prepared.bin', 'seed-answers.bin', 'seed-programs.bin')
|
||||
|
||||
def copy(src, dst):
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
@@ -205,7 +205,7 @@ class Romfs:
|
||||
def dictionary(self, dict_id):
|
||||
if self.dicts is None:
|
||||
self.dicts = {}
|
||||
pack = zstandard.ZstdDecompressor().decompressobj().decompress(self.raw(DICTIONARIES))
|
||||
pack = unpack(self.raw(DICTIONARIES))
|
||||
for lo, hi in sarc_members(pack).values():
|
||||
d = zstandard.ZstdCompressionDict(pack[lo:hi])
|
||||
self.dicts[d.dict_id()] = d
|
||||
@@ -218,7 +218,27 @@ class Romfs:
|
||||
return raw, None
|
||||
dict_id = zstandard.get_frame_parameters(raw).dict_id
|
||||
d = self.dictionary(dict_id) if dict_id else None
|
||||
return zstandard.ZstdDecompressor(dict_data=d).decompressobj().decompress(raw), d
|
||||
return unpack(raw, d), d
|
||||
|
||||
|
||||
# How much of a packed file unpack gives the decompressor at a time.
|
||||
PIECE = 1 << 16
|
||||
|
||||
|
||||
def unpack(raw, dictionary=None):
|
||||
"""The contents of a zstd frame, with the dictionary it was packed with; bytes after the frame are left out.
|
||||
|
||||
The file goes to the decompressor in pieces of PIECE bytes and the pieces' outputs are joined once. Given the
|
||||
whole file in one call, the library returns one bytes object that it grows by copying as the output comes: made-up
|
||||
data of the size of the interface texture archive, 69 MB packed and 244 MB unpacked, took 21 s that way on a
|
||||
fast PC against 0.3 s in pieces. Players' builds waited 44 to 52 s before this tool's first folder, most of
|
||||
its minute."""
|
||||
out, parts = zstandard.ZstdDecompressor(dict_data=dictionary).decompressobj(), []
|
||||
for at in range(0, len(raw), PIECE):
|
||||
parts.append(out.decompress(raw[at:at + PIECE]))
|
||||
if out.eof:
|
||||
break
|
||||
return b''.join(parts)
|
||||
|
||||
|
||||
def sarc_members(data):
|
||||
@@ -727,6 +747,11 @@ def mask_of(img, rel):
|
||||
return out
|
||||
|
||||
|
||||
def note_texture(art, rel, box):
|
||||
pic = fit(load_art(art, rel), box, (93, 93))
|
||||
return astc_encode(bleed(pic))
|
||||
|
||||
|
||||
def draw_textures(data, glyph, note, art, cache):
|
||||
"""Paints the action prompt's letters (from glyph) and the ocarina's note buttons (from note; both Switch button ->
|
||||
picture) into the BNTX data (a bytearray) in place. A new letter takes the height and centre of the game's own
|
||||
@@ -752,8 +777,9 @@ def draw_textures(data, glyph, note, art, cache):
|
||||
assert t['fmt'] == ASTC_4X4_SRGB and (t['w'], t['h']) == (93, 93), f'{name}: format {t["fmt"]:#x}, ' \
|
||||
f'{t["w"]}x{t["h"]}'
|
||||
box = NOTE_BOXES['shoulder' if button in 'LR' else 'disc']
|
||||
pic = fit(load_art(art, note[button]), box, (t['w'], t['h']))
|
||||
write_level(data, t, astc_encode(bleed(pic)), wb * 16, hb)
|
||||
rel = note[button]
|
||||
encoded = cache(('note', rel, box), lambda: note_texture(art, rel, box))
|
||||
write_level(data, t, encoded, wb * 16, hb)
|
||||
|
||||
|
||||
# Where the game's own note buttons are opaque in their 93x93 textures (1.1.1), which the new ones fill: the face
|
||||
|
||||
196
tools/pipeline_list.py
Normal file
196
tools/pipeline_list.py
Normal file
@@ -0,0 +1,196 @@
|
||||
"""The shared pipeline list: graphics pipelines players met, for an install to build before play.
|
||||
|
||||
A game folder records every graphics pipeline a run builds in pipeline-keys.bin (runtime/vk_pipeline.c "pipeline
|
||||
keys": the pipeline state with its shaders named by input id) and every shader variant in shader-answers.bin
|
||||
(runtime/gfx_shader.c "answers met in play": stage, code hash, input id, translator answers). The next start of that
|
||||
folder builds its recorded pipelines ahead of play; another install, or the same one after its caches were removed,
|
||||
meets each of them in play first. A mesh pipeline met that way has no stage libraries to be linked from, so its draws
|
||||
are left out of the picture for its whole compile, 0.2 to 1.1 s.
|
||||
|
||||
This tool folds the two files of any number of game folders into one list file. tools/prepare_shaders.py reads the
|
||||
list beside the shared shader list: the shader variants it names go into the seed, so the preparation translates them,
|
||||
and its pipeline records go next to the game as seed-pipeline-keys.bin, which `nx2navi.exe --prepare-shaders` (and
|
||||
`--prepare-pipelines` on a folder that is prepared already) builds once into the pipeline cache and the graphics
|
||||
driver's own.
|
||||
|
||||
What a record depends on. Its shaders are named by input id, a hash of the game's shader code and the answers, which
|
||||
no translator, GPU or driver changes: a record made on one machine names the same shaders on another with the same
|
||||
game version. Its state is the game's (formats, blend, depth and stencil, topology, sample count) and a few choices of
|
||||
the runtime's settings (a culled foliage shadow pass, the depth range); everything that depends on the GPU is derived
|
||||
from it when the pipeline is built. A record whose state this install never reaches costs one pipeline in the caches,
|
||||
and one whose shader this game version lacks is left out by the preparation. The record layout belongs to the runtime
|
||||
that wrote it (KEYS_VERSION and the record size, in the file's header): a list is made again for a runtime that
|
||||
changes either, and the runtime ignores one that does not match.
|
||||
|
||||
The list is derived from game data: it stays out of the repository (tools/check_content.py refuses it) and travels
|
||||
with the installer archive as data/pipeline_list.bin, as the shader list does.
|
||||
|
||||
python tools/pipeline_list.py merge OUT.bin GAME_DIR [GAME_DIR ...] [--mesh-only]
|
||||
python tools/pipeline_list.py info FILE_OR_GAME_DIR [...]
|
||||
|
||||
File: "NX2NPLST", version u32, answers bytes u32, keys bytes u32, then a shader-answers.bin image (header and the
|
||||
records the keys name, input id kept) and a pipeline-keys.bin image (header and records).
|
||||
"""
|
||||
import argparse
|
||||
import struct
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
MAGIC = b'NX2NPLST'
|
||||
VERSION = 1
|
||||
ANS_MAGIC, ANS_REC = b'NX2NSANS', 4 + 32 + 32 + 147 + 1 # gfx_shader.c: stage, code hash, input id, answers, pad
|
||||
KEYS_MAGIC = b'NX2NPKEY'
|
||||
ID = 32
|
||||
STAGE_NAMES = {0: 'vertex', 4: 'fragment', 5: 'compute', 6: 'task', 7: 'mesh'} # runtime/gfx.h GFX_STAGE_*
|
||||
STAGE_MESH = 7
|
||||
|
||||
|
||||
def read_answers(data):
|
||||
"""shader-answers.bin image -> its records (bytes each); [] for another magic or record size."""
|
||||
if len(data) < 16 or data[:8] != ANS_MAGIC:
|
||||
return []
|
||||
_, rec = struct.unpack_from('<II', data, 8)
|
||||
if rec != ANS_REC:
|
||||
return []
|
||||
n = (len(data) - 16) // rec
|
||||
return [data[16 + i * rec:16 + (i + 1) * rec] for i in range(n)]
|
||||
|
||||
|
||||
def read_keys(data):
|
||||
"""pipeline-keys.bin image -> (magic, version, record size, records); records [] for a file too short."""
|
||||
if len(data) < 16:
|
||||
return b'', 0, 0, []
|
||||
magic = data[:8]
|
||||
version, rec = struct.unpack_from('<II', data, 8)
|
||||
if rec < 3 * ID + 8:
|
||||
return magic, version, rec, []
|
||||
n = (len(data) - 16) // rec
|
||||
return magic, version, rec, [data[16 + i * rec:16 + (i + 1) * rec] for i in range(n)]
|
||||
|
||||
|
||||
def key_ids(rec):
|
||||
"""The input ids a key record names: vertex (or mesh), then fragment and task when it has them."""
|
||||
has_fs, has_ts = struct.unpack_from('<II', rec, 3 * ID)
|
||||
ids = [rec[:ID]]
|
||||
if has_fs:
|
||||
ids.append(rec[ID:2 * ID])
|
||||
if has_ts:
|
||||
ids.append(rec[2 * ID:3 * ID])
|
||||
return ids
|
||||
|
||||
|
||||
def read_dir(d):
|
||||
"""A game folder's recorded keys and answers; a missing file gives no records."""
|
||||
d = Path(d)
|
||||
kp, ap = d / 'pipeline-keys.bin', d / 'shader-answers.bin'
|
||||
keys = read_keys(kp.read_bytes()) if kp.is_file() else (b'', 0, 0, [])
|
||||
return keys, read_answers(ap.read_bytes()) if ap.is_file() else []
|
||||
|
||||
|
||||
def merge(dirs, mesh_only=False, log=print):
|
||||
"""One list from the folders' files: each key record once, and the answers record of every shader they name."""
|
||||
header, keys, seen, answers = None, [], set(), {}
|
||||
for d in dirs:
|
||||
(magic, version, rec, recs), ans = read_dir(d)
|
||||
if not recs:
|
||||
log(f'{d}: no pipeline key records')
|
||||
continue
|
||||
if header is None:
|
||||
header = (magic, version, rec)
|
||||
elif header != (magic, version, rec):
|
||||
log(f'{d}: its pipeline-keys.bin is of another layout ({magic!r}, version {version}, {rec} bytes a record); '
|
||||
'left out')
|
||||
continue
|
||||
by_id = {a[36:68]: a for a in ans}
|
||||
added = lacking = 0
|
||||
for r in recs:
|
||||
ids = key_ids(r)
|
||||
if r in seen:
|
||||
continue
|
||||
if any(i not in by_id and i not in answers for i in ids):
|
||||
lacking += 1 # a shader the folder's answers do not name: another install could not translate it
|
||||
continue
|
||||
if mesh_only and by_id.get(ids[0], answers.get(ids[0]))[0:4] != struct.pack('<I', STAGE_MESH):
|
||||
continue
|
||||
seen.add(r)
|
||||
keys.append(r)
|
||||
added += 1
|
||||
for i in ids:
|
||||
answers.setdefault(i, by_id.get(i) or answers[i])
|
||||
log(f'{d}: {len(recs)} key records, {added} new to the list, {lacking} without their shaders\' answers')
|
||||
if header is None:
|
||||
return None
|
||||
return header, keys, list(answers.values())
|
||||
|
||||
|
||||
def pack(header, keys, answers):
|
||||
magic, version, rec = header
|
||||
a = ANS_MAGIC + struct.pack('<II', 1, ANS_REC) + b''.join(answers)
|
||||
k = magic + struct.pack('<II', version, rec) + b''.join(keys)
|
||||
return MAGIC + struct.pack('<III', VERSION, len(a), len(k)) + a + k
|
||||
|
||||
|
||||
def unpack(data):
|
||||
"""A list file -> (answers records, pipeline-keys.bin image); None when it is not one."""
|
||||
if len(data) < 20 or data[:8] != MAGIC:
|
||||
return None
|
||||
version, na, nk = struct.unpack_from('<III', data, 8)
|
||||
if version != VERSION or len(data) != 20 + na + nk:
|
||||
return None
|
||||
return read_answers(data[20:20 + na]), data[20 + na:]
|
||||
|
||||
|
||||
def describe(name, keys_image, answers):
|
||||
magic, version, rec, recs = read_keys(keys_image)
|
||||
stage = {a[36:68]: struct.unpack_from('<I', a)[0] for a in answers}
|
||||
mesh = sum(1 for r in recs if stage.get(r[:ID]) == STAGE_MESH)
|
||||
task = sum(1 for r in recs if struct.unpack_from('<I', r, 3 * ID + 4)[0])
|
||||
named = {i for r in recs for i in key_ids(r)}
|
||||
by = {}
|
||||
for i in named:
|
||||
s = STAGE_NAMES.get(stage.get(i), 'not in the answers')
|
||||
by[s] = by.get(s, 0) + 1
|
||||
print(f'{name}: {len(recs)} pipelines ({mesh} mesh, {task} of them with a task stage), layout {magic.decode("ascii", "replace")} '
|
||||
f'version {version}, {rec} bytes a record; {len(named)} shaders named: '
|
||||
+ ', '.join(f'{v} {k}' for k, v in sorted(by.items())))
|
||||
return len(recs), mesh
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
sub = ap.add_subparsers(dest='cmd', required=True)
|
||||
m = sub.add_parser('merge', help='fold game folders into a list file')
|
||||
m.add_argument('out', type=Path)
|
||||
m.add_argument('dirs', nargs='+', type=Path)
|
||||
m.add_argument('--mesh-only', action='store_true', help='keep the mesh pipelines only')
|
||||
i = sub.add_parser('info', help='what a list file or a game folder holds')
|
||||
i.add_argument('paths', nargs='+', type=Path)
|
||||
a = ap.parse_args()
|
||||
if a.cmd == 'info':
|
||||
for p in a.paths:
|
||||
if p.is_dir():
|
||||
(magic, version, rec, recs), ans = read_dir(p)
|
||||
image = magic + struct.pack('<II', version, rec) + b''.join(recs)
|
||||
describe(str(p), image, ans)
|
||||
else:
|
||||
got = unpack(p.read_bytes())
|
||||
if got is None:
|
||||
print(f'{p}: not a pipeline list')
|
||||
return 1
|
||||
describe(str(p), got[1], got[0])
|
||||
return 0
|
||||
got = merge(a.dirs, a.mesh_only)
|
||||
if got is None:
|
||||
print('no pipeline key records in any folder')
|
||||
return 1
|
||||
header, keys, answers = got
|
||||
data = pack(header, keys, answers)
|
||||
a.out.parent.mkdir(parents=True, exist_ok=True)
|
||||
a.out.write_bytes(data)
|
||||
describe(str(a.out), unpack(data)[1], answers)
|
||||
print(f'{a.out}: {len(data)} bytes')
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -21,6 +21,13 @@ shader-prepared.bin (one record per variant, in priority order) and builds every
|
||||
pipeline once, so the GPU driver's own disk cache holds them (vk_pipeline.c "stage library prebuild"). The seed files
|
||||
can be deleted afterwards.
|
||||
|
||||
The shared pipeline list (tools/pipeline_list.py: graphics pipelines players met, with the shader variants they name)
|
||||
is read beside the shader list when there is one: tools/data/pipeline_list.bin, data/pipeline_list.bin in a release
|
||||
folder, or --pipeline-list. Its variants of shaders this game version has join the seed, and its key records are
|
||||
written as seed-pipeline-keys.bin, which --prepare-shaders builds into the pipeline caches once the seed is translated
|
||||
(vk_pipeline.c "the shared pipeline list"; mesh pipelines by default). That file stays in the build directory:
|
||||
`nx2navi.exe --warm-shaders` and `--prepare-pipelines` build from it again after an update or a new driver.
|
||||
|
||||
A driver whose shader compiler crashes on a variant crashes on it in every run (NVIDIA 615.78.08 on an RTX 5080, issue
|
||||
#30). The translations and shader-prepared.bin are written by then, and the journal the run leaves names the variants
|
||||
the driver was building (vk_pipeline.c skip_init), so only the driver step runs again (`nx2navi.exe --warm-shaders`), up to
|
||||
@@ -47,6 +54,7 @@ ROOT = Path(__file__).resolve().parents[1]
|
||||
while not (ROOT / 'tools' / 'shader_census.py').is_file() and ROOT != ROOT.parent:
|
||||
ROOT = ROOT.parent
|
||||
sys.path.insert(0, str(ROOT / 'tools'))
|
||||
import pipeline_list # noqa: E402
|
||||
import shader_list # noqa: E402
|
||||
from shader_census import inventory # noqa: E402
|
||||
|
||||
@@ -83,6 +91,44 @@ if not BUNDLED_LIST.is_file():
|
||||
BUNDLED_LIST = ROOT / 'data' / 'shader_list.tsv' # release folder layout (tools/make_release.py)
|
||||
|
||||
|
||||
BUNDLED_PIPELINES = ROOT / 'tools' / 'data' / 'pipeline_list.bin' # as BUNDLED_LIST: out of the repository
|
||||
if not BUNDLED_PIPELINES.is_file():
|
||||
BUNDLED_PIPELINES = ROOT / 'data' / 'pipeline_list.bin'
|
||||
SEED_KEYS = 'seed-pipeline-keys.bin'
|
||||
LIST_PRIORITY = 1 # a pipeline list variant's seed priority: with the warm-up rows
|
||||
|
||||
|
||||
def read_pipeline_list(path):
|
||||
"""(answers records, pipeline-keys.bin image) of the pipeline list at path, or of the bundled one; None without
|
||||
a list. A file that is not a list stops the run: a release that names one must not go on without it."""
|
||||
p = Path(path) if path else BUNDLED_PIPELINES
|
||||
if not p.is_file():
|
||||
if path:
|
||||
sys.exit(f'prepare_shaders: no pipeline list at {p}')
|
||||
return None
|
||||
got = pipeline_list.unpack(p.read_bytes())
|
||||
if got is None:
|
||||
sys.exit(f'prepare_shaders: {p} is not a pipeline list (tools/pipeline_list.py merge writes one)')
|
||||
return got
|
||||
|
||||
|
||||
def list_variants(answers_recs, have, seen):
|
||||
"""Seed records for the pipeline list's shader variants: those whose shader (runtime code hash) is in `have` and
|
||||
whose (hash, answers) the seed does not hold yet. The input id is left zero, as for every seed record: the runtime
|
||||
finds it, and finds the id the list's key records name, since it depends on the code and the answers alone."""
|
||||
out, absent = [], 0
|
||||
for a in answers_recs:
|
||||
h, packed = a[4:36], a[68:68 + 147]
|
||||
if h not in have:
|
||||
absent += 1
|
||||
continue
|
||||
if (h, packed) in seen:
|
||||
continue
|
||||
seen.add((h, packed))
|
||||
out.append(a[:36] + bytes(32) + packed + bytes([LIST_PRIORITY]))
|
||||
return out, absent
|
||||
|
||||
|
||||
def read_list(path):
|
||||
if path:
|
||||
return Path(path).read_text(encoding='utf-8')
|
||||
@@ -190,6 +236,8 @@ def main():
|
||||
ap.add_argument('--list', help='a shader_list.tsv instead of tools/data/shader_list.tsv or refs/MacNativeGames')
|
||||
ap.add_argument('--no-run', action='store_true', help='write the seed files only')
|
||||
ap.add_argument('--no-list', action='store_true', help='default variants only, without a shared shader list')
|
||||
ap.add_argument('--pipeline-list', help='a pipeline list instead of tools/data/pipeline_list.bin (tools/pipeline_list.py)')
|
||||
ap.add_argument('--no-pipeline-list', action='store_true', help='no shared pipeline list, also when one is bundled')
|
||||
ap.add_argument('--all-shaders', action='store_true',
|
||||
help='also the default variants SLOW_UNUSED leaves out (an hour more of GPU driver builds)')
|
||||
ap.add_argument('--exe', default='nx2navi.exe' if os.name == 'nt' else 'nx2navi',
|
||||
@@ -235,8 +283,17 @@ def main():
|
||||
rec = struct.pack('<I', stage) + h + bytes(32) + packed + bytes([prio.get(key, 2) | (DEFAULT_FLAG if default else 0)])
|
||||
assert len(rec) == ANS_REC
|
||||
recs.append((prio.get(key, 2), rec))
|
||||
recs.sort(key=lambda r: r[0]) # stable: list order within a priority
|
||||
out = a.build.resolve()
|
||||
plist = None if a.no_pipeline_list else read_pipeline_list(a.pipeline_list)
|
||||
(out / SEED_KEYS).unlink(missing_ok=True) # a list this build no longer has
|
||||
if plist:
|
||||
extra, absent = list_variants(plist[0], {h for h, _ in progs.values()}, seen)
|
||||
recs += [(LIST_PRIORITY, r) for r in extra]
|
||||
(out / SEED_KEYS).write_bytes(plist[1])
|
||||
n_keys = len(pipeline_list.read_keys(plist[1])[3])
|
||||
print(f'pipeline list: {n_keys} pipelines; of the {len(plist[0])} shader variants they name, {len(extra)} join the '
|
||||
f'seed, {absent} are of shaders {a.version} does not have, the others are in it already', flush=True)
|
||||
recs.sort(key=lambda r: r[0]) # stable: list order within a priority
|
||||
with open(out / 'seed-programs.bin', 'wb') as f:
|
||||
f.write(b'NX2NSPRG' + struct.pack('<II', 1, 0))
|
||||
for _, body in progs.values():
|
||||
|
||||
@@ -35,7 +35,12 @@ import zstandard
|
||||
STANDIN_MOD = 'Damaged file stand-ins'
|
||||
DICTIONARIES = 'Pack/ZsDic.pack.zs'
|
||||
ZSTD_MAGIC = b'\x28\xb5\x2f\xfd'
|
||||
PIECE = 1 << 22
|
||||
# How much of a file the decompressor is given at a time. Each call returns its output as one bytes object, which
|
||||
# the library grows as the output comes: with 4 MiB of input a call's output is tens of MiB, grown by copying, and
|
||||
# the copying holds the interpreter's lock, so the worker threads wait on each other. Measured on 32 MiB frames
|
||||
# with 12 threads: about 120 MB/s of packed data with 4 MiB pieces, about 2,500 MB/s with 64 KiB pieces; files
|
||||
# smaller than a piece are the same either way.
|
||||
PIECE = 1 << 16
|
||||
TEXTURE = re.compile(r'Tex/([^/]+)\.bntx\.zs')
|
||||
|
||||
|
||||
@@ -110,6 +115,8 @@ def problem(image, entry):
|
||||
size = 0
|
||||
for at in range(0, len(data), PIECE):
|
||||
size += len(out.decompress(data[at:at + PIECE]))
|
||||
if out.eof: # the frame is whole: bytes after it are not the frame's, whichever piece they fall in
|
||||
break
|
||||
except zstandard.ZstdError as e:
|
||||
return f'does not decompress ({e})'
|
||||
if params.content_size and size != params.content_size:
|
||||
|
||||
200
tools/validate_shader_cache.py
Normal file
200
tools/validate_shader_cache.py
Normal file
@@ -0,0 +1,200 @@
|
||||
"""Check the translated shaders an install's shader cache holds, offline: every <key>.spv with its <key>.res.json, from
|
||||
<cache>/shader-cache.pack and from the loose files (a loose file stands for its packed copy, as the game reads them).
|
||||
|
||||
The game checks little of a cached shader when it loads one (runtime/gfx_shader.c disk_load_from: the reflection is one
|
||||
JSON object that names its stage, the SPIR-V has its magic and whole instructions) and hands the module to the driver
|
||||
as it is. A module the driver then compiles wrongly, or crashes on, shows as a missing draw or a crash in play, far
|
||||
from its cause. This tool puts the question to the whole cache at once:
|
||||
|
||||
1. the game's own checks, so the count says how many entries the game would translate again;
|
||||
2. spirv-val on each module that passes them (--target-env vulkan1.3, as tools/shader_census.py validates fresh
|
||||
translations), unless --no-val is given or spirv-val is not found.
|
||||
|
||||
The modules are the translator's output; the pipeline code rewrites a copy of each before the driver sees it
|
||||
(runtime/vk_pipeline.c make_module), and that copy is not what is checked here. tools/shader_census.py --translate
|
||||
validates translations made now from the game files; this validates what a folder already holds, which is also what
|
||||
earlier translators wrote and what an interrupted write left.
|
||||
|
||||
python tools\\validate_shader_cache.py <install folder>\\shader-cache [--every N] [--jobs N] [--no-val] [--out FILE]
|
||||
|
||||
--every N checks every N-th shader (by key order), for a quick look at a large cache; --out writes one line per failed
|
||||
shader (key, origin, what failed). Exit code 0 when every checked entry passed, 1 otherwise, 2 for a folder that is not
|
||||
a shader cache. The game may be running: nothing is written to the cache.
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from collections import Counter
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import pack_shader_cache as psc # noqa: E402
|
||||
|
||||
SPV, RES = psc.KINDS['.spv'], psc.KINDS['.res.json']
|
||||
SPIRV_MAGIC = 0x07230203
|
||||
STAGES = (b'"vs"', b'"ps"', b'"cs"', b'"ts"', b'"ms"') # gfx_shader.c reflection_valid
|
||||
TARGET_ENV = 'vulkan1.3'
|
||||
|
||||
|
||||
def find_spirv_val():
|
||||
"""The path of spirv-val, or None: MSYS2 UCRT64's on Windows, else the one on PATH."""
|
||||
msys = Path(r'C:\msys64\ucrt64\bin\spirv-val.exe')
|
||||
if os.name == 'nt' and msys.is_file():
|
||||
return str(msys)
|
||||
return shutil.which('spirv-val')
|
||||
|
||||
|
||||
def spirv_problem(data):
|
||||
"""What gfx_shader.c disk_load_from refuses in a module ('' when it takes it): size, magic, and every instruction
|
||||
after the five header words having a word count that ends inside the module (spirv_complete)."""
|
||||
if data is None:
|
||||
return 'no .spv'
|
||||
if len(data) < 20 or len(data) % 4:
|
||||
return 'the .spv is %d bytes: not a whole SPIR-V module' % len(data)
|
||||
words = len(data) // 4
|
||||
if struct.unpack_from('<I', data, 0)[0] != SPIRV_MAGIC:
|
||||
return 'the .spv has another magic number'
|
||||
i = 5
|
||||
while i < words:
|
||||
n = struct.unpack_from('<I', data, 4 * i)[0] >> 16
|
||||
if not n:
|
||||
return 'an instruction of zero words at word %d (a zero-filled stretch)' % i
|
||||
i += n
|
||||
return '' if i == words else 'the last instruction runs past the end of the module'
|
||||
|
||||
|
||||
def reflection_problem(data):
|
||||
"""What gfx_shader.c reflection_valid refuses in a .res.json ('' when it takes it)."""
|
||||
if data is None:
|
||||
return 'no .res.json'
|
||||
if b'\0' in data:
|
||||
return 'the .res.json holds NUL bytes (an interrupted write)'
|
||||
body = data.strip()
|
||||
if len(body) < 2 or body[:1] != b'{' or body[-1:] != b'}':
|
||||
return 'the .res.json is not one JSON object'
|
||||
at = body.find(b'"stage"')
|
||||
quote = body.find(b'"', at + 7) if at >= 0 else -1
|
||||
if quote < 0 or not body.startswith(STAGES, quote):
|
||||
return 'the .res.json names no stage the game knows'
|
||||
return ''
|
||||
|
||||
|
||||
class Cache:
|
||||
"""The .spv and .res.json entries of a shader cache folder: loose files, then the pack."""
|
||||
|
||||
def __init__(self, folder):
|
||||
self.folder = Path(folder)
|
||||
self.loose = {k: p for k, p in psc.loose_files(self.folder).items() if k[0] in (SPV, RES)}
|
||||
self.pack_path = self.folder / psc.PACK
|
||||
self.packed = {k: v for k, v in psc.read_pack(self.pack_path).items() if k[0] in (SPV, RES)}
|
||||
|
||||
def keys(self):
|
||||
"""Every key with a .spv or a .res.json, sorted."""
|
||||
return sorted({name for _, name in self.loose} | {name for _, name in self.packed})
|
||||
|
||||
def read(self, kind, name, pack_file):
|
||||
"""(data or None, 'loose' | 'pack' | '') of one entry."""
|
||||
path = self.loose.get((kind, name))
|
||||
if path is not None:
|
||||
return psc.read_loose(path), 'loose'
|
||||
at = self.packed.get((kind, name))
|
||||
if at is None or pack_file is None:
|
||||
return None, ''
|
||||
return os.pread(pack_file, at[1], at[0]) if hasattr(os, 'pread') else pread(pack_file, at[1], at[0]), 'pack'
|
||||
|
||||
|
||||
def pread(fd, size, offset):
|
||||
"""os.pread where the platform has none (Windows): callers hold one descriptor per thread."""
|
||||
os.lseek(fd, offset, os.SEEK_SET)
|
||||
return os.read(fd, size)
|
||||
|
||||
|
||||
def check_one(cache, name, val, pack_file):
|
||||
"""(key hex, origin, problem): problem '' when the entry passed every check asked for."""
|
||||
spv, origin = cache.read(SPV, name, pack_file)
|
||||
res, res_origin = cache.read(RES, name, pack_file)
|
||||
origin = origin or res_origin
|
||||
problem = reflection_problem(res) or spirv_problem(spv)
|
||||
if not problem and val:
|
||||
r = subprocess.run([val, '--target-env', TARGET_ENV, '-'], input=spv, capture_output=True)
|
||||
if r.returncode:
|
||||
text = (r.stderr or r.stdout).decode('utf-8', 'replace').strip().splitlines()
|
||||
problem = 'spirv-val: ' + (text[0] if text else 'exit code %d' % r.returncode)
|
||||
return name.hex(), origin, problem
|
||||
|
||||
|
||||
def group_of(problem):
|
||||
"""A problem with its numbers and ids taken out, so the same fault in many shaders counts as one group."""
|
||||
return re.sub(r"'\d+\[%[^\]]*\]'|\b\d+\b", 'N', re.sub(r'^spirv-val: error: line \d+: ', 'spirv-val: ', problem))
|
||||
|
||||
|
||||
def validate(folder, every=1, jobs=0, val=None, out=None, say=print):
|
||||
"""Checks the cache at folder; returns (checked, failures) with failures a list of (key, origin, problem)."""
|
||||
cache = Cache(folder)
|
||||
names = cache.keys()[::max(1, every)]
|
||||
jobs = jobs or min(32, (os.cpu_count() or 4))
|
||||
failures, done, t0, last = [], 0, time.monotonic(), time.monotonic()
|
||||
|
||||
def work(chunk):
|
||||
# one descriptor of the pack per task: positioned reads without a shared file position
|
||||
fd = os.open(cache.pack_path, os.O_RDONLY | getattr(os, 'O_BINARY', 0)) if cache.packed else None
|
||||
try:
|
||||
return [check_one(cache, n, val, fd) for n in chunk]
|
||||
finally:
|
||||
if fd is not None:
|
||||
os.close(fd)
|
||||
|
||||
chunks = [names[i:i + 64] for i in range(0, len(names), 64)]
|
||||
with ThreadPoolExecutor(max_workers=jobs) as ex:
|
||||
for results in ex.map(work, chunks):
|
||||
done += len(results)
|
||||
failures += [r for r in results if r[2]]
|
||||
if time.monotonic() - last >= 10:
|
||||
last = time.monotonic()
|
||||
say(f' {done} of {len(names)} shaders checked, {len(failures)} failed, {last - t0:.0f} s')
|
||||
if out:
|
||||
Path(out).write_text(''.join(f'{k} {o or "-"} {p}\n' for k, o, p in failures), encoding='utf-8')
|
||||
return len(names), failures
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.split('\n\n')[0])
|
||||
ap.add_argument('cache', help='the shader cache folder (<install folder>/shader-cache)')
|
||||
ap.add_argument('--every', type=int, default=1, help='check every N-th shader only')
|
||||
ap.add_argument('--jobs', type=int, default=0, help='threads (default: the CPUs, at most 32)')
|
||||
ap.add_argument('--no-val', action='store_true', help="the game's own checks only, without spirv-val")
|
||||
ap.add_argument('--out', help='write one line per failed shader to this file')
|
||||
args = ap.parse_args()
|
||||
folder = Path(args.cache)
|
||||
if not folder.is_dir():
|
||||
print(f'{folder}: not a folder', file=sys.stderr)
|
||||
return 2
|
||||
val = None if args.no_val else find_spirv_val()
|
||||
if not args.no_val and not val:
|
||||
print("spirv-val not found (MSYS2: pacman -S mingw-w64-ucrt-x86_64-spirv-tools): the game's own checks only")
|
||||
t0 = time.monotonic()
|
||||
checked, failures = validate(folder, args.every, args.jobs, val, args.out)
|
||||
if not checked:
|
||||
print(f'{folder}: no translated shaders (no {psc.PACK} and no loose .spv files)', file=sys.stderr)
|
||||
return 2
|
||||
refused = sum(1 for f in failures if not f[2].startswith('spirv-val: '))
|
||||
print(f'{checked} shaders checked in {time.monotonic() - t0:.0f} s: {checked - len(failures)} passed, {refused} '
|
||||
f"the game would translate again (its own checks), {len(failures) - refused} refused by spirv-val"
|
||||
+ ('' if val else ' (not run)'))
|
||||
groups = Counter(group_of(f[2]) for f in failures)
|
||||
for text, n in groups.most_common(20):
|
||||
first = next(f for f in failures if group_of(f[2]) == text)
|
||||
print(f' {n:6d} {text} (first: {first[0][:16]}, {first[1] or "no file"})')
|
||||
if len(groups) > 20:
|
||||
print(f' ... and {len(groups) - 20} more kinds; --out FILE lists every failed shader')
|
||||
return 1 if failures else 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
109
tools/vulkan_pin.py
Normal file
109
tools/vulkan_pin.py
Normal file
@@ -0,0 +1,109 @@
|
||||
"""The Vulkan headers the runtime is compiled against where the system's are older.
|
||||
|
||||
The runtime names structures and constants of Vulkan extensions as they come out (runtime/vk_device.c asks for
|
||||
VK_KHR_pipeline_binary, VK_KHR_dynamic_rendering_local_read and VK_KHR_shader_maximal_reconvergence when the driver
|
||||
has them), so it needs headers at least as new as the ones it was written against. On Windows MSYS2 supplies them and
|
||||
the Linux release (tools/make_release_linux.py) carries them in runtime/deps. A build with the system's own compiler
|
||||
and headers (tools/make_appimage.py's kit, a source checkout) met distributions whose packaged headers are years
|
||||
older: every file that names a newer extension then fails to compile. Such a build takes the headers from here
|
||||
instead: the release the others use, from Khronos's Vulkan-Headers repository, checked against its SHA-256.
|
||||
|
||||
Only the C headers are kept (include/vulkan/*.h and include/vk_video/*.h, which vulkan_core.h includes) with the
|
||||
repository's licence note. The headers are Apache-2.0 OR MIT (THIRD_PARTY_NOTICES.md). Standard library only: the
|
||||
build kit's Python runs this.
|
||||
|
||||
python tools/vulkan_pin.py [--dest toolchain/vulkan-headers] fetch and unpack the pinned headers
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import io
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
import tarfile
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
VERSION = '1.4.363'
|
||||
HEADER_VERSION = 363 # VK_HEADER_VERSION of that release; the numbers only grow, across 1.3 and 1.4 alike
|
||||
# (address, SHA-256): the archive GitHub makes of the release's tag
|
||||
SOURCE = (f'https://github.com/KhronosGroup/Vulkan-Headers/archive/refs/tags/v{VERSION}.tar.gz',
|
||||
'cbaf687d3c59b9666fe080e7f8396b6b4f4d344a768ee6b337c59852f4526b68')
|
||||
TOP = f'Vulkan-Headers-{VERSION}/' # the archive's top folder
|
||||
FOLDER = Path('toolchain') / 'vulkan-headers' # below a checkout's or a kit's root: include/ and LICENSE.md
|
||||
STAMP = 'pinned.txt' # in the folder: the release it holds, written last
|
||||
KEEP = re.compile(r'include/(vulkan|vk_video)/[A-Za-z0-9_]+\.h|LICENSE\.md')
|
||||
|
||||
|
||||
def have(folder):
|
||||
"""True when folder holds the pinned headers, complete (the stamp is written after the last file)."""
|
||||
try:
|
||||
return (folder / STAMP).read_text(encoding='ascii').strip() == VERSION and \
|
||||
(folder / 'include' / 'vulkan' / 'vulkan_core.h').is_file()
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def unpack(archive, folder):
|
||||
"""The C headers and the licence note of the release's archive (bytes) into folder, replacing what is there."""
|
||||
shutil.rmtree(folder, ignore_errors=True)
|
||||
n = 0
|
||||
with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar:
|
||||
for m in tar:
|
||||
rel = m.name[len(TOP):] if m.name.startswith(TOP) else ''
|
||||
if not m.isfile() or not KEEP.fullmatch(rel): # a name the pattern takes has no '..' and no other folder
|
||||
continue
|
||||
dst = folder / rel
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
dst.write_bytes(tar.extractfile(m).read())
|
||||
n += 1
|
||||
if not (folder / 'include' / 'vulkan' / 'vulkan_core.h').is_file() or not (folder / 'LICENSE.md').is_file():
|
||||
shutil.rmtree(folder, ignore_errors=True)
|
||||
sys.exit(f'vulkan_pin: the archive of Vulkan-Headers {VERSION} lacks vulkan_core.h or LICENSE.md')
|
||||
(folder / STAMP).write_text(VERSION + '\n', encoding='ascii')
|
||||
return n
|
||||
|
||||
|
||||
def resolve(folder=None, archive=None):
|
||||
"""The folder holding the pinned headers (default <root>/toolchain/vulkan-headers), fetched and unpacked there
|
||||
when it does not hold them yet. archive: the release's archive on disk instead of the download; it must be the
|
||||
pinned file either way. Returns the folder; its include/ goes on the include path."""
|
||||
folder = Path(folder) if folder else ROOT / FOLDER
|
||||
if have(folder):
|
||||
return folder
|
||||
url, want = SOURCE
|
||||
if archive:
|
||||
data = Path(archive).read_bytes()
|
||||
else:
|
||||
print(f'vulkan_pin: fetching the Vulkan headers {VERSION} into {folder}', flush=True)
|
||||
try:
|
||||
data = urllib.request.urlopen(url, timeout=120).read()
|
||||
except OSError as e:
|
||||
sys.exit(f'vulkan_pin: fetching {url} failed ({e}). Install Vulkan headers {VERSION} or newer, or put the '
|
||||
f'archive on disk and run tools/vulkan_pin.py --archive <file>')
|
||||
if hashlib.sha256(data).hexdigest() != want:
|
||||
sys.exit(f'vulkan_pin: {archive or url} is not the pinned archive of Vulkan-Headers {VERSION} (SHA-256 differs)')
|
||||
unpack(data, folder)
|
||||
return folder
|
||||
|
||||
|
||||
def header_version(text):
|
||||
"""VK_HEADER_VERSION as vulkan_core.h defines it in text, 0 when it does not."""
|
||||
m = re.search(r'^[ \t]*#[ \t]*define[ \t]+VK_HEADER_VERSION[ \t]+(\d+)', text, re.M)
|
||||
return int(m.group(1)) if m else 0
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument('--dest', type=Path, default=ROOT / FOLDER, help='where the headers go (default toolchain/vulkan-headers)')
|
||||
ap.add_argument('--archive', type=Path, help=f'the archive of Vulkan-Headers {VERSION} on disk instead of the download')
|
||||
a = ap.parse_args()
|
||||
folder = resolve(a.dest, a.archive)
|
||||
n = sum(1 for _ in (folder / 'include').rglob('*.h'))
|
||||
print(f'{folder}: Vulkan headers {VERSION}, {n} files')
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -16,7 +16,7 @@
|
||||
+ // NX2NAVI (2026-10-20): an image declared without a format (QueryHostSupportsImageLoadFormatted) that the
|
||||
+ // shader stores to needs StorageImageWriteWithoutFormat; upstream never declares it, so spirv-val rejects
|
||||
+ // 22 agl_technique_pfx compute shaders with formatted SUST. Vulkan's shaderStorageImageWriteWithoutFormat
|
||||
+ // feature backs it. Reads without a format need no capability under Vulkan 1.3 (format feature flags 2).
|
||||
+ // feature backs it. A read of such an image needs StorageImageReadWithoutFormat, which GenerateImageLoad declares.
|
||||
+ writeWithoutFormat |= image.Format == TextureFormat.Unknown && image.Flags.HasFlag(TextureUsageFlags.ImageStore);
|
||||
+
|
||||
SpvInstruction imagePointerType = context.TypePointer(StorageClass.UniformConstant, imageType);
|
||||
|
||||
28
tools/xlate/patches/0039-image-read-without-format.patch
Normal file
28
tools/xlate/patches/0039-image-read-without-format.patch
Normal file
@@ -0,0 +1,28 @@
|
||||
--- a/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/CodeGenContext.cs
|
||||
+++ b/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/CodeGenContext.cs
|
||||
@@ -83,6 +83,7 @@
|
||||
public bool IsMainFunction { get; private set; }
|
||||
public bool MayHaveReturned { get; set; }
|
||||
public bool WasNonUniformAccessDeclared { get; set; }
|
||||
+ public bool WasReadWithoutFormatDeclared { get; set; }
|
||||
|
||||
public CodeGenContext(
|
||||
StructuredProgramInfo info,
|
||||
--- a/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/Instructions.cs
|
||||
+++ b/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/Instructions.cs
|
||||
@@ -743,6 +743,15 @@
|
||||
SpvInstruction imageComponentType = context.GetType(componentType);
|
||||
AggregateType swizzledResultType = texOp.GetVectorType(componentType);
|
||||
|
||||
+ // NX2NAVI: an image declared without a format (ShaderProperties.GetTextureFormat) is read with OpImageRead,
|
||||
+ // which SPIR-V allows only when the module declares StorageImageReadWithoutFormat; Vulkan 1.3, which the
|
||||
+ // runtime requires, allows that capability on every device.
|
||||
+ if (texOp.Format == TextureFormat.Unknown && !context.WasReadWithoutFormatDeclared)
|
||||
+ {
|
||||
+ context.AddCapability(Capability.StorageImageReadWithoutFormat);
|
||||
+ context.WasReadWithoutFormatDeclared = true;
|
||||
+ }
|
||||
+
|
||||
SpvInstruction texel = context.ImageRead(context.TypeVector(imageComponentType, 4), image, pCoords, ImageOperandsMask.MaskNone);
|
||||
SpvInstruction result = GetSwizzledResult(context, texel, swizzledResultType, texOp.Index);
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
--- a/src/Ryujinx.Graphics.Shader/Instructions/InstEmitTexture.cs
|
||||
+++ b/src/Ryujinx.Graphics.Shader/Instructions/InstEmitTexture.cs
|
||||
@@ -1116,6 +1116,21 @@
|
||||
else
|
||||
{
|
||||
type = context.TranslatorContext.GpuAccessor.QuerySamplerType(imm);
|
||||
+
|
||||
+ // NX2NAVI: a size query that reads the component after a 1D or 2D type's dimensions asks for the layer
|
||||
+ // count, so the texture is an array. That holds for any 1D or 2D type here: the accessor's answer, or the 2D
|
||||
+ // it answers for a handle it has no type for. The game's particle vertex shaders read only the layer count
|
||||
+ // of such a handle, which their pixel shaders sample as a 2D array, and use it as the frame count of a
|
||||
+ // flipbook; declared 2D, the binding is a one-layer view and the count reads as 1. A plain texture bound
|
||||
+ // through an array view still reports one layer. Not handled: a program that samples the same handle as a
|
||||
+ // plain 2D texture before this query gets an arrayed declaration for that sample (invalid SPIR-V).
|
||||
+ if (query == TexQuery.TexHeaderDimension &&
|
||||
+ ((type & SamplerType.Mask) == SamplerType.Texture1D || (type & SamplerType.Mask) == SamplerType.Texture2D) &&
|
||||
+ !type.HasFlag(SamplerType.Array) &&
|
||||
+ (componentMask & (1 << type.Dimensions)) != 0)
|
||||
+ {
|
||||
+ type |= SamplerType.Array;
|
||||
+ }
|
||||
}
|
||||
|
||||
TextureFlags flags = isBindless ? TextureFlags.Bindless : TextureFlags.None;
|
||||
@@ -1,8 +1,11 @@
|
||||
"""Smoke test for build/xlate/nx2navi_xlate(.exe): no game files needed.
|
||||
|
||||
Sends one hand-assembled Maxwell fragment shader (writes the constant colour 1.0, 0.5, 0.25, 1.0 to render target 0)
|
||||
through --serve, checks the reply, validates the SPIR-V with spirv-val, then sends a malformed line and checks the
|
||||
service survives and still answers.
|
||||
through --serve, checks the reply, then sends a malformed line and checks the service survives and still answers. Two
|
||||
hand-assembled compute shaders follow. One loads a texel from a float image and stores it back: the translator
|
||||
declares such an image without a format, and its read needs StorageImageReadWithoutFormat. The other reads only the
|
||||
layer count of texture handle 0x10, for which the translator has no type answer: the texture must be declared as an
|
||||
array, or the count reads as 1. Every module is validated with spirv-val.
|
||||
|
||||
python tools/xlate_smoke.py
|
||||
"""
|
||||
@@ -31,6 +34,8 @@ def mov32i(dest: int, value: float) -> int:
|
||||
|
||||
EXIT = 0xE30000000000000F | PT
|
||||
SCHED = 0x001F8000FC0007E0 # scheduling control word; the decoder skips it
|
||||
RZ = 0xFF
|
||||
IMAGE0 = 0x120 // 4 # first NVN image handle in c[2] (tools/xlate/Program.cs ImgSlot): image slot 0
|
||||
|
||||
|
||||
def fragment_code() -> bytes:
|
||||
@@ -42,6 +47,30 @@ def fragment_code() -> bytes:
|
||||
return struct.pack("<20I", *header) + struct.pack(f"<{len(words)}Q", *words)
|
||||
|
||||
|
||||
SULD_P, SUST_P = 0x1D61, 0x1D65 # opcode bits 51-63 of SULD.P and SUST.P
|
||||
TXQ = 0x1BE9 # opcode bits 51-63 of TXQ (bound handle)
|
||||
|
||||
|
||||
def surface_p(opcode: int, data: int, coord: int, handle: int) -> int:
|
||||
"""SULD.P / SUST.P 1D RGBA: the data registers from `data` on, coordinate register, image handle (TidB)."""
|
||||
no_pred_out = 7 << 30 if opcode == SULD_P else 0 # SULD's second destination predicate: PT, none written
|
||||
return (opcode << 51) | (handle << 36) | no_pred_out | (0xF << 20) | PT | (coord << 8) | data
|
||||
|
||||
|
||||
def image_read_code() -> bytes:
|
||||
"""Compute code: R0-R3 = image 0 at x 0 (SULD.P), stored back there (SUST.P)."""
|
||||
words = [SCHED, surface_p(SULD_P, 0, RZ, IMAGE0), surface_p(SUST_P, 0, RZ, IMAGE0), EXIT]
|
||||
return struct.pack(f"<{len(words)}Q", *words)
|
||||
|
||||
|
||||
def layer_count_code() -> bytes:
|
||||
"""Compute code: R0 = the layer count (component 2 of the dimension query) of texture handle 0x10, stored to image 0."""
|
||||
txq = (TXQ << 51) | (0x10 << 36) | (0x4 << 31) | (1 << 22) | PT | (RZ << 8) | 0 # TexHeaderDimension, mask z, LOD RZ
|
||||
sust = (SUST_P << 51) | (IMAGE0 << 36) | (0x1 << 20) | PT | (RZ << 8) | 0 # SUST.P 1D R of R0 at x 0
|
||||
words = [SCHED, txq, sust, EXIT]
|
||||
return struct.pack(f"<{len(words)}Q", *words)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if not EXE.is_file():
|
||||
print(f"missing {EXE}; run tools/build_xlate.py", file=sys.stderr)
|
||||
@@ -50,27 +79,43 @@ def main() -> int:
|
||||
tmp = Path(tmp)
|
||||
code = tmp / "ps.bin"
|
||||
code.write_bytes(fragment_code())
|
||||
image_code = tmp / "cs.bin"
|
||||
image_code.write_bytes(image_read_code())
|
||||
layer_code = tmp / "layers.bin"
|
||||
layer_code.write_bytes(layer_count_code())
|
||||
out = tmp / "out"
|
||||
job = {"key": "const_ps", "stage": "ps", "code": str(code), "debug": False, "fout": {"0": "float"}}
|
||||
image_job = {"key": "image_read_cs", "stage": "cs", "code": str(image_code), "debug": False, "local": [1, 1, 1]}
|
||||
layer_job = {"key": "layer_count_cs", "stage": "cs", "code": str(layer_code), "debug": False, "local": [1, 1, 1]}
|
||||
lines = [
|
||||
json.dumps({"job": job, "out": str(out)}),
|
||||
"this is not json",
|
||||
json.dumps({"job": dict(job, key="again"), "out": str(out)}),
|
||||
json.dumps({"job": image_job, "out": str(out)}),
|
||||
json.dumps({"job": layer_job, "out": str(out)}),
|
||||
]
|
||||
r = subprocess.run([str(EXE), "--serve"], input="\n".join(lines) + "\n", capture_output=True, text=True, timeout=60)
|
||||
replies = r.stdout.splitlines()
|
||||
print("replies:", replies)
|
||||
if len(replies) != 3 or replies[0] != "ok const_ps" or not replies[1].startswith("fail ") or replies[2] != "ok again":
|
||||
if (len(replies) != 5 or replies[0] != "ok const_ps" or not replies[1].startswith("fail ") or replies[2] != "ok again"
|
||||
or replies[3] != "ok image_read_cs" or replies[4] != "ok layer_count_cs"):
|
||||
print("unexpected replies; stderr:", r.stderr, file=sys.stderr)
|
||||
return 1
|
||||
spv = out / "const_ps.spv"
|
||||
data = spv.read_bytes()
|
||||
if len(data) < 20 or struct.unpack("<I", data[:4])[0] != 0x07230203:
|
||||
print("reply is not SPIR-V", file=sys.stderr)
|
||||
return 1
|
||||
v = subprocess.run([str(SPIRV_VAL), "--target-env", "vulkan1.3", str(spv)], capture_output=True, text=True)
|
||||
print(f"spirv-val exit {v.returncode} ({len(data)} bytes) {v.stdout}{v.stderr}")
|
||||
return v.returncode
|
||||
failed = 0
|
||||
layer_types = [t["type"] for t in json.loads((out / "layer_count_cs.res.json").read_text())["textures"]]
|
||||
if layer_types != [0x103]: # SamplerType Texture2D (3) | Array (0x100)
|
||||
print(f"layer_count_cs: texture types {layer_types}, expected a 2D array [259]", file=sys.stderr)
|
||||
failed = 1
|
||||
for key in ("const_ps", "image_read_cs", "layer_count_cs"):
|
||||
spv = out / f"{key}.spv"
|
||||
data = spv.read_bytes()
|
||||
if len(data) < 20 or struct.unpack("<I", data[:4])[0] != 0x07230203:
|
||||
print(f"{key}: reply is not SPIR-V", file=sys.stderr)
|
||||
return 1
|
||||
v = subprocess.run([str(SPIRV_VAL), "--target-env", "vulkan1.3", str(spv)], capture_output=True, text=True)
|
||||
print(f"{key}: spirv-val exit {v.returncode} ({len(data)} bytes) {v.stdout}{v.stderr}")
|
||||
failed |= v.returncode != 0
|
||||
return int(failed)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user