From f3d8f086891771e40b526f2c8318409c97b05e4e Mon Sep 17 00:00:00 2001 From: chrg6 Date: Fri, 13 Nov 2026 17:12:31 -0500 Subject: [PATCH] Make every packaging script refuse a translator not built from the checkout's sources, early. build_xlate.py --id-only wrote an nx2navi_xlate.id for whatever exe sat in the folder, from the sources as they are, so make_launcher's guard (check_xlate, which compares only the .id) passed an old exe beside a fresh .id. Nothing calls the option; it is gone, and only a build writes an .id. make_release.check_translator compared build/xlate's .id with build_xlate.identity(), which hashes the tree the last build prepared there, so a patch or vendor change since that build passed unseen; make_appimage.py and make_kit.py checked nothing. The fresh-prepare identity moves from make_launcher into build_xlate as source_identity(), and build_xlate.check_built(folder, exe) refuses a missing exe or .id and an .id that differs from it. make_launcher, make_release, make_appimage and make_kit stop on its reason, each before its downloads, builds or deletions: make_launcher's check moves from build_app to the top of main, before the --out folder is deleted and the toolchain lock is resolved. A translator folder built from the current sources, as the release scripts pass it (work\xlate_6866a13e), passes as before. Not reproduced before the change beyond the audit's probes. tests/xlate_package_check.py (new) checks a matching folder, missing files, another identity and a stale build (refused although identity() over its own tree equals its .id), that --id-only is refused, and that make_launcher, make_release and make_kit stop on a foreign translator, the kit before any download; check_built passes on work\xlate_6866a13e. Co-Authored-By: Claude Opus 5.5 --- docs/building.md | 6 ++ tests/xlate_package_check.py | 143 +++++++++++++++++++++++++++++++++++ tools/build_xlate.py | 46 +++++++++-- tools/make_appimage.py | 6 +- tools/make_kit.py | 7 +- tools/make_launcher.py | 34 ++------- tools/make_release.py | 12 +-- 7 files changed, 213 insertions(+), 41 deletions(-) create mode 100644 tests/xlate_package_check.py diff --git a/docs/building.md b/docs/building.md index 290e37b9..92da41be 100644 --- a/docs/building.md +++ b/docs/building.md @@ -813,6 +813,12 @@ A change to the SM86 tensor lowering (`tools/xlate/AmpereLowering.cs` Hmma, Ldsm `tests\maxwell_warp_emu.py`, a model of what the shader translator makes of Maxwell code (the docstring names the vector file and what is checked). `tests\maxwell_warp_emu_selftest.py` checks the emulator itself. No game files. +A change to how the packaging scripts check the translator they ship (`tools/build_xlate.py` `source_identity`, +`check_built`; `make_launcher.py`, `make_release.py`, `make_appimage.py`, `make_kit.py`) passes +`python tests\xlate_package_check.py`: a folder whose `nx2navi_xlate.id` names this checkout's sources passes, a +missing file, another identity and a folder built before a patch changed are refused, `build_xlate.py` writes no `.id` +without a build, and each script stops before its slow steps. It needs git on PATH; no game files. + ## Reverse engineering (optional) - Ghidra 12.1.2 with the Ghidra-Switch-Loader extension in `toolchain\ghidra\` (JDK 21) opens the game modules (NSO). diff --git a/tests/xlate_package_check.py b/tests/xlate_package_check.py new file mode 100644 index 00000000..70bc4bc5 --- /dev/null +++ b/tests/xlate_package_check.py @@ -0,0 +1,143 @@ +"""Check the shader translator guard of the packaging scripts (tools/build_xlate.py source_identity and check_built, +used by tools/make_launcher.py check_xlate, tools/make_release.py check_translator, tools/make_kit.py and +tools/make_appimage.py): + +- a folder whose nx2navi_xlate.id names this checkout's sources passes; a missing exe or .id is refused; +- a folder built before a patch changed is refused, although identity() over its own prepared tree (what + make_release.py compared before) equals its .id; +- build_xlate.py has no option that writes an .id beside an exe it did not build (--id-only is gone); +- make_launcher.check_xlate, make_release.check_translator and make_kit stop on a foreign translator, make_kit before + it downloads or deletes anything; make_launcher and make_appimage run the check before their slow steps. + +Needs git on PATH (prepare_sources applies the patches with it) and tools/vendor. No game files; any Python 3.10+. + +Usage: xlate_package_check.py +""" +import contextlib +import inspect +import io +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(ROOT / 'tools')) +import build_xlate # noqa: E402 + +EXE = 'nx2navi_xlate.exe' + + +def folder(base, name, ident, exe=True): + d = base / name + d.mkdir(parents=True) + if exe: + (d / EXE).write_bytes(b'MZ stand-in, not a translator') + if ident is not None: + (d / 'nx2navi_xlate.id').write_text(ident + '\n', encoding='ascii') + return d + + +def refused(fn, *args): + try: + with contextlib.redirect_stdout(io.StringIO()): + fn(*args) + except SystemExit as e: + return str(e.code) + return None + + +def main(): + if not shutil.which('git'): + print('xlate package check: git is not on PATH; skipped') + return 0 + with contextlib.redirect_stdout(io.StringIO()): + want = build_xlate.source_identity() + assert len(want) == 64 and build_xlate.BUILD == ROOT / 'build' / 'xlate', 'source_identity restores BUILD' + + with tempfile.TemporaryDirectory(prefix='xlate-pkg-') as t: + base = Path(t) + good = folder(base, 'good', want) + no_exe = folder(base, 'no_exe', want, exe=False) + no_id = folder(base, 'no_id', None) + other = folder(base, 'other', '4d051299931d390e' + '0' * 48) + + # A folder built before a patch changed: its prepared tree differs from a fresh one by one file, and its .id + # is identity() over that tree, as build_xlate.main wrote it then. + stale = base / 'stale' + old = build_xlate.BUILD + build_xlate.BUILD = stale + try: + with contextlib.redirect_stdout(io.StringIO()): + tree = build_xlate.prepare_sources() + victim = sorted(p for p in (tree / 'src').rglob('*.cs'))[0] + victim.unlink() + stale_id = build_xlate.identity() + finally: + build_xlate.BUILD = old + (stale / EXE).write_bytes(b'MZ stand-in, not a translator') + (stale / 'nx2navi_xlate.id').write_text(stale_id + '\n', encoding='ascii') + assert stale_id != want + + def check(d): + with contextlib.redirect_stdout(io.StringIO()): + return build_xlate.check_built(d, EXE) + + assert check(good) is None + assert 'missing' in check(no_exe) + assert 'missing' in check(no_id) + assert 'other sources' in check(other) + assert 'other sources' in check(stale) + print('check_built: this checkout\'s translator passes; missing files, another identity and a stale build ' + 'are refused') + + # --id-only is gone: the option is refused and writes nothing. + bare = folder(base, 'bare', None) + r = subprocess.run([sys.executable, str(ROOT / 'tools' / 'build_xlate.py'), '--id-only', '--out', str(bare)], + capture_output=True, text=True) + assert r.returncode != 0 and not (bare / 'nx2navi_xlate.id').exists(), r.stdout + r.stderr + print('build_xlate.py: --id-only refused, no .id written') + + # The packaging scripts, with the fresh identity computed once. + build_xlate.source_identity = lambda: want + import make_launcher + import make_release + import make_kit + assert refused(make_launcher.check_xlate, good) is None + for bad in (other, stale, no_exe): + assert refused(make_launcher.check_xlate, bad) + main_root = make_release.MAIN_ROOT + try: + for d, ok in ((good, True), (stale, False), (other, False)): + rel = base / f'rel_{d.name}' + shutil.copytree(d, rel / 'build' / 'xlate') + make_release.MAIN_ROOT = rel + assert (refused(make_release.check_translator) is None) == ok, d.name + finally: + make_release.MAIN_ROOT = main_root + out = base / 'kit_out' + argv = sys.argv + try: + sys.argv = ['make_kit.py', '--xlate', str(stale), '--out', str(out)] + why = refused(make_kit.main) + finally: + sys.argv = argv + assert why and 'other sources' in why and not out.exists() + print('make_launcher, make_release and make_kit stop on a foreign translator (make_kit before any download)') + + # The check comes before the slow steps. + src = inspect.getsource(make_launcher.main) + assert src.index('check_xlate(a.xlate)') < src.index('shutil.rmtree(out') < src.index('toolchain.make_lock') + import make_appimage + src = inspect.getsource(make_appimage.main) + assert src.index('build_xlate.check_built(') < src.index('dlss_pin.resolve(') < src.index('shutil.rmtree(app') + src = inspect.getsource(make_release.main) + assert src.index('check_translator()') < src.index('shutil.rmtree(out)') + print('xlate package check: passed') + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/build_xlate.py b/tools/build_xlate.py index 3f74b071..e70b12f5 100644 --- a/tools/build_xlate.py +++ b/tools/build_xlate.py @@ -7,10 +7,11 @@ NativeAOT links with the system's clang (no MSVC); NativeAOT builds for the OS i Next to the exe it writes nx2navi_xlate.id: the translator identity, a sha256 over everything the build reads (this script, Program.cs, the csproj, the patches, the vendor tree). runtime/gfx_shader.c puts it into every shader cache key, so a -translator change re-keys the cache without a hand-made KEY_VERSION bump. --id-only writes it for the exe already built -(the caller vouches that the exe matches the sources). +translator change re-keys the cache without a hand-made KEY_VERSION bump. Only a build writes it: the packaging scripts +(make_launcher.py, make_release.py, make_appimage.py, make_kit.py) take a translator whose .id names this checkout's +sources for one built from them (check_built), so no option writes an .id beside an exe it did not build. - python tools/build_xlate.py [--id-only] [--out DIR] + python tools/build_xlate.py [--out DIR] --out builds into DIR instead of build/xlate (a task build dir: nx2navi.exe takes the translator next to it). """ @@ -202,9 +203,43 @@ def write_identity(exe: Path) -> None: print(f"translator identity {ident[:16]} -> {exe.with_suffix('.id')}") +def source_identity() -> str: + """The identity a translator built from this checkout's sources gets: the vendor tree copied and patched in a + temporary folder (prepare_sources) and hashed as the build hashes it (identity). identity() over a build folder + hashes the tree its last build prepared, so a patch or vendor change made since then would not show. Nothing is + compiled; it takes a few seconds and needs git on PATH (prepare_sources applies the patches with it).""" + import tempfile + global BUILD + old = BUILD + with tempfile.TemporaryDirectory(prefix="xlate-id-") as d: + BUILD = Path(d) / "xlate" + try: + prepare_sources() + return identity() + finally: + BUILD = old + + +def check_built(folder: Path, exe: str = EXE) -> str | None: + """None when folder holds the translator exe and an nx2navi_xlate.id naming this checkout's sources + (source_identity); otherwise why not. The packaging scripts stop on the reason: 0.10.0 and 0.10.1 shipped the + 0.9.2 translator (identity 4d051299) after tools/xlate/Program.cs had raised the texture pool to 40960 slots, so + their translations still clamped every pool handle to slot 32767 and the ground went see-through after some area + loads. exe: the file name on the release's OS (EXE is this machine's).""" + folder = Path(folder) + for name in (exe, "nx2navi_xlate.id"): + if not (folder / name).is_file(): + return f"{folder / name} is missing: build the shader translator with tools/build_xlate.py --out {folder}" + have = (folder / "nx2navi_xlate.id").read_text(encoding="ascii", errors="replace").strip() + want = source_identity() + if have != want: + return (f"{folder}: its translator was built from other sources (identity {have[:16]}) than this checkout's " + f"({want[:16]}): build it again with tools/build_xlate.py --out {folder}") + return None + + def main() -> None: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) - parser.add_argument("--id-only", action="store_true", help="write nx2navi_xlate.id for the exe already built") parser.add_argument("--out", type=Path, help="build directory instead of build/xlate") parser.add_argument("--llvm", type=Path, help="Linux: an LLVM directory whose bin/clang and lld link the exe") parser.add_argument("--sysroot", type=Path, help="Linux: link against this sysroot's glibc instead of the host's " @@ -213,9 +248,6 @@ def main() -> None: if a.out: global BUILD BUILD = a.out.resolve() - if a.id_only: - write_identity(BUILD / EXE) - return started = time.time() dotnet = shutil.which("dotnet") if not dotnet: diff --git a/tools/make_appimage.py b/tools/make_appimage.py index c2a2af22..7331ff92 100644 --- a/tools/make_appimage.py +++ b/tools/make_appimage.py @@ -7,7 +7,7 @@ with --events and reads its progress lines). [--dlss libnvidia-ngx-dlss.so. | --no-dlss] The AppImage holds no game files: the kit's tools/, recomp/, runtime/ and docs, the shader translator built by -tools/build_xlate.py (bin/nx2navi_xlate), NVIDIA's DLSS library with its notice (bin/, tools/dlss_pin.py: downloaded +tools/build_xlate.py from this checkout's sources (bin/nx2navi_xlate; the script stops first on another, check_built), NVIDIA's DLSS library with its notice (bin/, tools/dlss_pin.py: downloaded from NVIDIA's DLSS SDK repository and checked, or --dlss; not with --no-dlss: an AppImage made without the library downloads it after a build on machines with NVIDIA's driver, AppRun's get_dlss), the pinned Vulkan Memory Allocator header, the pinned Vulkan headers (toolchain/vulkan-headers, tools/vulkan_pin.py: the runtime names Vulkan extensions @@ -201,6 +201,10 @@ def main(): if not need.exists(): sys.exit(f'{need} is missing (tools/build_xlate.py builds the translator; tools/build_game.py fetches VMA)') sys.path.insert(0, str(ROOT / 'tools')) + import build_xlate + reason = build_xlate.check_built(xlate, 'nx2navi_xlate') # the translator of this checkout's sources, or none + if reason: + sys.exit(reason) import dlss_pin dlss = None if a.no_dlss else dlss_pin.resolve(a.dlss, ROOT / 'build' / 'dl', linux=True) diff --git a/tools/make_kit.py b/tools/make_kit.py index bc304307..0483eabe 100644 --- a/tools/make_kit.py +++ b/tools/make_kit.py @@ -13,7 +13,8 @@ or --dlss; not with --no-dlss), DLSS Frame Generation in bin/dlssg/ with --dlssg tools/build_dlssg.py beside NVIDIA Streamline's runtime, tools/streamline_pin.py: downloaded from NVIDIA's Streamline SDK release and checked, or --streamline), AMD's FSR 3.1 library with --ffx, its two FSR 4 libraries with --ffx4, the button art with --prompt-art, and README-BuildKit.md; no game code, data or anything generated from the game. -check_content runs over the folder before it is zipped. +check_content runs over the folder before it is zipped. The --xlate folder must hold a translator built from this +checkout's sources (tools/build_xlate.py check_built); the script stops first if not. --prompt-art names folders that hold XGamepad, P5Gamepad and Keyboard_Mouse, or those three folders themselves: the pictures tools/pad_prompts.py reads go into the kit's prompt-art folder (tools/prompt_art.py art_files), and @@ -73,11 +74,15 @@ def main(): a = ap.parse_args() sys.path.insert(0, str(ROOT / 'tools')) import build_shaders + import build_xlate import dlss_pin import prompt_art import streamline_pin if a.streamline and not a.dlssg: sys.exit('--streamline goes with --dlssg') + reason = build_xlate.check_built(a.xlate, 'nx2navi_xlate.exe') # the translator of this checkout's sources + if reason: + sys.exit(reason) if a.prompt_art: prompt_art.art_files(a.prompt_art) # the art is all there before anything is downloaded dlss = None if a.no_dlss else dlss_pin.resolve(a.dlss, ROOT / 'build' / 'dl') diff --git a/tools/make_launcher.py b/tools/make_launcher.py index 2ed51a43..a990d233 100644 --- a/tools/make_launcher.py +++ b/tools/make_launcher.py @@ -188,32 +188,17 @@ def build_python(py): # ---------------------------------------------------------------- app// -def xlate_source_identity(): - """The identity tools/build_xlate.py writes for a translator built from this checkout: the vendor tree copied and - patched in a folder of its own (build_xlate.prepare_sources) and hashed as the build hashes it (build_xlate.identity). - Nothing is compiled; it takes a few seconds.""" - import tempfile - import build_xlate - old = build_xlate.BUILD - with tempfile.TemporaryDirectory(prefix='xlate-id-') as d: - build_xlate.BUILD = Path(d) / 'xlate' - try: - build_xlate.prepare_sources() - return build_xlate.identity() - finally: - build_xlate.BUILD = old - - def check_xlate(xlate): """A release carries the translator of its own sources. 0.10.0 and 0.10.1 shipped the 0.9.2 translator (identity 4d051299) after tools/xlate/Program.cs had raised the texture pool to 40960 slots, so their translations still clamped every pool handle to slot 32767 and the ground went see-through after some area loads (issues #569 and - #824). A translator whose nx2navi_xlate.id differs from this checkout's sources stops the build.""" + #824). A missing translator, or one whose nx2navi_xlate.id differs from this checkout's sources + (build_xlate.check_built), stops the build; main runs this before anything is resolved, downloaded or deleted.""" + import build_xlate + reason = build_xlate.check_built(xlate, 'nx2navi_xlate.exe') + if reason: + sys.exit(reason) have = (xlate / 'nx2navi_xlate.id').read_text(encoding='ascii').strip() - want = xlate_source_identity() - if have != want: - sys.exit(f'{xlate}: its translator was built from other sources (identity {have[:16]}) than this checkout\'s ' - f'({want[:16]}): build it again with tools/build_xlate.py --out {xlate}') print(f'translator identity {have[:16]}: built from this checkout\'s sources') @@ -233,11 +218,7 @@ def build_app(app, ver, xlate, feed_repo, lock, ffx=None, dlss=None, art=None, d shutil.copy2(ROOT / t, app / t) (app / 'VERSION').write_text(ver + '\n', encoding='utf-8') (app / 'bin').mkdir() - for n in ('nx2navi_xlate.exe', 'nx2navi_xlate.id'): - if not (xlate / n).is_file(): - sys.exit(f'{xlate / n} is missing: build the shader translator first (tools/build_xlate.py)') - check_xlate(xlate) - for n in ('nx2navi_xlate.exe', 'nx2navi_xlate.id'): + for n in ('nx2navi_xlate.exe', 'nx2navi_xlate.id'): # main checked them first (check_xlate) shutil.copy2(xlate / n, app / 'bin' / n) if ffx: # FSR 3.1 loads it from beside the game (runtime/vk_ffx.cpp); make_game.py copies it there shutil.copy2(ffx, app / 'bin' / 'amd_fidelityfx_vk.dll') @@ -415,6 +396,7 @@ def main(): prompt_art.art_files(a.prompt_art) # the art is all there before anything is downloaded or built if a.pgo: pgo.fitting(a.pgo) # the profile fits this checkout's recomp/ before anything is downloaded or built + check_xlate(a.xlate) # the translator is this checkout's before anything is downloaded, built or deleted out = a.out shutil.rmtree(out, ignore_errors=True) folder = out / 'NX2navi' diff --git a/tools/make_release.py b/tools/make_release.py index 9df4d11f..1704be43 100644 --- a/tools/make_release.py +++ b/tools/make_release.py @@ -306,12 +306,12 @@ def build_llvm(out): def check_translator(): """Stop on a translator older than the sources. build/xlate is built by hand and only copied here (the Linux release builds its own), so without a rebuild a translator change reaches no Windows release, and nothing fails: the runtime - sends job fields the old exe ignores, as happened to gather_bias.""" - idf = MAIN_ROOT / 'build' / 'xlate' / 'nx2navi_xlate.id' - have = idf.read_text(encoding='ascii').strip() if idf.is_file() else 'none' - want = build_xlate.identity() - if have != want: - sys.exit(f'build/xlate has translator {have[:16]}, the sources are {want[:16]}: run tools/build_xlate.py') + sends job fields the old exe ignores, as happened to gather_bias. The identity it must have is that of this + checkout's sources prepared afresh (build_xlate.check_built), not build_xlate.identity() over build/xlate, which + hashes the tree its last build prepared and so matches a translator built before a patch changed.""" + reason = build_xlate.check_built(MAIN_ROOT / 'build' / 'xlate', 'nx2navi_xlate.exe') + if reason: + sys.exit(reason) def build_bin(out):