From 78bfb83b2b2e340a202d2271c0edca90b72a1bc2 Mon Sep 17 00:00:00 2001 From: chrg6 Date: Fri, 13 Nov 2026 17:50:38 -0500 Subject: [PATCH] Ask the shipped translator itself for its texture pool length before make_launcher packages it. The review of the packaging guard (b4c21998) found that D-1's optional behaviour probe was left out without a reason: check_built compares only nx2navi_xlate.id with the checkout's sources, so an old exe beside a current .id still passed, and while the exe build record waits (G-12), a probe of the exe is the only landable check on the exe itself. It also passes the release scripts' folder, as that translator declares 40960 entries and clamps to 40959. tools/xlate_smoke.py gains pool_check(exe): it sends a hand-assembled fragment shader that reads a texture handle from an input (IPA R4 = a[0x80]; TXQ.B R0, R4; EXIT), which the translator reads from the texture pool array, and refuses an exe that does not run or answer, whose res.json texture pool entry (pool 1) has another array length than tools/xlate/Program.cs TexturePoolLength, or whose SPIR-V lacks that array or a GLSL.std.450 UMin by length - 1 (spirv_clamps reads the module itself, no disassembler needed). xlate_smoke.py runs it last and takes --exe. make_launcher.check_xlate runs pool_check on the exe after check_built on Windows, still before anything is downloaded, built or deleted. Not reproduced before the change beyond the audit's probe. On this PC, pool_check passes on work\xlate_6866a13e (and make_launcher.check_xlate on that folder), refuses the 0.9.2 translator (identity 4d051299) with "declares the texture pool with [32768] entries, where its sources say 40960", also through check_xlate beside a current .id, and xlate_smoke.py --exe passes on work\xlate_6866a13e. tests/xlate_package_check.py now checks spirv_clamps on a module it assembles, that pool_check and check_xlate refuse an exe that does not run beside a matching .id, and that a translator built in build/xlate passes when there is one; it passes. docs/building.md names pool_check. Co-Authored-By: Claude Opus 5.5 --- docs/building.md | 11 +++-- tests/xlate_package_check.py | 43 ++++++++++++++++- tools/make_launcher.py | 9 +++- tools/xlate_smoke.py | 91 ++++++++++++++++++++++++++++++++++-- 4 files changed, 143 insertions(+), 11 deletions(-) diff --git a/docs/building.md b/docs/building.md index 451f4645..db8220d0 100644 --- a/docs/building.md +++ b/docs/building.md @@ -822,10 +822,13 @@ A change to the SM86 tensor lowering (`tools/xlate/AmpereLowering.cs` Hmma, Ldsm vector file and what is checked). `tests\maxwell_warp_emu_selftest.py` checks the emulator itself. No game files. A change to how the packaging scripts check the translator they ship (`tools/build_xlate.py` `source_identity`, -`check_built`; `make_launcher.py`, `make_release.py`, `make_appimage.py`, `make_kit.py`) passes -`python tests\xlate_package_check.py`: a folder whose `nx2navi_xlate.id` names this checkout's sources passes, a -missing file, another identity and a folder built before a patch changed are refused, `build_xlate.py` writes no `.id` -without a build, and each script stops before its slow steps. It needs git on PATH; no game files. +`check_built`; `make_launcher.py`, `make_release.py`, `make_appimage.py`, `make_kit.py`; `tools/xlate_smoke.py` +`pool_check`) passes `python tests\xlate_package_check.py`: a folder whose `nx2navi_xlate.id` names this checkout's +sources passes, a missing file, another identity and a folder built before a patch changed are refused, +`build_xlate.py` writes no `.id` without a build, each script stops before its slow steps, and on Windows +`make_launcher.py` also refuses an exe that does not run beside a matching `.id`. It needs git on PATH; no game files. +`tools\xlate_smoke.py --exe ` checks a translator's texture pool length against +`tools/xlate/Program.cs` `TexturePoolLength`: one older than its sources fails it. ## Reverse engineering (optional) diff --git a/tests/xlate_package_check.py b/tests/xlate_package_check.py index 70bc4bc5..8974e296 100644 --- a/tests/xlate_package_check.py +++ b/tests/xlate_package_check.py @@ -7,7 +7,10 @@ tools/make_appimage.py): make_release.py compared before) equals its .id; - build_xlate.py has no option that writes an .id beside an exe it did not build (--id-only is gone); - make_launcher.check_xlate, make_release.check_translator and make_kit stop on a foreign translator, make_kit before - it downloads or deletes anything; make_launcher and make_appimage run the check before their slow steps. + it downloads or deletes anything; make_launcher and make_appimage run the check before their slow steps; +- tools/xlate_smoke.py pool_check, which make_launcher.check_xlate runs on the exe on Windows: spirv_clamps reads the + array lengths and the UMin clamps of a module made here, pool_check refuses an exe that does not run, check_xlate + refuses such an exe beside a matching .id, and a translator built here (build/xlate) passes when there is one. Needs git on PATH (prepare_sources applies the patches with it) and tools/vendor. No game files; any Python 3.10+. @@ -16,7 +19,9 @@ Usage: xlate_package_check.py import contextlib import inspect import io +import os import shutil +import struct import subprocess import sys import tempfile @@ -26,6 +31,7 @@ ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) sys.path.insert(0, str(ROOT / 'tools')) import build_xlate # noqa: E402 +import xlate_smoke # noqa: E402 EXE = 'nx2navi_xlate.exe' @@ -40,6 +46,39 @@ def folder(base, name, ident, exe=True): return d +def spirv(*instructions): + """A SPIR-V module of these instructions, each (opcode, operand words...), after a five-word header.""" + words = [0x07230203, 0x00010300, 0, 100, 0] + for op, *ops in instructions: + words += [(len(ops) + 1) << 16 | op, *ops] + return struct.pack(f'<{len(words)}I', *words) + + +def pool_checks(base): + name = struct.unpack('<4I', b'GLSL.std.450\0\0\0\0') + module = spirv((11, 1, *name), (43, 2, 3, 40960), (43, 2, 4, 40959), (43, 2, 5, 7), (28, 6, 2, 3), + (12, 2, 7, 1, 38, 5, 4), (12, 2, 8, 1, 37, 5, 3), (12, 2, 9, 99, 38, 5, 5)) + assert xlate_smoke.spirv_clamps(module) == ([40960], [7, 40959]), xlate_smoke.spirv_clamps(module) + print('spirv_clamps: array lengths and the GLSL.std.450 UMin constants only') + if os.name != 'nt': + return + stand_in = folder(base, 'stand_in', build_xlate.source_identity()) + why = xlate_smoke.pool_check(stand_in / EXE) + assert why and 'does not run' in why, why + why = refused(make_launcher_check(), stand_in) + assert why and 'does not run' in why, why + built = ROOT / 'build' / 'xlate' / EXE + if built.is_file() and not build_xlate.check_built(built.parent, EXE): + assert xlate_smoke.pool_check(built) is None, xlate_smoke.pool_check(built) + print('pool_check: the translator built here passes') + print('pool_check: an exe that does not run is refused, also by make_launcher.check_xlate beside a matching .id') + + +def make_launcher_check(): + import make_launcher + return make_launcher.check_xlate + + def refused(fn, *args): try: with contextlib.redirect_stdout(io.StringIO()): @@ -102,9 +141,11 @@ def main(): # The packaging scripts, with the fresh identity computed once. build_xlate.source_identity = lambda: want + pool_checks(base) import make_launcher import make_release import make_kit + xlate_smoke.pool_check = lambda exe: None # the stand-in exes do not run: pool_checks covers the real one assert refused(make_launcher.check_xlate, good) is None for bad in (other, stale, no_exe): assert refused(make_launcher.check_xlate, bad) diff --git a/tools/make_launcher.py b/tools/make_launcher.py index a990d233..99688afc 100644 --- a/tools/make_launcher.py +++ b/tools/make_launcher.py @@ -193,11 +193,18 @@ def check_xlate(xlate): 4d051299) after tools/xlate/Program.cs had raised the texture pool to 40960 slots, so their translations still clamped every pool handle to slot 32767 and the ground went see-through after some area loads (issues #569 and #824). A missing translator, or one whose nx2navi_xlate.id differs from this checkout's sources - (build_xlate.check_built), stops the build; main runs this before anything is resolved, downloaded or deleted.""" + (build_xlate.check_built), stops the build; main runs this before anything is resolved, downloaded or deleted. + On Windows the exe itself is asked too, as the .id says nothing of the exe beside it: a translator that declares + the texture pool with another length than its sources, or does not run, stops the build (xlate_smoke.pool_check).""" import build_xlate reason = build_xlate.check_built(xlate, 'nx2navi_xlate.exe') if reason: sys.exit(reason) + if os.name == 'nt': + import xlate_smoke + reason = xlate_smoke.pool_check(xlate / 'nx2navi_xlate.exe') + if reason: + sys.exit(f'{xlate / "nx2navi_xlate.exe"}: {reason}') have = (xlate / 'nx2navi_xlate.id').read_text(encoding='ascii').strip() print(f'translator identity {have[:16]}: built from this checkout\'s sources') diff --git a/tools/xlate_smoke.py b/tools/xlate_smoke.py index 1b22f707..5b7fd7e1 100644 --- a/tools/xlate_smoke.py +++ b/tools/xlate_smoke.py @@ -7,11 +7,17 @@ declares such an image without a format, and its read needs StorageImageReadWith layer count of texture handle 0x10, for which the translator has no type answer: the texture must be declared as an array, or the count reads as 1. Every module is validated with spirv-val. - python tools/xlate_smoke.py +Last, pool_check sends a fragment shader that queries a texture whose handle it reads from an input, which the +translator reads from the texture pool array: the array must have tools/xlate/Program.cs TexturePoolLength entries and +the handle must be clamped to the last of them. A translator older than its sources (0.10.0 and 0.10.1 shipped one +whose pool had 32768 entries) fails it. tools/make_launcher.py check_xlate runs pool_check on the translator it ships. + + python tools/xlate_smoke.py [--exe ] """ import json import os +import re import shutil import struct import subprocess @@ -71,9 +77,82 @@ def layer_count_code() -> bytes: return struct.pack(f"<{len(words)}Q", *words) +IPA = 0xE0 << 56 # IPA, pass mode +TXQ_B = 0x1BEA # opcode bits 51-63 of TXQ.B (handle in a register) + + +def pool_query_code() -> bytes: + """Fragment code: R4 = user attribute 0.x (IPA of a[0x80]), R0 = the width of the texture whose handle is R4 (TXQ.B + of the dimension, mask x), R1-R3 constants; render target 0 = R0-R3.""" + header = [0] * 20 + header[0] = 2 | (3 << 5) | (5 << 10) # SPH type 2, version 3, stage fragment + header[18] = 0xF # OMAP: render target 0, components xyzw + ipa = IPA | (7 << 47) | (RZ << 39) | (0x80 << 28) | (RZ << 20) | PT | (RZ << 8) | 4 + txq = (TXQ_B << 51) | (0x1 << 31) | (1 << 22) | PT | (4 << 8) | 0 + words = [SCHED, ipa, txq, mov32i(1, 0.0), SCHED, mov32i(2, 0.0), mov32i(3, 1.0), EXIT, 0] + return struct.pack("<20I", *header) + struct.pack(f"<{len(words)}Q", *words) + + +def spirv_clamps(spv: bytes): + """The lengths of a module's OpTypeArray types and the 32-bit constants its GLSL.std.450 UMin instructions take.""" + w = struct.unpack(f"<{len(spv) // 4}I", spv[:len(spv) // 4 * 4]) + consts, lengths, umin, glsl = {}, [], [], None + i = 5 # after the header + while i < len(w): + n, op = w[i] >> 16, w[i] & 0xFFFF + if not n: + break + ins = w[i:i + n] + if op == 11 and struct.pack(f"<{n - 2}I", *ins[2:]).split(b"\0")[0] == b"GLSL.std.450": # OpExtInstImport + glsl = ins[1] + elif op == 43 and n == 4: # OpConstant: type, id, value + consts[ins[2]] = ins[3] + elif op == 28: # OpTypeArray: id, element type, length + lengths.append(ins[3]) + elif op == 12 and n > 5 and ins[3] == glsl and ins[4] == 38: # OpExtInst UMin + umin += ins[5:] + i += n + return [consts.get(x) for x in lengths], [consts[x] for x in umin if x in consts] + + +def pool_check(exe: Path): + """None when exe declares the texture pool array with tools/xlate/Program.cs TexturePoolLength entries and clamps a + pool handle to the last of them; otherwise why not.""" + m = re.search(r"TexturePoolLength = (\d+)", (ROOT / "tools" / "xlate" / "Program.cs").read_text(encoding="utf-8")) + if not m: + return "tools/xlate/Program.cs names no TexturePoolLength" + length = int(m.group(1)) + with tempfile.TemporaryDirectory() as tmp: + tmp = Path(tmp) + code = tmp / "pool_ps.bin" + code.write_bytes(pool_query_code()) + out = tmp / "out" + job = {"key": "pool_ps", "stage": "ps", "code": str(code), "debug": False, "fout": {"0": "float"}} + try: + r = subprocess.run([str(exe), "--serve"], input=json.dumps({"job": job, "out": str(out)}) + "\n", + capture_output=True, text=True, timeout=60) + except (OSError, subprocess.SubprocessError) as e: + return f"the translator does not run: {e}" + if r.stdout.strip() != "ok pool_ps": + return f"the translator answered {r.stdout.strip()[:120]!r} to the texture pool shader" + try: + textures = json.loads((out / "pool_ps.res.json").read_text(encoding="utf-8")).get("textures") or [] + lengths, clamps = spirv_clamps((out / "pool_ps.spv").read_bytes()) + except (OSError, ValueError, struct.error) as e: + return f"the translator's texture pool shader cannot be read: {e}" + pools = [t.get("array") for t in textures if t.get("pool") == 1] + if pools != [length] or length not in lengths: + return (f"the translator declares the texture pool with {pools or 'no'} entries, where its sources say {length} " + f"(tools/xlate/Program.cs TexturePoolLength): it is older than its sources") + if length - 1 not in clamps: + return f"the translator clamps a texture pool handle to {clamps}, not to {length - 1}" + return None + + def main() -> int: - if not EXE.is_file(): - print(f"missing {EXE}; run tools/build_xlate.py", file=sys.stderr) + exe = Path(sys.argv[sys.argv.index("--exe") + 1]) if "--exe" in sys.argv[1:-1] else EXE + if not exe.is_file(): + print(f"missing {exe}; run tools/build_xlate.py", file=sys.stderr) return 2 with tempfile.TemporaryDirectory() as tmp: tmp = Path(tmp) @@ -94,7 +173,7 @@ def main() -> int: json.dumps({"job": image_job, "out": str(out)}), json.dumps({"job": layer_job, "out": str(out)}), ] - r = subprocess.run([str(EXE), "--serve"], input="\n".join(lines) + "\n", capture_output=True, text=True, timeout=60) + r = subprocess.run([str(exe), "--serve"], input="\n".join(lines) + "\n", capture_output=True, text=True, timeout=60) replies = r.stdout.splitlines() print("replies:", replies) if (len(replies) != 5 or replies[0] != "ok const_ps" or not replies[1].startswith("fail ") or replies[2] != "ok again" @@ -115,7 +194,9 @@ def main() -> int: v = subprocess.run([str(SPIRV_VAL), "--target-env", "vulkan1.3", str(spv)], capture_output=True, text=True) print(f"{key}: spirv-val exit {v.returncode} ({len(data)} bytes) {v.stdout}{v.stderr}") failed |= v.returncode != 0 - return int(failed) + why = pool_check(exe) + print(f"pool_ps: {why or 'the texture pool array and its clamp have the length tools/xlate/Program.cs names'}") + return int(failed or why is not None) if __name__ == "__main__":