diff --git a/docs/building.md b/docs/building.md index 451f4645..db8220d0 100644 --- a/docs/building.md +++ b/docs/building.md @@ -822,10 +822,13 @@ A change to the SM86 tensor lowering (`tools/xlate/AmpereLowering.cs` Hmma, Ldsm vector file and what is checked). `tests\maxwell_warp_emu_selftest.py` checks the emulator itself. No game files. A change to how the packaging scripts check the translator they ship (`tools/build_xlate.py` `source_identity`, -`check_built`; `make_launcher.py`, `make_release.py`, `make_appimage.py`, `make_kit.py`) passes -`python tests\xlate_package_check.py`: a folder whose `nx2navi_xlate.id` names this checkout's sources passes, a -missing file, another identity and a folder built before a patch changed are refused, `build_xlate.py` writes no `.id` -without a build, and each script stops before its slow steps. It needs git on PATH; no game files. +`check_built`; `make_launcher.py`, `make_release.py`, `make_appimage.py`, `make_kit.py`; `tools/xlate_smoke.py` +`pool_check`) passes `python tests\xlate_package_check.py`: a folder whose `nx2navi_xlate.id` names this checkout's +sources passes, a missing file, another identity and a folder built before a patch changed are refused, +`build_xlate.py` writes no `.id` without a build, each script stops before its slow steps, and on Windows +`make_launcher.py` also refuses an exe that does not run beside a matching `.id`. It needs git on PATH; no game files. +`tools\xlate_smoke.py --exe ` checks a translator's texture pool length against +`tools/xlate/Program.cs` `TexturePoolLength`: one older than its sources fails it. ## Reverse engineering (optional) diff --git a/tests/xlate_package_check.py b/tests/xlate_package_check.py index 70bc4bc5..8974e296 100644 --- a/tests/xlate_package_check.py +++ b/tests/xlate_package_check.py @@ -7,7 +7,10 @@ tools/make_appimage.py): make_release.py compared before) equals its .id; - build_xlate.py has no option that writes an .id beside an exe it did not build (--id-only is gone); - make_launcher.check_xlate, make_release.check_translator and make_kit stop on a foreign translator, make_kit before - it downloads or deletes anything; make_launcher and make_appimage run the check before their slow steps. + it downloads or deletes anything; make_launcher and make_appimage run the check before their slow steps; +- tools/xlate_smoke.py pool_check, which make_launcher.check_xlate runs on the exe on Windows: spirv_clamps reads the + array lengths and the UMin clamps of a module made here, pool_check refuses an exe that does not run, check_xlate + refuses such an exe beside a matching .id, and a translator built here (build/xlate) passes when there is one. Needs git on PATH (prepare_sources applies the patches with it) and tools/vendor. No game files; any Python 3.10+. @@ -16,7 +19,9 @@ Usage: xlate_package_check.py import contextlib import inspect import io +import os import shutil +import struct import subprocess import sys import tempfile @@ -26,6 +31,7 @@ ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) sys.path.insert(0, str(ROOT / 'tools')) import build_xlate # noqa: E402 +import xlate_smoke # noqa: E402 EXE = 'nx2navi_xlate.exe' @@ -40,6 +46,39 @@ def folder(base, name, ident, exe=True): return d +def spirv(*instructions): + """A SPIR-V module of these instructions, each (opcode, operand words...), after a five-word header.""" + words = [0x07230203, 0x00010300, 0, 100, 0] + for op, *ops in instructions: + words += [(len(ops) + 1) << 16 | op, *ops] + return struct.pack(f'<{len(words)}I', *words) + + +def pool_checks(base): + name = struct.unpack('<4I', b'GLSL.std.450\0\0\0\0') + module = spirv((11, 1, *name), (43, 2, 3, 40960), (43, 2, 4, 40959), (43, 2, 5, 7), (28, 6, 2, 3), + (12, 2, 7, 1, 38, 5, 4), (12, 2, 8, 1, 37, 5, 3), (12, 2, 9, 99, 38, 5, 5)) + assert xlate_smoke.spirv_clamps(module) == ([40960], [7, 40959]), xlate_smoke.spirv_clamps(module) + print('spirv_clamps: array lengths and the GLSL.std.450 UMin constants only') + if os.name != 'nt': + return + stand_in = folder(base, 'stand_in', build_xlate.source_identity()) + why = xlate_smoke.pool_check(stand_in / EXE) + assert why and 'does not run' in why, why + why = refused(make_launcher_check(), stand_in) + assert why and 'does not run' in why, why + built = ROOT / 'build' / 'xlate' / EXE + if built.is_file() and not build_xlate.check_built(built.parent, EXE): + assert xlate_smoke.pool_check(built) is None, xlate_smoke.pool_check(built) + print('pool_check: the translator built here passes') + print('pool_check: an exe that does not run is refused, also by make_launcher.check_xlate beside a matching .id') + + +def make_launcher_check(): + import make_launcher + return make_launcher.check_xlate + + def refused(fn, *args): try: with contextlib.redirect_stdout(io.StringIO()): @@ -102,9 +141,11 @@ def main(): # The packaging scripts, with the fresh identity computed once. build_xlate.source_identity = lambda: want + pool_checks(base) import make_launcher import make_release import make_kit + xlate_smoke.pool_check = lambda exe: None # the stand-in exes do not run: pool_checks covers the real one assert refused(make_launcher.check_xlate, good) is None for bad in (other, stale, no_exe): assert refused(make_launcher.check_xlate, bad) diff --git a/tools/make_launcher.py b/tools/make_launcher.py index a990d233..99688afc 100644 --- a/tools/make_launcher.py +++ b/tools/make_launcher.py @@ -193,11 +193,18 @@ def check_xlate(xlate): 4d051299) after tools/xlate/Program.cs had raised the texture pool to 40960 slots, so their translations still clamped every pool handle to slot 32767 and the ground went see-through after some area loads (issues #569 and #824). A missing translator, or one whose nx2navi_xlate.id differs from this checkout's sources - (build_xlate.check_built), stops the build; main runs this before anything is resolved, downloaded or deleted.""" + (build_xlate.check_built), stops the build; main runs this before anything is resolved, downloaded or deleted. + On Windows the exe itself is asked too, as the .id says nothing of the exe beside it: a translator that declares + the texture pool with another length than its sources, or does not run, stops the build (xlate_smoke.pool_check).""" import build_xlate reason = build_xlate.check_built(xlate, 'nx2navi_xlate.exe') if reason: sys.exit(reason) + if os.name == 'nt': + import xlate_smoke + reason = xlate_smoke.pool_check(xlate / 'nx2navi_xlate.exe') + if reason: + sys.exit(f'{xlate / "nx2navi_xlate.exe"}: {reason}') have = (xlate / 'nx2navi_xlate.id').read_text(encoding='ascii').strip() print(f'translator identity {have[:16]}: built from this checkout\'s sources') diff --git a/tools/xlate_smoke.py b/tools/xlate_smoke.py index 1b22f707..5b7fd7e1 100644 --- a/tools/xlate_smoke.py +++ b/tools/xlate_smoke.py @@ -7,11 +7,17 @@ declares such an image without a format, and its read needs StorageImageReadWith layer count of texture handle 0x10, for which the translator has no type answer: the texture must be declared as an array, or the count reads as 1. Every module is validated with spirv-val. - python tools/xlate_smoke.py +Last, pool_check sends a fragment shader that queries a texture whose handle it reads from an input, which the +translator reads from the texture pool array: the array must have tools/xlate/Program.cs TexturePoolLength entries and +the handle must be clamped to the last of them. A translator older than its sources (0.10.0 and 0.10.1 shipped one +whose pool had 32768 entries) fails it. tools/make_launcher.py check_xlate runs pool_check on the translator it ships. + + python tools/xlate_smoke.py [--exe ] """ import json import os +import re import shutil import struct import subprocess @@ -71,9 +77,82 @@ def layer_count_code() -> bytes: return struct.pack(f"<{len(words)}Q", *words) +IPA = 0xE0 << 56 # IPA, pass mode +TXQ_B = 0x1BEA # opcode bits 51-63 of TXQ.B (handle in a register) + + +def pool_query_code() -> bytes: + """Fragment code: R4 = user attribute 0.x (IPA of a[0x80]), R0 = the width of the texture whose handle is R4 (TXQ.B + of the dimension, mask x), R1-R3 constants; render target 0 = R0-R3.""" + header = [0] * 20 + header[0] = 2 | (3 << 5) | (5 << 10) # SPH type 2, version 3, stage fragment + header[18] = 0xF # OMAP: render target 0, components xyzw + ipa = IPA | (7 << 47) | (RZ << 39) | (0x80 << 28) | (RZ << 20) | PT | (RZ << 8) | 4 + txq = (TXQ_B << 51) | (0x1 << 31) | (1 << 22) | PT | (4 << 8) | 0 + words = [SCHED, ipa, txq, mov32i(1, 0.0), SCHED, mov32i(2, 0.0), mov32i(3, 1.0), EXIT, 0] + return struct.pack("<20I", *header) + struct.pack(f"<{len(words)}Q", *words) + + +def spirv_clamps(spv: bytes): + """The lengths of a module's OpTypeArray types and the 32-bit constants its GLSL.std.450 UMin instructions take.""" + w = struct.unpack(f"<{len(spv) // 4}I", spv[:len(spv) // 4 * 4]) + consts, lengths, umin, glsl = {}, [], [], None + i = 5 # after the header + while i < len(w): + n, op = w[i] >> 16, w[i] & 0xFFFF + if not n: + break + ins = w[i:i + n] + if op == 11 and struct.pack(f"<{n - 2}I", *ins[2:]).split(b"\0")[0] == b"GLSL.std.450": # OpExtInstImport + glsl = ins[1] + elif op == 43 and n == 4: # OpConstant: type, id, value + consts[ins[2]] = ins[3] + elif op == 28: # OpTypeArray: id, element type, length + lengths.append(ins[3]) + elif op == 12 and n > 5 and ins[3] == glsl and ins[4] == 38: # OpExtInst UMin + umin += ins[5:] + i += n + return [consts.get(x) for x in lengths], [consts[x] for x in umin if x in consts] + + +def pool_check(exe: Path): + """None when exe declares the texture pool array with tools/xlate/Program.cs TexturePoolLength entries and clamps a + pool handle to the last of them; otherwise why not.""" + m = re.search(r"TexturePoolLength = (\d+)", (ROOT / "tools" / "xlate" / "Program.cs").read_text(encoding="utf-8")) + if not m: + return "tools/xlate/Program.cs names no TexturePoolLength" + length = int(m.group(1)) + with tempfile.TemporaryDirectory() as tmp: + tmp = Path(tmp) + code = tmp / "pool_ps.bin" + code.write_bytes(pool_query_code()) + out = tmp / "out" + job = {"key": "pool_ps", "stage": "ps", "code": str(code), "debug": False, "fout": {"0": "float"}} + try: + r = subprocess.run([str(exe), "--serve"], input=json.dumps({"job": job, "out": str(out)}) + "\n", + capture_output=True, text=True, timeout=60) + except (OSError, subprocess.SubprocessError) as e: + return f"the translator does not run: {e}" + if r.stdout.strip() != "ok pool_ps": + return f"the translator answered {r.stdout.strip()[:120]!r} to the texture pool shader" + try: + textures = json.loads((out / "pool_ps.res.json").read_text(encoding="utf-8")).get("textures") or [] + lengths, clamps = spirv_clamps((out / "pool_ps.spv").read_bytes()) + except (OSError, ValueError, struct.error) as e: + return f"the translator's texture pool shader cannot be read: {e}" + pools = [t.get("array") for t in textures if t.get("pool") == 1] + if pools != [length] or length not in lengths: + return (f"the translator declares the texture pool with {pools or 'no'} entries, where its sources say {length} " + f"(tools/xlate/Program.cs TexturePoolLength): it is older than its sources") + if length - 1 not in clamps: + return f"the translator clamps a texture pool handle to {clamps}, not to {length - 1}" + return None + + def main() -> int: - if not EXE.is_file(): - print(f"missing {EXE}; run tools/build_xlate.py", file=sys.stderr) + exe = Path(sys.argv[sys.argv.index("--exe") + 1]) if "--exe" in sys.argv[1:-1] else EXE + if not exe.is_file(): + print(f"missing {exe}; run tools/build_xlate.py", file=sys.stderr) return 2 with tempfile.TemporaryDirectory() as tmp: tmp = Path(tmp) @@ -94,7 +173,7 @@ def main() -> int: json.dumps({"job": image_job, "out": str(out)}), json.dumps({"job": layer_job, "out": str(out)}), ] - r = subprocess.run([str(EXE), "--serve"], input="\n".join(lines) + "\n", capture_output=True, text=True, timeout=60) + r = subprocess.run([str(exe), "--serve"], input="\n".join(lines) + "\n", capture_output=True, text=True, timeout=60) replies = r.stdout.splitlines() print("replies:", replies) if (len(replies) != 5 or replies[0] != "ok const_ps" or not replies[1].startswith("fail ") or replies[2] != "ok again" @@ -115,7 +194,9 @@ def main() -> int: v = subprocess.run([str(SPIRV_VAL), "--target-env", "vulkan1.3", str(spv)], capture_output=True, text=True) print(f"{key}: spirv-val exit {v.returncode} ({len(data)} bytes) {v.stdout}{v.stderr}") failed |= v.returncode != 0 - return int(failed) + why = pool_check(exe) + print(f"pool_ps: {why or 'the texture pool array and its clamp have the length tools/xlate/Program.cs names'}") + return int(failed or why is not None) if __name__ == "__main__":