Files
NX2navi/tools/build_xlate.py
chrg6 fa6f9717fb Reach buffers through a device address table where the translator finds no storage buffer, and lower MATCH.ALL and ATOMS.POPC.INC.
Kokiri Forest drew black, in tiles and in shadowed and indoor areas, where shaders read light, probe and instance data through pointers they load from other buffers at an index (a phi of a pushed pointer and a loaded one, a pointer spilled to local memory): those reads returned 0. Translator patch 0030 keeps such accesses global, with a new first source naming the table, and the SPIR-V back end looks the address's 4 KiB page up in it and loads through the buffer device address it gives (PhysicalStorageBuffer64). runtime/vk_gpa.c keeps the table: two levels over the 40-bit GPU address space, page entries naming immutable records, records reused only after the GPU finished with them. Imported pools, their VRAM mirrors and the pages of virtual pools go in it; pools in it are destroyed after the submissions that may still read them. The last failing compute program needed MATCH.ALL (subgroupAllEqual) and the aggregated shared increment ATOMS.POPC.INC, now lowered in AmpereLowering.cs; Kokiri Forest translates without a failure.

NX2NAVI_GPA=0 turns the table off and NX2NAVI_GPA_EMPTY=1 keeps it empty. A first try synced every VRAM mirror at each table reader's draw, which overwrote what the GPU had written into them and made the black worse; mirrors sync only when bound.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-07 05:04:40 -05:00

187 lines
8.3 KiB
Python

"""Build the shader translator: build/xlate/nx2navi_xlate.exe (vendored translator 1.3.2 + patches, NativeAOT, SPIR-V for Vulkan).
Copies tools/vendor/ryujinx to build/xlate/ryujinx, applies tools/xlate/patches/*.patch there (tools/vendor stays
byte-identical), then runs `dotnet publish`. Needs the .NET 10 SDK, the MSVC build
tools and vswhere.exe (docs/building.md). Run with .venv. On Linux it builds build/xlate/nx2navi_xlate for linux-x64, which
NativeAOT links with the system's clang (no MSVC); NativeAOT builds for the OS it runs on only.
Next to the exe it writes nx2navi_xlate.id: the translator identity, a sha256 over everything the build reads (this script,
Program.cs, the csproj, the patches, the vendor tree). runtime/gfx_shader.c puts it into every shader cache key, so a
translator change re-keys the cache without a hand-made KEY_VERSION bump. --id-only writes it for the exe already built
(the caller vouches that the exe matches the sources).
python tools/build_xlate.py [--id-only] [--out DIR]
--out builds into DIR instead of build/xlate (a task build dir: nx2navi.exe takes the translator next to it).
"""
import argparse
import hashlib
import os
import shutil
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
VENDOR = ROOT / "tools" / "vendor" / "ryujinx"
MNG_VENDOR = ROOT / "refs" / "MacNativeGames" / "vendor" / "ryujinx"
XLATE = ROOT / "tools" / "xlate"
BUILD = ROOT / "build" / "xlate"
WIN = os.name == "nt"
EXE = "nx2navi_xlate.exe" if WIN else "nx2navi_xlate"
VSWHERE_DIR = Path(os.environ.get("ProgramFiles(x86)", r"C:\Program Files (x86)")) / "Microsoft Visual Studio" / "Installer"
def tree_hashes(base: Path) -> dict[str, str]:
"""Hash every source file; src/*/obj and src/*/bin are build output (an IDE restore writes them, the vendor
.gitignore ignores them and prepare_sources() drops them), so they are left out."""
def build_output(rel: Path) -> bool:
return len(rel.parts) > 2 and rel.parts[0] == "src" and rel.parts[2] in ("obj", "bin")
return {
p.relative_to(base).as_posix(): hashlib.sha256(p.read_bytes()).hexdigest()
for p in sorted(base.rglob("*"))
if p.is_file() and not build_output(p.relative_to(base))
}
def mng_pin() -> str:
"""The MacNativeGames commit tools/fetch_refs.sh pins."""
for line in (ROOT / "tools" / "fetch_refs.sh").read_text(encoding="utf-8").splitlines():
f = line.split()
if len(f) >= 4 and f[:2] == ["fetch", "MacNativeGames"]: # a fifth word asks for the history
return f[3]
sys.exit("tools/fetch_refs.sh pins no MacNativeGames commit")
def pinned_blobs() -> dict[str, str]:
"""Git blob ids of vendor/ryujinx in the refs/ checkout at the pinned commit. The checkout itself may be newer than the pin or
carry CRLF line ends (core.autocrlf), so its files are not compared."""
out = subprocess.run(["git", "-C", str(MNG_VENDOR.parent.parent), "ls-tree", "-r", mng_pin(), "vendor/ryujinx"],
capture_output=True, text=True)
if out.returncode != 0:
sys.exit(f"git ls-tree of the MacNativeGames pin failed: {out.stderr.strip()}")
blobs = {}
for line in out.stdout.splitlines():
meta, path = line.split("\t", 1)
blobs[path.removeprefix("vendor/ryujinx/")] = meta.split()[2]
return blobs
def blob_ids(base: Path) -> dict[str, str]:
"""Git blob ids of the source files tree_hashes() hashes."""
ids = {}
for rel in tree_hashes(base):
data = (base / rel).read_bytes()
ids[rel] = hashlib.sha1(b"blob %d\0" % len(data) + data).hexdigest()
return ids
def verify_vendor() -> None:
"""tools/vendor/ryujinx must be byte-identical to the refs/ checkout's copy at the pinned commit (when present)."""
if not MNG_VENDOR.is_dir():
print("refs/MacNativeGames missing: vendor comparison skipped")
return
ours, theirs = blob_ids(VENDOR), pinned_blobs()
if ours != theirs:
diff = sorted(k for k in ours.keys() | theirs.keys() if ours.get(k) != theirs.get(k))
sys.exit(f"tools/vendor/ryujinx differs from MNG in {len(diff)} files, first: {diff[:5]}")
print(f"vendor identical to MNG: {len(ours)} files")
def prepare_sources() -> Path:
src = BUILD / "ryujinx"
if BUILD.exists():
shutil.rmtree(BUILD / "ryujinx", ignore_errors=True)
shutil.copytree(VENDOR, src)
for junk in list(src.glob("src/*/obj")) + list(src.glob("src/*/bin")):
shutil.rmtree(junk)
patches = sorted((XLATE / "patches").glob("*.patch"))
if len(patches) != 29:
sys.exit(f"expected 29 patches, found {len(patches)}")
for patch in patches:
# git apply works outside a repository when given a directory; --check first so a failure names the patch.
for extra in (["--check"], []):
r = subprocess.run(
["git", "apply", *extra, "--whitespace=nowarn", "--directory", str(src.relative_to(BUILD)), str(patch)],
cwd=BUILD, capture_output=True, text=True,
)
if r.returncode != 0:
sys.exit(f"patch does not apply: {patch.name}\n{r.stdout}{r.stderr}")
print(f"applied {patch.name}")
return src
def identity() -> str:
"""sha256 over the translator's inputs, in a fixed order: path and content hash of each file."""
h = hashlib.sha256()
files = [Path(__file__).resolve(), XLATE / "Program.cs", XLATE / "AmpereLowering.cs", XLATE / "MeshSpirv.cs", XLATE / "nx2navi_xlate.csproj",
*sorted((XLATE / "patches").glob("*.patch"))]
for f in files:
h.update(f"{f.name}\0{hashlib.sha256(f.read_bytes()).hexdigest()}\n".encode())
for rel, digest in tree_hashes(VENDOR).items():
h.update(f"vendor/{rel}\0{digest}\n".encode())
return h.hexdigest()
def write_identity(exe: Path) -> None:
ident = identity()
exe.with_suffix(".id").write_text(ident + "\n", encoding="ascii")
print(f"translator identity {ident[:16]} -> {exe.with_suffix('.id')}")
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("--id-only", action="store_true", help="write nx2navi_xlate.id for the exe already built")
parser.add_argument("--out", type=Path, help="build directory instead of build/xlate")
parser.add_argument("--llvm", type=Path, help="Linux: an LLVM directory whose bin/clang and lld link the exe")
parser.add_argument("--sysroot", type=Path, help="Linux: link against this sysroot's glibc instead of the host's "
"(tools/make_release_linux.py)")
a = parser.parse_args()
if a.out:
global BUILD
BUILD = a.out.resolve()
if a.id_only:
write_identity(BUILD / EXE)
return
started = time.time()
dotnet = shutil.which("dotnet")
if not dotnet:
sys.exit("missing: dotnet (.NET 10 SDK)")
env = dict(os.environ)
if WIN:
if not (VSWHERE_DIR / "vswhere.exe").is_file() and not shutil.which("vswhere"):
sys.exit(f"missing: vswhere.exe (expected in {VSWHERE_DIR}); NativeAOT cannot link without it")
env["PATH"] = str(VSWHERE_DIR) + os.pathsep + env["PATH"]
verify_vendor()
src = prepare_sources()
work = BUILD / "xlate"
shutil.rmtree(work, ignore_errors=True)
work.mkdir(parents=True)
for name in ("Program.cs", "AmpereLowering.cs", "MeshSpirv.cs", "nx2navi_xlate.csproj"):
shutil.copy2(XLATE / name, work / name)
out = BUILD / "publish"
shutil.rmtree(out, ignore_errors=True)
cmd = [
dotnet, "publish", str(work / "nx2navi_xlate.csproj"), "-c", "Release", "-r", "win-x64" if WIN else "linux-x64",
"-p:PublishAot=true", "-p:InvariantGlobalization=true", f"-p:RyuSrc={src}", "-o", str(out),
]
if a.llvm:
env["PATH"] = str(a.llvm.resolve() / "bin") + os.pathsep + env["PATH"]
cmd += ["-p:CppCompilerAndLinker=clang", "-p:LinkerFlavor=lld"]
if a.sysroot:
cmd.append(f"-p:SysRoot={a.sysroot.resolve()}")
if subprocess.run(cmd, env=env).returncode != 0:
sys.exit("dotnet publish failed")
exe = BUILD / EXE
shutil.copy2(out / EXE, exe)
write_identity(exe)
print(f"built {exe} ({exe.stat().st_size / 1e6:.1f} MB) in {time.time() - started:.0f} s")
if __name__ == "__main__":
main()