mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-16 15:59:59 -04:00
Large compute shaders that no cache holds are built on one background thread after the boot prebuild, smallest first, so their first dispatch no longer compiles them on the replay thread. NX2NAVI_BIG_BUILD_MAX_MB stops that builder once the process has grown by that much: on a 31 GB machine one of these shaders took the process up by 12.8 GB and the game stuttered while Windows paged. The builder takes the queue under its lock and copies each entry out before compiling it, so a shader queued meanwhile cannot free the arrays under it. The same change reports what pipeline-cache.bin gives back, and starts over when the file grows past NX2NAVI_PIPELINE_CACHE_MAX_MB. A tensor-core program is now tested before its shader is asked for, in both dispatch paths. It runs the stand-in while NX2NAVI_NATIVE_TENSOR is off, and translating it anyway recorded four AI-upscaler shaders of 0.87 to 4.4 million SPIR-V words that this game never dispatches natively. Process memory went from 29.3 GB to 24 GB and the multi-gigabyte builds during play are gone. The frame-time summary says which part a slow frame went to, and with NX2NAVI_REPLAY_TIME how the replay splits between draws, binds and transfers; NX2NAVI_SVC_SLOW_MS logs the kernel services that block, with the thread they blocked on. prof_report.py reads the map this build writes and adds the image base to a runtime Rip, so a profile resolves to function names instead of question marks. build_xlate.py applies the patches with GIT_CEILING_DIRECTORIES set to the build folder's parent. Inside a checkout git read 0021-prmt-byte-permute.patch's paths from the repository's top, skipped every file of it and still exited 0, so PRMT was missing from locally built translators and every byte permute was a no-op. A Skipped patch line now fails the build. The note in AmpereLowering.cs that said the translator emits nothing for PRMT is corrected.
258 lines
12 KiB
Python
258 lines
12 KiB
Python
"""Build the shader translator: build/xlate/nx2navi_xlate.exe (vendored translator 1.3.2 + patches, NativeAOT, SPIR-V for Vulkan).
|
|
|
|
Copies tools/vendor/ryujinx to build/xlate/ryujinx, applies tools/xlate/patches/*.patch there (tools/vendor stays
|
|
byte-identical), then runs `dotnet publish`. Needs the .NET 10 SDK, the MSVC build
|
|
tools and vswhere.exe (docs/building.md). Run with .venv. On Linux it builds build/xlate/nx2navi_xlate for linux-x64, which
|
|
NativeAOT links with the system's clang (no MSVC); NativeAOT builds for the OS it runs on only.
|
|
|
|
Next to the exe it writes nx2navi_xlate.id: the translator identity, a sha256 over everything the build reads (this script,
|
|
Program.cs, the csproj, the patches, the vendor tree). runtime/gfx_shader.c puts it into every shader cache key, so a
|
|
translator change re-keys the cache without a hand-made KEY_VERSION bump. --id-only writes it for the exe already built
|
|
(the caller vouches that the exe matches the sources).
|
|
|
|
python tools/build_xlate.py [--id-only] [--out DIR]
|
|
|
|
--out builds into DIR instead of build/xlate (a task build dir: nx2navi.exe takes the translator next to it).
|
|
"""
|
|
|
|
import argparse
|
|
import hashlib
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
VENDOR = ROOT / "tools" / "vendor" / "ryujinx"
|
|
MNG_VENDOR = ROOT / "refs" / "MacNativeGames" / "vendor" / "ryujinx"
|
|
XLATE = ROOT / "tools" / "xlate"
|
|
BUILD = ROOT / "build" / "xlate"
|
|
WIN = os.name == "nt"
|
|
EXE = "nx2navi_xlate.exe" if WIN else "nx2navi_xlate"
|
|
VSWHERE_DIR = Path(os.environ.get("ProgramFiles(x86)", r"C:\Program Files (x86)")) / "Microsoft Visual Studio" / "Installer"
|
|
|
|
|
|
def tree_hashes(base: Path) -> dict[str, str]:
|
|
"""Hash every source file; src/*/obj and src/*/bin are build output (an IDE restore writes them, the vendor
|
|
.gitignore ignores them and prepare_sources() drops them), so they are left out."""
|
|
def build_output(rel: Path) -> bool:
|
|
return len(rel.parts) > 2 and rel.parts[0] == "src" and rel.parts[2] in ("obj", "bin")
|
|
|
|
return {
|
|
p.relative_to(base).as_posix(): hashlib.sha256(p.read_bytes()).hexdigest()
|
|
for p in sorted(base.rglob("*"))
|
|
if p.is_file() and not build_output(p.relative_to(base))
|
|
}
|
|
|
|
|
|
def mng_pin() -> str:
|
|
"""The MacNativeGames commit tools/fetch_refs.sh pins."""
|
|
for line in (ROOT / "tools" / "fetch_refs.sh").read_text(encoding="utf-8").splitlines():
|
|
f = line.split()
|
|
if len(f) >= 4 and f[:2] == ["fetch", "MacNativeGames"]: # a fifth word asks for the history
|
|
return f[3]
|
|
sys.exit("tools/fetch_refs.sh pins no MacNativeGames commit")
|
|
|
|
|
|
def pinned_blobs() -> dict[str, str]:
|
|
"""Git blob ids of vendor/ryujinx in the refs/ checkout at the pinned commit. The checkout itself may be newer than the pin or
|
|
carry CRLF line ends (core.autocrlf), so its files are not compared."""
|
|
out = subprocess.run(["git", "-C", str(MNG_VENDOR.parent.parent), "ls-tree", "-r", mng_pin(), "vendor/ryujinx"],
|
|
capture_output=True, text=True)
|
|
if out.returncode != 0:
|
|
sys.exit(f"git ls-tree of the MacNativeGames pin failed: {out.stderr.strip()}")
|
|
blobs = {}
|
|
for line in out.stdout.splitlines():
|
|
meta, path = line.split("\t", 1)
|
|
blobs[path.removeprefix("vendor/ryujinx/")] = meta.split()[2]
|
|
return blobs
|
|
|
|
|
|
def blob_ids(base: Path) -> dict[str, str]:
|
|
"""Git blob ids of the source files tree_hashes() hashes."""
|
|
ids = {}
|
|
for rel in tree_hashes(base):
|
|
data = (base / rel).read_bytes()
|
|
ids[rel] = hashlib.sha1(b"blob %d\0" % len(data) + data).hexdigest()
|
|
return ids
|
|
|
|
|
|
def verify_vendor() -> None:
|
|
"""tools/vendor/ryujinx must be byte-identical to the refs/ checkout's copy at the pinned commit (when present)."""
|
|
if not MNG_VENDOR.is_dir():
|
|
print("refs/MacNativeGames missing: vendor comparison skipped")
|
|
return
|
|
ours, theirs = blob_ids(VENDOR), pinned_blobs()
|
|
if ours != theirs:
|
|
diff = sorted(k for k in ours.keys() | theirs.keys() if ours.get(k) != theirs.get(k))
|
|
sys.exit(f"tools/vendor/ryujinx differs from MNG in {len(diff)} files, first: {diff[:5]}")
|
|
print(f"vendor identical to MNG: {len(ours)} files")
|
|
|
|
|
|
def prepare_sources() -> Path:
|
|
src = BUILD / "ryujinx"
|
|
if BUILD.exists():
|
|
shutil.rmtree(BUILD / "ryujinx", ignore_errors=True)
|
|
shutil.copytree(VENDOR, src)
|
|
for junk in list(src.glob("src/*/obj")) + list(src.glob("src/*/bin")):
|
|
shutil.rmtree(junk)
|
|
patches = sorted((XLATE / "patches").glob("*.patch"))
|
|
if len(patches) != 35:
|
|
sys.exit(f"expected 35 patches, found {len(patches)}")
|
|
# build/xlate is inside this checkout, and git inside a repository reads a "diff --git" patch's paths from the
|
|
# repository's top: those hunks fall outside the directory handed to --directory, git skips every file of the patch
|
|
# and still exits 0. 0021-prmt-byte-permute.patch is the only one written in that form, so it used to be skipped in
|
|
# silence and every byte permute stayed a no-op. GIT_CEILING_DIRECTORIES stops git from walking up to the checkout,
|
|
# so it reads the patch as it would outside a repository, and a "Skipped patch" line fails the build.
|
|
env = dict(os.environ, GIT_CEILING_DIRECTORIES=str(BUILD.parent), LC_ALL="C")
|
|
for patch in patches:
|
|
for extra in (["--check"], []):
|
|
r = subprocess.run(
|
|
["git", "apply", *extra, "--verbose", "--whitespace=nowarn",
|
|
"--directory", str(src.relative_to(BUILD)), str(patch)],
|
|
cwd=BUILD, capture_output=True, text=True, env=env,
|
|
)
|
|
skipped = "Skipped patch" in r.stdout or "Skipped patch" in r.stderr
|
|
if r.returncode != 0 or skipped:
|
|
sys.exit(f"patch does not apply: {patch.name}\n{r.stdout}{r.stderr}")
|
|
print(f"applied {patch.name}")
|
|
return src
|
|
|
|
|
|
# What the identity covers besides the sources: the publish settings in main(). Change it with them.
|
|
BUILD_SETTINGS = "PublishAot=true InvariantGlobalization=true Release"
|
|
|
|
|
|
def strip_code(text: str, xml: bool = False) -> str:
|
|
"""C# (or MSBuild XML) without its comments, trailing spaces and blank lines, string and character literals kept:
|
|
what the compiler sees, so that a comment-only edit keeps the identity (and so every shader cache key)."""
|
|
out, i, n = [], 0, len(text)
|
|
while i < n:
|
|
c = text[i]
|
|
if xml:
|
|
if text.startswith("<!--", i):
|
|
j = text.find("-->", i + 4)
|
|
i = n if j < 0 else j + 3
|
|
continue
|
|
elif text.startswith("//", i):
|
|
j = text.find("\n", i)
|
|
i = n if j < 0 else j
|
|
continue
|
|
elif text.startswith("/*", i):
|
|
j = text.find("*/", i + 2)
|
|
i = n if j < 0 else j + 2
|
|
continue
|
|
elif c in "\"'" or text.startswith('@"', i):
|
|
verbatim = text.startswith('@"', i)
|
|
q = '"' if verbatim else c
|
|
j = i + (2 if verbatim else 1)
|
|
while j < n:
|
|
if verbatim and text.startswith('""', j):
|
|
j += 2
|
|
elif not verbatim and text[j] == "\\":
|
|
j += 2
|
|
elif text[j] == q:
|
|
j += 1
|
|
break
|
|
else:
|
|
j += 1
|
|
out.append(text[i:j])
|
|
i = j
|
|
continue
|
|
out.append(c)
|
|
i += 1
|
|
lines = (line.rstrip() for line in "".join(out).splitlines())
|
|
return "\n".join(line for line in lines if line)
|
|
|
|
|
|
def code_digest(path: Path) -> str:
|
|
"""sha256 of a file, comments left out of C# and MSBuild files (strip_code)."""
|
|
data = path.read_bytes()
|
|
if path.suffix in (".cs", ".csproj", ".props", ".targets"):
|
|
data = strip_code(data.decode("utf-8-sig", errors="replace"), xml=path.suffix != ".cs").encode()
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def identity() -> str:
|
|
"""sha256 over what the translator is built from, comments left out: the patched tree as the build compiles it
|
|
(build/xlate/ryujinx from prepare_sources), tools/xlate's own sources and the publish settings. Without a prepared
|
|
tree, the vendor tree and the patch files as they are."""
|
|
h = hashlib.sha256()
|
|
h.update(f"settings\0{BUILD_SETTINGS}\n".encode())
|
|
for name in ("Program.cs", "AmpereLowering.cs", "MeshSpirv.cs", "nx2navi_xlate.csproj"):
|
|
h.update(f"{name}\0{code_digest(XLATE / name)}\n".encode())
|
|
tree = BUILD / "ryujinx"
|
|
if (tree / "src").is_dir():
|
|
for f in sorted(tree.rglob("*")):
|
|
rel = f.relative_to(tree)
|
|
if f.is_file() and not (len(rel.parts) > 2 and rel.parts[0] == "src" and rel.parts[2] in ("obj", "bin")):
|
|
h.update(f"tree/{rel.as_posix()}\0{code_digest(f)}\n".encode())
|
|
else:
|
|
for f in sorted((XLATE / "patches").glob("*.patch")):
|
|
h.update(f"{f.name}\0{hashlib.sha256(f.read_bytes()).hexdigest()}\n".encode())
|
|
for rel, digest in tree_hashes(VENDOR).items():
|
|
h.update(f"vendor/{rel}\0{digest}\n".encode())
|
|
return h.hexdigest()
|
|
|
|
|
|
def write_identity(exe: Path) -> None:
|
|
ident = identity()
|
|
exe.with_suffix(".id").write_text(ident + "\n", encoding="ascii")
|
|
print(f"translator identity {ident[:16]} -> {exe.with_suffix('.id')}")
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
|
parser.add_argument("--id-only", action="store_true", help="write nx2navi_xlate.id for the exe already built")
|
|
parser.add_argument("--out", type=Path, help="build directory instead of build/xlate")
|
|
parser.add_argument("--llvm", type=Path, help="Linux: an LLVM directory whose bin/clang and lld link the exe")
|
|
parser.add_argument("--sysroot", type=Path, help="Linux: link against this sysroot's glibc instead of the host's "
|
|
"(tools/make_release_linux.py)")
|
|
a = parser.parse_args()
|
|
if a.out:
|
|
global BUILD
|
|
BUILD = a.out.resolve()
|
|
if a.id_only:
|
|
write_identity(BUILD / EXE)
|
|
return
|
|
started = time.time()
|
|
dotnet = shutil.which("dotnet")
|
|
if not dotnet:
|
|
sys.exit("missing: dotnet (.NET 10 SDK)")
|
|
env = dict(os.environ)
|
|
if WIN:
|
|
if not (VSWHERE_DIR / "vswhere.exe").is_file() and not shutil.which("vswhere"):
|
|
sys.exit(f"missing: vswhere.exe (expected in {VSWHERE_DIR}); NativeAOT cannot link without it")
|
|
env["PATH"] = str(VSWHERE_DIR) + os.pathsep + env["PATH"]
|
|
|
|
verify_vendor()
|
|
src = prepare_sources()
|
|
work = BUILD / "xlate"
|
|
shutil.rmtree(work, ignore_errors=True)
|
|
work.mkdir(parents=True)
|
|
for name in ("Program.cs", "AmpereLowering.cs", "MeshSpirv.cs", "nx2navi_xlate.csproj"):
|
|
shutil.copy2(XLATE / name, work / name)
|
|
out = BUILD / "publish"
|
|
shutil.rmtree(out, ignore_errors=True)
|
|
cmd = [
|
|
dotnet, "publish", str(work / "nx2navi_xlate.csproj"), "-c", "Release", "-r", "win-x64" if WIN else "linux-x64",
|
|
"-p:PublishAot=true", "-p:InvariantGlobalization=true", f"-p:RyuSrc={src}", "-o", str(out),
|
|
]
|
|
if a.llvm:
|
|
env["PATH"] = str(a.llvm.resolve() / "bin") + os.pathsep + env["PATH"]
|
|
cmd += ["-p:CppCompilerAndLinker=clang", "-p:LinkerFlavor=lld"]
|
|
if a.sysroot:
|
|
cmd.append(f"-p:SysRoot={a.sysroot.resolve()}")
|
|
if subprocess.run(cmd, env=env).returncode != 0:
|
|
sys.exit("dotnet publish failed")
|
|
exe = BUILD / EXE
|
|
shutil.copy2(out / EXE, exe)
|
|
write_identity(exe)
|
|
print(f"built {exe} ({exe.stat().st_size / 1e6:.1f} MB) in {time.time() - started:.0f} s")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|