mirror of
https://git.triforceproject.com/triforce/source
synced 2026-11-19 16:20:01 -04:00
In-game mods (GameBanana browser, search and installs, with the Mask Salesman), code-patch and cheat mods, the devs' test18 to test45 work (full performance overlay, frame pacing on, steady frame rate, save export and import, bug capture, new options rows, Setup's Mods page and Add to Steam), occlusion queries and new defaults, Auto resolution that scales, replay helper threads and robust buffers off by default, the options in sections.
227 lines
11 KiB
Python
227 lines
11 KiB
Python
"""The SPIR-V Hookshot's translator kept in hookshot_programs.bin, checked offline before a driver sees it again.
|
|
|
|
Each stage goes through the Vulkan SDK's spirv-val, and through the device limits spirv-val does not know: the workgroup's
|
|
invocations and its workgroup memory (the mesh fallback keeps a mesh program's outputs there, on top of the game's own
|
|
shared memory). A stage that fails is written to --out as <key>_<n>_<stage>.spv, its disassembly (.spvasm, spirv-dis) and
|
|
the reasons (.txt); the summary groups the validator's errors by message.
|
|
|
|
python -I scripts/validate_spirv.py "D:\\Games\\Hookshot\\hookshot_programs.bin" (the mesh-as-compute programs)
|
|
python -I scripts/validate_spirv.py <file> --all (every program)
|
|
python -I scripts/validate_spirv.py <file> --key a5da9f84b72d1471 (the programs whose key starts so)
|
|
|
|
The exit status is 1 when a stage failed, 0 when none did.
|
|
"""
|
|
import argparse
|
|
import collections
|
|
import concurrent.futures
|
|
import os
|
|
import re
|
|
import shutil
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
|
|
CACHE_MAGIC = 0x31505348 # (gpu/vk/vk_shader.cpp kDiskMagic)
|
|
FALLBACK = 4 # (a record's kind: a task or mesh program run as compute, issue 640)
|
|
SPV_MAGIC = b"\x03\x02\x23\x07"
|
|
STAGES = {0x1: "vertex", 0x2: "tesscontrol", 0x4: "tesseval", 0x8: "geometry", 0x10: "fragment", 0x20: "compute",
|
|
0x40: "task", 0x80: "mesh"}
|
|
|
|
|
|
def records(path, want):
|
|
"""(key, kind, bytes) of each record want(key, kind) takes, in file order. The layout is disk_init's: a 16-byte head
|
|
(magic, the build of the translator that last opened it, stamp), then records of a u64 key, a u64 size and that many bytes,
|
|
which start with the u32 kind. A record cut short ends the walk, as it does there"""
|
|
with open(path, "rb") as f:
|
|
head = f.read(16)
|
|
if len(head) < 16 or struct.unpack_from("<I", head)[0] != CACHE_MAGIC:
|
|
raise SystemExit("%s: not a Hookshot program cache" % path)
|
|
while True:
|
|
rec = f.read(16)
|
|
if len(rec) < 16:
|
|
return
|
|
key, size = struct.unpack("<QQ", rec)
|
|
at = f.tell()
|
|
kind = f.read(4)
|
|
if len(kind) < 4:
|
|
return
|
|
kind = struct.unpack("<I", kind)[0]
|
|
if want(key, kind):
|
|
f.seek(at)
|
|
data = f.read(size)
|
|
if len(data) < size:
|
|
return
|
|
yield key, kind, data
|
|
f.seek(at + size)
|
|
|
|
|
|
def stages(data):
|
|
"""(nvn stage, vk stage, SPIR-V bytes) of each stage of a record. program_cache_save writes a stage as int nvn_stage,
|
|
VkShaderStageFlagBits vk_stage, then the SPIR-V as a list (a u64 word count, the words): each is found by its magic
|
|
word and kept when the three fields before it agree, so the record's other fields (the answers, Shader::Info) need no
|
|
layout of their own here"""
|
|
out = []
|
|
i = data.find(SPV_MAGIC)
|
|
while i != -1:
|
|
if i >= 16:
|
|
nvn, vk = struct.unpack_from("<iI", data, i - 16)
|
|
n = struct.unpack_from("<Q", data, i - 8)[0]
|
|
if 0 <= nvn <= 7 and vk in STAGES and 5 <= n and i + 4 * n <= len(data):
|
|
out.append((nvn, vk, data[i:i + 4 * n]))
|
|
i = data.find(SPV_MAGIC, i + 4 * n)
|
|
continue
|
|
i = data.find(SPV_MAGIC, i + 1)
|
|
return out
|
|
|
|
|
|
def reflect(spirv):
|
|
"""(local size or None, workgroup memory bytes) of a module. The bytes are its Workgroup variables' types summed with
|
|
no padding: a lower bound of what the driver lays out, so a module over the limit by it is over by any layout"""
|
|
w = struct.unpack("<%dI" % (len(spirv) // 4), spirv[:len(spirv) // 4 * 4])
|
|
size, const, pointee = {}, {}, {}
|
|
local, local_ids, shared = None, None, 0
|
|
i = 5
|
|
while i < len(w):
|
|
n, op = w[i] >> 16, w[i] & 0xFFFF
|
|
if n == 0 or i + n > len(w):
|
|
break
|
|
a = w[i + 1:i + n]
|
|
if op == 16 and len(a) >= 5 and a[1] == 17: # OpExecutionMode LocalSize x y z
|
|
local = tuple(a[2:5])
|
|
elif op == 331 and len(a) >= 5 and a[1] == 38: # OpExecutionModeId LocalSizeId (constants defined later)
|
|
local_ids = a[2:5]
|
|
elif op in (43, 50) and len(a) >= 3: # OpConstant, OpSpecConstant (its default): the low word
|
|
const[a[1]] = a[2]
|
|
elif op == 20: # OpTypeBool
|
|
size[a[0]] = 4
|
|
elif op in (21, 22): # OpTypeInt, OpTypeFloat: width in bits
|
|
size[a[0]] = a[1] // 8
|
|
elif op in (23, 24): # OpTypeVector, OpTypeMatrix: component type, count
|
|
size[a[0]] = size.get(a[1], 0) * a[2]
|
|
elif op == 28: # OpTypeArray: element type, length constant
|
|
size[a[0]] = size.get(a[1], 0) * const.get(a[2], 0)
|
|
elif op == 30: # OpTypeStruct: members
|
|
size[a[0]] = sum(size.get(m, 0) for m in a[1:])
|
|
elif op == 32: # OpTypePointer: id, storage class, pointee
|
|
pointee[a[0]] = a[2]
|
|
elif op == 59 and len(a) >= 3 and a[2] == 4: # OpVariable of storage class Workgroup
|
|
shared += size.get(pointee.get(a[0]), 0)
|
|
i += n
|
|
if local_ids is not None:
|
|
local = tuple(const.get(x, 0) for x in local_ids)
|
|
return local, shared
|
|
|
|
|
|
def find_tool(name):
|
|
"""A Vulkan SDK tool: on PATH, else in %VULKAN_SDK%\\Bin"""
|
|
found = shutil.which(name)
|
|
if not found and os.environ.get("VULKAN_SDK"):
|
|
candidate = os.path.join(os.environ["VULKAN_SDK"], "Bin", name + (".exe" if os.name == "nt" else ""))
|
|
found = candidate if os.path.isfile(candidate) else None
|
|
return found
|
|
|
|
|
|
def check(job, a, val):
|
|
"""The reasons a stage fails ([] when it passes): spirv-val's errors (on stdin), then the device limits"""
|
|
key, nvn, vk, spirv = job
|
|
reasons = []
|
|
r = subprocess.run([val, "--target-env", a.target_env, *a.val_arg, "-"], input=spirv, capture_output=True)
|
|
if r.returncode:
|
|
text = (r.stdout + r.stderr).decode("utf-8", "replace").strip()
|
|
reasons.append(("spirv-val", text or "spirv-val exit status %d" % r.returncode))
|
|
local, shared = reflect(spirv)
|
|
if local and local[0] * local[1] * local[2] > a.max_invocations:
|
|
reasons.append(("limit", "workgroup %dx%dx%d is %d invocations, over maxComputeWorkGroupInvocations %d"
|
|
% (*local, local[0] * local[1] * local[2], a.max_invocations)))
|
|
if shared > a.max_shared:
|
|
reasons.append(("limit", "workgroup memory at least %d bytes, over maxComputeSharedMemorySize %d"
|
|
% (shared, a.max_shared)))
|
|
return reasons, local, shared
|
|
|
|
|
|
def first_error(text):
|
|
"""A validator message with its numbers taken out, so the same fault in many stages counts as one"""
|
|
line = next((x for x in text.splitlines() if x.startswith("error")), text.splitlines()[0] if text else "")
|
|
return re.sub(r"\d+", "N", line).strip()
|
|
|
|
|
|
def dump(out, job, reasons, local, shared, dis):
|
|
key, nvn, vk, spirv = job
|
|
base = os.path.join(out, "%016x_%d_%s" % (key, nvn, STAGES[vk]))
|
|
with open(base + ".spv", "wb") as f:
|
|
f.write(spirv)
|
|
with open(base + ".txt", "w", encoding="utf-8") as f:
|
|
f.write("program %016x, NVN stage %d, %s stage, %d words\n" % (key, nvn, STAGES[vk], len(spirv) // 4))
|
|
f.write("local size %s, workgroup memory at least %d bytes\n\n" % (local, shared))
|
|
for kind, text in reasons:
|
|
f.write("[%s]\n%s\n\n" % (kind, text))
|
|
if dis:
|
|
subprocess.run([dis, base + ".spv", "-o", base + ".spvasm"], capture_output=True)
|
|
return base
|
|
|
|
|
|
def main(argv=None):
|
|
p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
|
p.add_argument("cache", help="hookshot_programs.bin")
|
|
p.add_argument("--all", action="store_true", help="every program, not only the mesh-as-compute ones")
|
|
p.add_argument("--key", action="append", default=[], help="a program key (hex) or its start; may be given again")
|
|
p.add_argument("--out", help="where failing stages are written (default: spirv_failures beside the cache)")
|
|
p.add_argument("--target-env", default="vulkan1.3", help="spirv-val's target (Hookshot asks for Vulkan 1.3)")
|
|
p.add_argument("--val-arg", action="append", default=[], help="an argument passed on to spirv-val")
|
|
p.add_argument("--max-shared", type=int, default=65536, help="maxComputeSharedMemorySize (RDNA1: 65536)")
|
|
p.add_argument("--max-invocations", type=int, default=1024, help="maxComputeWorkGroupInvocations (RDNA1: 1024)")
|
|
p.add_argument("--jobs", type=int, default=os.cpu_count() or 4)
|
|
a = p.parse_args(argv)
|
|
val, dis = find_tool("spirv-val"), find_tool("spirv-dis")
|
|
if not val:
|
|
raise SystemExit("spirv-val not found: install the Vulkan SDK or put its Bin folder on PATH")
|
|
keys = [k.lower().removeprefix("0x") for k in a.key]
|
|
out = a.out or os.path.join(os.path.dirname(os.path.abspath(a.cache)), "spirv_failures")
|
|
|
|
def want(key, kind):
|
|
if keys:
|
|
return any(("%016x" % key).startswith(k) for k in keys)
|
|
return a.all or bool(kind & FALLBACK)
|
|
|
|
programs, failed, groups = 0, [], collections.defaultdict(list)
|
|
with concurrent.futures.ThreadPoolExecutor(a.jobs) as pool:
|
|
pending = collections.deque()
|
|
|
|
def settle(future):
|
|
job, (reasons, local, shared) = future.job, future.result()
|
|
if not reasons:
|
|
return
|
|
os.makedirs(out, exist_ok=True)
|
|
base = dump(out, job, reasons, local, shared, dis)
|
|
failed.append(base)
|
|
for kind, text in reasons:
|
|
groups[first_error(text) if kind == "spirv-val" else re.sub(r"\d+", "N", text)].append(
|
|
"%016x %s" % (job[0], STAGES[job[2]]))
|
|
|
|
for key, kind, data in records(a.cache, want):
|
|
programs += 1
|
|
for nvn, vk, spirv in stages(data):
|
|
job = (key, nvn, vk, spirv)
|
|
future = pool.submit(check, job, a, val)
|
|
future.job = job
|
|
pending.append(future)
|
|
while len(pending) > 4 * a.jobs: # (a few batches in flight: --all reads gigabytes)
|
|
settle(pending.popleft())
|
|
while pending:
|
|
settle(pending.popleft())
|
|
|
|
checked = "programs" if a.all or keys else "mesh-as-compute programs"
|
|
print("%s: %d %s, %d stages failed" % (a.cache, programs, checked, len(failed)))
|
|
for message, where in sorted(groups.items(), key=lambda g: -len(g[1])):
|
|
print(" %4dx %s" % (len(where), message))
|
|
print(" e.g. %s" % ", ".join(where[:3]))
|
|
if failed:
|
|
print("written to %s (.spv, .spvasm, .txt)" % out)
|
|
if not programs:
|
|
print(" (no records matched%s)" % (": is the key right?" if keys else ""))
|
|
return 1 if failed else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|