mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-18 23:59:59 -04:00
Add the shared pipeline list: mesh pipelines players met are built at the shader preparation, before the first start.
The preparation translates every shader and builds each one's stage library, so a pipeline a draw meets for the first time is linked in 2 to 6 ms. A mesh pipeline has no stage libraries: on an install that never drew it, it compiles whole in play, 0.2 to 1.1 s with its draws left out of the picture (18 of them on a first visit of Kokiri Forest). Whole pipelines were built ahead only from the folder's own pipeline-keys.bin. tools/pipeline_list.py merge folds the pipeline-keys.bin and shader-answers.bin of game folders that were played into one list file: every key record once, with the answers record of each shader it names. A record names its shaders by input id, which depends on the game's code and the answers alone, and holds the game's pipeline state; what depends on the GPU is derived when the pipeline is built, so a record made on one machine is good on another with the same game version and runtime. Four folders on this PC give 5,432 records, 317 of them mesh pipelines (147 KB of keys). tools/prepare_shaders.py reads the list (tools/data/pipeline_list.bin, data/pipeline_list.bin in a release folder, or --pipeline-list), adds its shader variants to the seed and writes its records as seed-pipeline-keys.bin. nx2navi.exe --prepare-shaders builds the mesh pipelines among them after the seed is translated and saves the pipeline cache before the stage libraries are built (NX2NAVI_PREPARE_PIPELINES=all builds the vertex pipelines too); --warm-shaders does it again after an update or a new driver, and --prepare-pipelines is that step alone. The pipelines go into the caches and are not kept or added to the folder's own key file, so they cost no start-up time or video memory until a draw meets them. A marker file keeps a driver that crashes on one of them from failing every preparation. The list is derived from game data and stays out of the repository: check_content.py refuses files recorded from a game folder by their first bytes, and make_kit.py, make_launcher.py and make_release.py carry a list given with --pipeline-list. All three now also carry pipeline_list.py, which prepare_shaders.py imports. Checked without the game: tests/pipeline_list_test.py, and prepare_shaders.py --no-run on the game files with a list of 317 mesh pipelines (all 266 shader variants found in this game version, 177 new to the seed). The build step in nx2navi.exe compiles and links and has not been run. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
@@ -91,6 +91,23 @@ def compiled_shader(path: Path) -> bool:
|
||||
return path.suffix.lower() == ".spv" or path.name.lower().endswith("_spv.h")
|
||||
|
||||
|
||||
# What a game folder records in play and what tools/pipeline_list.py folds from it: the game's shader code, the
|
||||
# answers it was translated for, the pipelines it drew. All of it is derived from game data.
|
||||
RECORD_MAGICS = (b"NX2NSPRG", b"NX2NSANS", b"NX2NPKEY", b"NX2NPLST")
|
||||
|
||||
|
||||
def recorded_in_play(path: Path, data: bytes | None) -> bool:
|
||||
"""A shader-programs.bin, shader-answers.bin, pipeline-keys.bin or a pipeline list, by its first bytes. A build kit
|
||||
or a release folder may carry a pipeline list (data/pipeline_list.bin), so make_kit.py and make_release.py, which
|
||||
call problems_for, are not held to this."""
|
||||
if data is None:
|
||||
if not path.is_file():
|
||||
return False
|
||||
with open(path, "rb") as f:
|
||||
data = f.read(8)
|
||||
return data[:8] in RECORD_MAGICS
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
parser.add_argument("--staged", action="store_true", help="check staged files only")
|
||||
@@ -100,9 +117,12 @@ def main() -> int:
|
||||
for path in list_files(args.staged):
|
||||
# --staged checks what the commit takes: a marker staged and then removed from the working copy alone
|
||||
# passed when the file on disk was read
|
||||
problems = problems_for(path, data=staged_bytes(path) if args.staged else None)
|
||||
data = staged_bytes(path) if args.staged else None
|
||||
problems = problems_for(path, data=data)
|
||||
if compiled_shader(path):
|
||||
problems.append("compiled shader: the repository holds the source, tools/build_shaders.py builds this")
|
||||
if recorded_in_play(path, data):
|
||||
problems.append("recorded from a game folder (shaders, answers, pipeline keys or a pipeline list)")
|
||||
for problem in problems:
|
||||
print(f"{path}: {problem}")
|
||||
failed = True
|
||||
|
||||
@@ -1672,7 +1672,8 @@ def step_shaders(out, update=False):
|
||||
if rc:
|
||||
fail('preparing the shaders failed (the lines above say why); the game still runs, compiling shaders as it '
|
||||
'meets them. Run the build again to retry this step')
|
||||
for seed in ('seed-programs.bin', 'seed-answers.bin'): # translated now: the shader cache holds them
|
||||
# translated now: the shader cache holds them. seed-pipeline-keys.bin stays: --warm-shaders builds from it again
|
||||
for seed in ('seed-programs.bin', 'seed-answers.bin'):
|
||||
(out / seed).unlink(missing_ok=True)
|
||||
pack_shader_cache(out)
|
||||
|
||||
|
||||
@@ -28,7 +28,7 @@ from pathlib import Path
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
NAME = 'NX2navi-BuildKit'
|
||||
TOOLS = ('make_game.py', 'nup.py', 'romfs_index.py', 'check_content.py', 'hostmem.py', 'prepare_shaders.py',
|
||||
'shader_census.py', 'shader_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py', 'romfs_check.py',
|
||||
'shader_census.py', 'shader_list.py', 'pipeline_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py', 'romfs_check.py',
|
||||
'build_game.py', 'check_isa.py', 'build_shaders.py', 'make_dlaa_mask.py', 'make_mvobj_fs.py',
|
||||
'pack_shader_cache.py', 'vulkan_pin.py') # make_game.py imports build_game's vma_include and vulkan_include on
|
||||
# Linux for the VMA header and for Vulkan headers newer than the system's (which build_game takes from vulkan_pin), and
|
||||
@@ -60,6 +60,8 @@ def main():
|
||||
help='the Streamline SDK folder instead of downloading the SDK release with --dlssg')
|
||||
ap.add_argument('--prompt-art', type=Path, nargs='+', metavar='FOLDER',
|
||||
help='the button art: folders holding XGamepad, P5Gamepad and Keyboard_Mouse, or those folders')
|
||||
ap.add_argument('--pipeline-list', type=Path, metavar='FILE',
|
||||
help='the shared pipeline list to carry (tools/pipeline_list.py merge writes it; it is derived from game data and stays out of the repository)')
|
||||
ap.add_argument('--out', type=Path, default=ROOT / 'build' / 'kit')
|
||||
ap.add_argument('--zip', action='store_true')
|
||||
a = ap.parse_args()
|
||||
@@ -100,6 +102,12 @@ def main():
|
||||
if a.prompt_art: # make_game.py draws the button prompts from it (its prompts step)
|
||||
n, size = prompt_art.copy_art(a.prompt_art, kit / prompt_art.FOLDER)
|
||||
print(f'button art: {n} pictures, {size / 1e6:.1f} MB')
|
||||
if a.pipeline_list: # prepare_shaders.py reads it there (BUNDLED_PIPELINES)
|
||||
import pipeline_list
|
||||
if pipeline_list.unpack(a.pipeline_list.read_bytes()) is None:
|
||||
sys.exit(f'{a.pipeline_list} is not a pipeline list (tools/pipeline_list.py merge writes one)')
|
||||
(kit / 'tools' / 'data').mkdir(exist_ok=True)
|
||||
shutil.copy2(a.pipeline_list, kit / 'tools' / 'data' / 'pipeline_list.bin')
|
||||
import check_content
|
||||
bad = [f'{p.relative_to(kit)}: {m}' for p in sorted(kit.rglob('*')) if p.is_file()
|
||||
for m in check_content.problems_for(p, allow_large=p.suffix in ('.exe', '.dll'))] # the translator, the DLLs
|
||||
|
||||
@@ -77,7 +77,7 @@ PINS = {
|
||||
'ac87609db3f715404dd41a989c2f1706ea1512f3d86217f938d11c63af8eb505'),
|
||||
}
|
||||
TOOLS = ('make_game.py', 'nup.py', 'romfs_index.py', 'check_content.py', 'hostmem.py', 'prepare_shaders.py',
|
||||
'shader_census.py', 'shader_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py', 'romfs_check.py',
|
||||
'shader_census.py', 'shader_list.py', 'pipeline_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py', 'romfs_check.py',
|
||||
'pack_shader_cache.py')
|
||||
TOP = ('LICENSE', 'LICENSE-ADDITIONS', 'THIRD_PARTY_NOTICES.md', 'nx2navi.ini.example')
|
||||
SKIP_DIRS = {'__pycache__', '.pytest_cache', 'native'}
|
||||
@@ -177,7 +177,7 @@ def build_python(py):
|
||||
|
||||
# ---------------------------------------------------------------- app/<version>/
|
||||
|
||||
def build_app(app, ver, xlate, feed_repo, lock, ffx=None, dlss=None, art=None, dlssg=None):
|
||||
def build_app(app, ver, xlate, feed_repo, lock, ffx=None, dlss=None, art=None, dlssg=None, pipelines=None):
|
||||
# The runtime's own shaders are source in the repository and SPIR-V headers in a release: a player's PC compiles
|
||||
# runtime/ (make_game.py) and has no shader compiler. Build the missing and the stale ones before the copy.
|
||||
build_shaders.build(ROOT / 'runtime')
|
||||
@@ -205,6 +205,10 @@ def build_app(app, ver, xlate, feed_repo, lock, ffx=None, dlss=None, art=None, d
|
||||
if art: # make_game.py draws the button prompts from it (its prompts step)
|
||||
n, size = prompt_art.copy_art(art, app / prompt_art.FOLDER)
|
||||
say(f'button art: {n} pictures, {size / 1e6:.1f} MB')
|
||||
if pipelines: # prepare_shaders.py reads it there (BUNDLED_PIPELINES)
|
||||
(app / 'tools' / 'data').mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(pipelines, app / 'tools' / 'data' / 'pipeline_list.bin')
|
||||
say(f'pipeline list: {pipelines.stat().st_size / 1e6:.1f} MB')
|
||||
(app / 'toolchain.lock.json').write_text(json.dumps(lock, indent=1), encoding='utf-8')
|
||||
(app / 'feed.json').write_text(json.dumps({'type': 'github', 'repo': feed_repo}, indent=1), encoding='utf-8')
|
||||
|
||||
@@ -324,6 +328,8 @@ def main():
|
||||
'nvngx_dlssg.dll files, instead of downloading the SDK release with --dlssg')
|
||||
ap.add_argument('--prompt-art', type=Path, nargs='+', metavar='FOLDER',
|
||||
help='the button art: folders holding XGamepad, P5Gamepad and Keyboard_Mouse, or those folders')
|
||||
ap.add_argument('--pipeline-list', type=Path, metavar='FILE',
|
||||
help='the shared pipeline list to carry (tools/pipeline_list.py merge writes it; it is derived from game data and stays out of the repository)')
|
||||
ap.add_argument('--out', type=Path, default=MAIN_ROOT / 'build' / 'launcher-release')
|
||||
ap.add_argument('--feed', default=DEFAULT_FEED, help='the GitHub repository the launcher looks for releases in')
|
||||
ap.add_argument('--notes', type=Path, help='release notes (markdown) for the manifest and the GitHub release')
|
||||
@@ -356,7 +362,11 @@ def main():
|
||||
dlssg = (a.dlssg, streamline_pin.resolve(a.streamline, DL)) if a.dlssg else None
|
||||
say(f'DLSS Frame Generation: {a.dlssg}, Streamline {streamline_pin.VERSION}: {dlssg[1]}' if dlssg
|
||||
else 'DLSS Frame Generation left out (no --dlssg)')
|
||||
build_app(folder / 'app' / ver, ver, a.xlate, a.feed, lock, a.ffx, dlss, a.prompt_art, dlssg)
|
||||
if a.pipeline_list:
|
||||
import pipeline_list
|
||||
if pipeline_list.unpack(a.pipeline_list.read_bytes()) is None:
|
||||
sys.exit(f'{a.pipeline_list} is not a pipeline list (tools/pipeline_list.py merge writes one)')
|
||||
build_app(folder / 'app' / ver, ver, a.xlate, a.feed, lock, a.ffx, dlss, a.prompt_art, dlssg, a.pipeline_list)
|
||||
(folder / 'app' / 'current.txt').write_text(ver + '\n', encoding='utf-8')
|
||||
build_python(folder / 'python')
|
||||
cc, windres = native_compiler(lock)
|
||||
|
||||
@@ -113,7 +113,7 @@ LLVM_SKIP_SUFFIXES = {'.idl', '.tlb'}
|
||||
# tools/ files the installer steps run (and what they import), copied from the repo.
|
||||
TOOLS_FILES = ['build_game.py', 'build_shaders.py', 'button_icons.py', 'check_content.py', 'check_isa.py', 'extract.py',
|
||||
'hostmem.py', 'make_romfs_patches.py', 'ncz.py', 'pack_shader_cache.py', 'prepare_shaders.py',
|
||||
'romfs_index.py', 'shader_census.py', 'shader_list.py', 'ultrawide_layout.py', 'cheats.py',
|
||||
'romfs_index.py', 'shader_census.py', 'shader_list.py', 'pipeline_list.py', 'ultrawide_layout.py', 'cheats.py',
|
||||
'exefs_patches.py']
|
||||
TOOLS_DIRS = ['vendor/macnativegames', 'data'] # data/: expected hashes
|
||||
RUNTIME_SUFFIXES = {'.h', '.inc', '.ico'} # .ico: the installer window icon
|
||||
@@ -427,7 +427,7 @@ def runtime_lib(out, llvm, pgo):
|
||||
shutil.copy2(ROOT / 'runtime' / name, rt / name)
|
||||
|
||||
|
||||
def build_sources(out, pgo):
|
||||
def build_sources(out, pgo, pipelines=None):
|
||||
copy_tree(ROOT / 'recomp', out / 'recomp', lambda r: not r.endswith(('.py', '.txt')) or '__pycache__' in r)
|
||||
for f in TOOLS_FILES:
|
||||
(out / 'tools').mkdir(exist_ok=True)
|
||||
@@ -437,6 +437,8 @@ def build_sources(out, pgo):
|
||||
(out / 'data').mkdir(exist_ok=True)
|
||||
text = shader_list.history(MAIN_ROOT / 'refs' / 'MacNativeGames', MNG_COMMIT)
|
||||
(out / 'data' / 'shader_list.tsv').write_text(text, encoding='utf-8', newline='\n')
|
||||
if pipelines: # prepare_shaders.py reads it there (BUNDLED_PIPELINES)
|
||||
shutil.copy2(pipelines, out / 'data' / 'pipeline_list.bin')
|
||||
for p in pgo.values():
|
||||
for f in (p, p.with_suffix('.json')):
|
||||
if f.is_file():
|
||||
@@ -586,6 +588,8 @@ def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument('--out', type=Path, required=True, help='release folder (replaced)')
|
||||
ap.add_argument('--zip', type=Path, help='also write the folder as this zip')
|
||||
ap.add_argument('--pipeline-list', type=Path, metavar='FILE',
|
||||
help='the shared pipeline list to carry (tools/pipeline_list.py merge writes it; it is derived from game data and stays out of the repository)')
|
||||
ap.add_argument('--prove', action='store_true', help='only run the compile-and-probe check on an existing folder')
|
||||
ap.add_argument('--cpu', choices=('v2', 'v3'), default='v3',
|
||||
help='--prove: build and run the check for this CPU level (default v3)')
|
||||
@@ -604,7 +608,7 @@ def main():
|
||||
build_deps(out)
|
||||
pgo = profiles()
|
||||
runtime_lib(out, llvm, pgo)
|
||||
build_sources(out, pgo)
|
||||
build_sources(out, pgo, args.pipeline_list)
|
||||
build_launcher(out, llvm)
|
||||
collect_licenses(out)
|
||||
check_folder(out)
|
||||
|
||||
196
tools/pipeline_list.py
Normal file
196
tools/pipeline_list.py
Normal file
@@ -0,0 +1,196 @@
|
||||
"""The shared pipeline list: graphics pipelines players met, for an install to build before play.
|
||||
|
||||
A game folder records every graphics pipeline a run builds in pipeline-keys.bin (runtime/vk_pipeline.c "pipeline
|
||||
keys": the pipeline state with its shaders named by input id) and every shader variant in shader-answers.bin
|
||||
(runtime/gfx_shader.c "answers met in play": stage, code hash, input id, translator answers). The next start of that
|
||||
folder builds its recorded pipelines ahead of play; another install, or the same one after its caches were removed,
|
||||
meets each of them in play first. A mesh pipeline met that way has no stage libraries to be linked from, so its draws
|
||||
are left out of the picture for its whole compile, 0.2 to 1.1 s.
|
||||
|
||||
This tool folds the two files of any number of game folders into one list file. tools/prepare_shaders.py reads the
|
||||
list beside the shared shader list: the shader variants it names go into the seed, so the preparation translates them,
|
||||
and its pipeline records go next to the game as seed-pipeline-keys.bin, which `nx2navi.exe --prepare-shaders` (and
|
||||
`--prepare-pipelines` on a folder that is prepared already) builds once into the pipeline cache and the graphics
|
||||
driver's own.
|
||||
|
||||
What a record depends on. Its shaders are named by input id, a hash of the game's shader code and the answers, which
|
||||
no translator, GPU or driver changes: a record made on one machine names the same shaders on another with the same
|
||||
game version. Its state is the game's (formats, blend, depth and stencil, topology, sample count) and a few choices of
|
||||
the runtime's settings (a culled foliage shadow pass, the depth range); everything that depends on the GPU is derived
|
||||
from it when the pipeline is built. A record whose state this install never reaches costs one pipeline in the caches,
|
||||
and one whose shader this game version lacks is left out by the preparation. The record layout belongs to the runtime
|
||||
that wrote it (KEYS_VERSION and the record size, in the file's header): a list is made again for a runtime that
|
||||
changes either, and the runtime ignores one that does not match.
|
||||
|
||||
The list is derived from game data: it stays out of the repository (tools/check_content.py refuses it) and travels
|
||||
with the installer archive as data/pipeline_list.bin, as the shader list does.
|
||||
|
||||
python tools/pipeline_list.py merge OUT.bin GAME_DIR [GAME_DIR ...] [--mesh-only]
|
||||
python tools/pipeline_list.py info FILE_OR_GAME_DIR [...]
|
||||
|
||||
File: "NX2NPLST", version u32, answers bytes u32, keys bytes u32, then a shader-answers.bin image (header and the
|
||||
records the keys name, input id kept) and a pipeline-keys.bin image (header and records).
|
||||
"""
|
||||
import argparse
|
||||
import struct
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
MAGIC = b'NX2NPLST'
|
||||
VERSION = 1
|
||||
ANS_MAGIC, ANS_REC = b'NX2NSANS', 4 + 32 + 32 + 147 + 1 # gfx_shader.c: stage, code hash, input id, answers, pad
|
||||
KEYS_MAGIC = b'NX2NPKEY'
|
||||
ID = 32
|
||||
STAGE_NAMES = {0: 'vertex', 4: 'fragment', 5: 'compute', 6: 'task', 7: 'mesh'} # runtime/gfx.h GFX_STAGE_*
|
||||
STAGE_MESH = 7
|
||||
|
||||
|
||||
def read_answers(data):
|
||||
"""shader-answers.bin image -> its records (bytes each); [] for another magic or record size."""
|
||||
if len(data) < 16 or data[:8] != ANS_MAGIC:
|
||||
return []
|
||||
_, rec = struct.unpack_from('<II', data, 8)
|
||||
if rec != ANS_REC:
|
||||
return []
|
||||
n = (len(data) - 16) // rec
|
||||
return [data[16 + i * rec:16 + (i + 1) * rec] for i in range(n)]
|
||||
|
||||
|
||||
def read_keys(data):
|
||||
"""pipeline-keys.bin image -> (magic, version, record size, records); records [] for a file too short."""
|
||||
if len(data) < 16:
|
||||
return b'', 0, 0, []
|
||||
magic = data[:8]
|
||||
version, rec = struct.unpack_from('<II', data, 8)
|
||||
if rec < 3 * ID + 8:
|
||||
return magic, version, rec, []
|
||||
n = (len(data) - 16) // rec
|
||||
return magic, version, rec, [data[16 + i * rec:16 + (i + 1) * rec] for i in range(n)]
|
||||
|
||||
|
||||
def key_ids(rec):
|
||||
"""The input ids a key record names: vertex (or mesh), then fragment and task when it has them."""
|
||||
has_fs, has_ts = struct.unpack_from('<II', rec, 3 * ID)
|
||||
ids = [rec[:ID]]
|
||||
if has_fs:
|
||||
ids.append(rec[ID:2 * ID])
|
||||
if has_ts:
|
||||
ids.append(rec[2 * ID:3 * ID])
|
||||
return ids
|
||||
|
||||
|
||||
def read_dir(d):
|
||||
"""A game folder's recorded keys and answers; a missing file gives no records."""
|
||||
d = Path(d)
|
||||
kp, ap = d / 'pipeline-keys.bin', d / 'shader-answers.bin'
|
||||
keys = read_keys(kp.read_bytes()) if kp.is_file() else (b'', 0, 0, [])
|
||||
return keys, read_answers(ap.read_bytes()) if ap.is_file() else []
|
||||
|
||||
|
||||
def merge(dirs, mesh_only=False, log=print):
|
||||
"""One list from the folders' files: each key record once, and the answers record of every shader they name."""
|
||||
header, keys, seen, answers = None, [], set(), {}
|
||||
for d in dirs:
|
||||
(magic, version, rec, recs), ans = read_dir(d)
|
||||
if not recs:
|
||||
log(f'{d}: no pipeline key records')
|
||||
continue
|
||||
if header is None:
|
||||
header = (magic, version, rec)
|
||||
elif header != (magic, version, rec):
|
||||
log(f'{d}: its pipeline-keys.bin is of another layout ({magic!r}, version {version}, {rec} bytes a record); '
|
||||
'left out')
|
||||
continue
|
||||
by_id = {a[36:68]: a for a in ans}
|
||||
added = lacking = 0
|
||||
for r in recs:
|
||||
ids = key_ids(r)
|
||||
if r in seen:
|
||||
continue
|
||||
if any(i not in by_id and i not in answers for i in ids):
|
||||
lacking += 1 # a shader the folder's answers do not name: another install could not translate it
|
||||
continue
|
||||
if mesh_only and by_id.get(ids[0], answers.get(ids[0]))[0:4] != struct.pack('<I', STAGE_MESH):
|
||||
continue
|
||||
seen.add(r)
|
||||
keys.append(r)
|
||||
added += 1
|
||||
for i in ids:
|
||||
answers.setdefault(i, by_id.get(i) or answers[i])
|
||||
log(f'{d}: {len(recs)} key records, {added} new to the list, {lacking} without their shaders\' answers')
|
||||
if header is None:
|
||||
return None
|
||||
return header, keys, list(answers.values())
|
||||
|
||||
|
||||
def pack(header, keys, answers):
|
||||
magic, version, rec = header
|
||||
a = ANS_MAGIC + struct.pack('<II', 1, ANS_REC) + b''.join(answers)
|
||||
k = magic + struct.pack('<II', version, rec) + b''.join(keys)
|
||||
return MAGIC + struct.pack('<III', VERSION, len(a), len(k)) + a + k
|
||||
|
||||
|
||||
def unpack(data):
|
||||
"""A list file -> (answers records, pipeline-keys.bin image); None when it is not one."""
|
||||
if len(data) < 20 or data[:8] != MAGIC:
|
||||
return None
|
||||
version, na, nk = struct.unpack_from('<III', data, 8)
|
||||
if version != VERSION or len(data) != 20 + na + nk:
|
||||
return None
|
||||
return read_answers(data[20:20 + na]), data[20 + na:]
|
||||
|
||||
|
||||
def describe(name, keys_image, answers):
|
||||
magic, version, rec, recs = read_keys(keys_image)
|
||||
stage = {a[36:68]: struct.unpack_from('<I', a)[0] for a in answers}
|
||||
mesh = sum(1 for r in recs if stage.get(r[:ID]) == STAGE_MESH)
|
||||
task = sum(1 for r in recs if struct.unpack_from('<I', r, 3 * ID + 4)[0])
|
||||
named = {i for r in recs for i in key_ids(r)}
|
||||
by = {}
|
||||
for i in named:
|
||||
s = STAGE_NAMES.get(stage.get(i), 'not in the answers')
|
||||
by[s] = by.get(s, 0) + 1
|
||||
print(f'{name}: {len(recs)} pipelines ({mesh} mesh, {task} of them with a task stage), layout {magic.decode("ascii", "replace")} '
|
||||
f'version {version}, {rec} bytes a record; {len(named)} shaders named: '
|
||||
+ ', '.join(f'{v} {k}' for k, v in sorted(by.items())))
|
||||
return len(recs), mesh
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
sub = ap.add_subparsers(dest='cmd', required=True)
|
||||
m = sub.add_parser('merge', help='fold game folders into a list file')
|
||||
m.add_argument('out', type=Path)
|
||||
m.add_argument('dirs', nargs='+', type=Path)
|
||||
m.add_argument('--mesh-only', action='store_true', help='keep the mesh pipelines only')
|
||||
i = sub.add_parser('info', help='what a list file or a game folder holds')
|
||||
i.add_argument('paths', nargs='+', type=Path)
|
||||
a = ap.parse_args()
|
||||
if a.cmd == 'info':
|
||||
for p in a.paths:
|
||||
if p.is_dir():
|
||||
(magic, version, rec, recs), ans = read_dir(p)
|
||||
image = magic + struct.pack('<II', version, rec) + b''.join(recs)
|
||||
describe(str(p), image, ans)
|
||||
else:
|
||||
got = unpack(p.read_bytes())
|
||||
if got is None:
|
||||
print(f'{p}: not a pipeline list')
|
||||
return 1
|
||||
describe(str(p), got[1], got[0])
|
||||
return 0
|
||||
got = merge(a.dirs, a.mesh_only)
|
||||
if got is None:
|
||||
print('no pipeline key records in any folder')
|
||||
return 1
|
||||
header, keys, answers = got
|
||||
data = pack(header, keys, answers)
|
||||
a.out.parent.mkdir(parents=True, exist_ok=True)
|
||||
a.out.write_bytes(data)
|
||||
describe(str(a.out), unpack(data)[1], answers)
|
||||
print(f'{a.out}: {len(data)} bytes')
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -21,6 +21,13 @@ shader-prepared.bin (one record per variant, in priority order) and builds every
|
||||
pipeline once, so the GPU driver's own disk cache holds them (vk_pipeline.c "stage library prebuild"). The seed files
|
||||
can be deleted afterwards.
|
||||
|
||||
The shared pipeline list (tools/pipeline_list.py: graphics pipelines players met, with the shader variants they name)
|
||||
is read beside the shader list when there is one: tools/data/pipeline_list.bin, data/pipeline_list.bin in a release
|
||||
folder, or --pipeline-list. Its variants of shaders this game version has join the seed, and its key records are
|
||||
written as seed-pipeline-keys.bin, which --prepare-shaders builds into the pipeline caches once the seed is translated
|
||||
(vk_pipeline.c "the shared pipeline list"; mesh pipelines by default). That file stays in the build directory:
|
||||
`nx2navi.exe --warm-shaders` and `--prepare-pipelines` build from it again after an update or a new driver.
|
||||
|
||||
A driver whose shader compiler crashes on a variant crashes on it in every run (NVIDIA 615.78.08 on an RTX 5080, issue
|
||||
#30). The translations and shader-prepared.bin are written by then, and the journal the run leaves names the variants
|
||||
the driver was building (vk_pipeline.c skip_init), so only the driver step runs again (`nx2navi.exe --warm-shaders`), up to
|
||||
@@ -47,6 +54,7 @@ ROOT = Path(__file__).resolve().parents[1]
|
||||
while not (ROOT / 'tools' / 'shader_census.py').is_file() and ROOT != ROOT.parent:
|
||||
ROOT = ROOT.parent
|
||||
sys.path.insert(0, str(ROOT / 'tools'))
|
||||
import pipeline_list # noqa: E402
|
||||
import shader_list # noqa: E402
|
||||
from shader_census import inventory # noqa: E402
|
||||
|
||||
@@ -83,6 +91,44 @@ if not BUNDLED_LIST.is_file():
|
||||
BUNDLED_LIST = ROOT / 'data' / 'shader_list.tsv' # release folder layout (tools/make_release.py)
|
||||
|
||||
|
||||
BUNDLED_PIPELINES = ROOT / 'tools' / 'data' / 'pipeline_list.bin' # as BUNDLED_LIST: out of the repository
|
||||
if not BUNDLED_PIPELINES.is_file():
|
||||
BUNDLED_PIPELINES = ROOT / 'data' / 'pipeline_list.bin'
|
||||
SEED_KEYS = 'seed-pipeline-keys.bin'
|
||||
LIST_PRIORITY = 1 # a pipeline list variant's seed priority: with the warm-up rows
|
||||
|
||||
|
||||
def read_pipeline_list(path):
|
||||
"""(answers records, pipeline-keys.bin image) of the pipeline list at path, or of the bundled one; None without
|
||||
a list. A file that is not a list stops the run: a release that names one must not go on without it."""
|
||||
p = Path(path) if path else BUNDLED_PIPELINES
|
||||
if not p.is_file():
|
||||
if path:
|
||||
sys.exit(f'prepare_shaders: no pipeline list at {p}')
|
||||
return None
|
||||
got = pipeline_list.unpack(p.read_bytes())
|
||||
if got is None:
|
||||
sys.exit(f'prepare_shaders: {p} is not a pipeline list (tools/pipeline_list.py merge writes one)')
|
||||
return got
|
||||
|
||||
|
||||
def list_variants(answers_recs, have, seen):
|
||||
"""Seed records for the pipeline list's shader variants: those whose shader (runtime code hash) is in `have` and
|
||||
whose (hash, answers) the seed does not hold yet. The input id is left zero, as for every seed record: the runtime
|
||||
finds it, and finds the id the list's key records name, since it depends on the code and the answers alone."""
|
||||
out, absent = [], 0
|
||||
for a in answers_recs:
|
||||
h, packed = a[4:36], a[68:68 + 147]
|
||||
if h not in have:
|
||||
absent += 1
|
||||
continue
|
||||
if (h, packed) in seen:
|
||||
continue
|
||||
seen.add((h, packed))
|
||||
out.append(a[:36] + bytes(32) + packed + bytes([LIST_PRIORITY]))
|
||||
return out, absent
|
||||
|
||||
|
||||
def read_list(path):
|
||||
if path:
|
||||
return Path(path).read_text(encoding='utf-8')
|
||||
@@ -190,6 +236,8 @@ def main():
|
||||
ap.add_argument('--list', help='a shader_list.tsv instead of tools/data/shader_list.tsv or refs/MacNativeGames')
|
||||
ap.add_argument('--no-run', action='store_true', help='write the seed files only')
|
||||
ap.add_argument('--no-list', action='store_true', help='default variants only, without a shared shader list')
|
||||
ap.add_argument('--pipeline-list', help='a pipeline list instead of tools/data/pipeline_list.bin (tools/pipeline_list.py)')
|
||||
ap.add_argument('--no-pipeline-list', action='store_true', help='no shared pipeline list, also when one is bundled')
|
||||
ap.add_argument('--all-shaders', action='store_true',
|
||||
help='also the default variants SLOW_UNUSED leaves out (an hour more of GPU driver builds)')
|
||||
ap.add_argument('--exe', default='nx2navi.exe' if os.name == 'nt' else 'nx2navi',
|
||||
@@ -235,8 +283,17 @@ def main():
|
||||
rec = struct.pack('<I', stage) + h + bytes(32) + packed + bytes([prio.get(key, 2) | (DEFAULT_FLAG if default else 0)])
|
||||
assert len(rec) == ANS_REC
|
||||
recs.append((prio.get(key, 2), rec))
|
||||
recs.sort(key=lambda r: r[0]) # stable: list order within a priority
|
||||
out = a.build.resolve()
|
||||
plist = None if a.no_pipeline_list else read_pipeline_list(a.pipeline_list)
|
||||
(out / SEED_KEYS).unlink(missing_ok=True) # a list this build no longer has
|
||||
if plist:
|
||||
extra, absent = list_variants(plist[0], {h for h, _ in progs.values()}, seen)
|
||||
recs += [(LIST_PRIORITY, r) for r in extra]
|
||||
(out / SEED_KEYS).write_bytes(plist[1])
|
||||
n_keys = len(pipeline_list.read_keys(plist[1])[3])
|
||||
print(f'pipeline list: {n_keys} pipelines; of the {len(plist[0])} shader variants they name, {len(extra)} join the '
|
||||
f'seed, {absent} are of shaders {a.version} does not have, the others are in it already', flush=True)
|
||||
recs.sort(key=lambda r: r[0]) # stable: list order within a priority
|
||||
with open(out / 'seed-programs.bin', 'wb') as f:
|
||||
f.write(b'NX2NSPRG' + struct.pack('<II', 1, 0))
|
||||
for _, body in progs.values():
|
||||
|
||||
Reference in New Issue
Block a user