Wait 8 ms once for a pipeline met for the first time, and carry the compute variants in the pipeline list.

A draw that met a new pipeline waited 3 ms for its fast link, in every frame, which is just under what the link takes with the stage libraries in the driver's cache: in a first-visit session on a new install 66 of 87 such draws gave up at the end of the wait and their frame was held back. The wait is now 8 ms for each pipeline, counted from its first miss and shared by all its draws, and 16 ms for a frame, so a pipeline the driver has to compile costs one frame the wait and the frames after it nothing.

The pipeline list tool also takes the compute variants a game folder dispatched, which no pipeline record names and which were translated in the middle of play, and reads a list made before as one of its inputs.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
chrg6
2026-11-12 08:25:42 -05:00
parent d31c3a1036
commit 9f7c8ae810
8 changed files with 89 additions and 22 deletions

View File

@@ -7,7 +7,11 @@ folder builds its recorded pipelines ahead of play; another install, or the same
meets each of them in play first. A mesh pipeline met that way has no stage libraries to be linked from, so its draws
are left out of the picture for its whole compile, 0.2 to 1.1 s.
This tool folds the two files of any number of game folders into one list file. tools/prepare_shaders.py reads the
This tool folds the two files of any number of game folders into one list file; a list file made before can stand
in for a folder, so a list grows by the folders played since. Beside the shader variants the pipelines name, the list
takes the compute variants each folder dispatched: no pipeline record names a compute shader, and a variant with
other answers than the shader list's default one was translated in the middle of play, 30 to 40 ms on the frame that
first dispatched it. tools/prepare_shaders.py reads the
list beside the shared shader list: the shader variants it names go into the seed, so the preparation translates them,
and its pipeline records go next to the game as seed-pipeline-keys.bin, which every start of the game counts as
recorded pipelines beside its own (runtime/vk_pipeline.c keys_open_seed), so a new install builds them before play,
@@ -27,11 +31,11 @@ changes either, and the runtime ignores one that does not match.
The list is derived from game data: it stays out of the repository (tools/check_content.py refuses it) and travels
with the installer archive as data/pipeline_list.bin, as the shader list does.
python tools/pipeline_list.py merge OUT.bin GAME_DIR [GAME_DIR ...] [--mesh-only]
python tools/pipeline_list.py merge OUT.bin GAME_DIR_OR_LIST [GAME_DIR_OR_LIST ...] [--mesh-only]
python tools/pipeline_list.py info FILE_OR_GAME_DIR [...]
File: "NX2NPLST", version u32, answers bytes u32, keys bytes u32, then a shader-answers.bin image (header and the
records the keys name, input id kept) and a pipeline-keys.bin image (header and records).
File: "NX2NPLST", version u32, answers bytes u32, keys bytes u32, then a shader-answers.bin image (header, the
records the keys name and the compute variants', input id kept) and a pipeline-keys.bin image (header and records).
"""
import argparse
import struct
@@ -44,7 +48,7 @@ ANS_MAGIC, ANS_REC = b'NX2NSANS', 4 + 32 + 32 + 147 + 1 # gfx_shader.c: stage,
KEYS_MAGIC = b'NX2NPKEY'
ID = 32
STAGE_NAMES = {0: 'vertex', 4: 'fragment', 5: 'compute', 6: 'task', 7: 'mesh'} # runtime/gfx.h GFX_STAGE_*
STAGE_MESH = 7
STAGE_COMPUTE, STAGE_MESH = 5, 7
def read_answers(data):
@@ -82,15 +86,19 @@ def key_ids(rec):
def read_dir(d):
"""A game folder's recorded keys and answers; a missing file gives no records."""
"""A game folder's recorded keys and answers; a missing file gives no records. A list file gives what it holds."""
d = Path(d)
if d.is_file():
got = unpack(d.read_bytes())
return (read_keys(got[1]), got[0]) if got else ((b'', 0, 0, []), [])
kp, ap = d / 'pipeline-keys.bin', d / 'shader-answers.bin'
keys = read_keys(kp.read_bytes()) if kp.is_file() else (b'', 0, 0, [])
return keys, read_answers(ap.read_bytes()) if ap.is_file() else []
def merge(dirs, mesh_only=False, log=print):
"""One list from the folders' files: each key record once, and the answers record of every shader they name."""
"""One list from the folders' files (or earlier lists): each key record once, the answers record of every shader
they name and, unless mesh_only, of every compute variant a folder dispatched."""
header, keys, seen, answers = None, [], set(), {}
for d in dirs:
(magic, version, rec, recs), ans = read_dir(d)
@@ -119,7 +127,13 @@ def merge(dirs, mesh_only=False, log=print):
added += 1
for i in ids:
answers.setdefault(i, by_id.get(i) or answers[i])
log(f'{d}: {len(recs)} key records, {added} new to the list, {lacking} without their shaders\' answers')
compute = 0
for a in ans if not mesh_only else ():
if a[0:4] == struct.pack('<I', STAGE_COMPUTE) and a[36:68] not in answers:
answers[a[36:68]] = a
compute += 1
log(f'{d}: {len(recs)} key records, {added} new to the list, {lacking} without their shaders\' answers; '
f'{compute} compute variants new to the list')
if header is None:
return None
return header, keys, list(answers.values())
@@ -152,16 +166,17 @@ def describe(name, keys_image, answers):
for i in named:
s = STAGE_NAMES.get(stage.get(i), 'not in the answers')
by[s] = by.get(s, 0) + 1
compute = sum(1 for a in answers if a[0:4] == struct.pack('<I', STAGE_COMPUTE))
print(f'{name}: {len(recs)} pipelines ({mesh} mesh, {task} of them with a task stage), layout {magic.decode("ascii", "replace")} '
f'version {version}, {rec} bytes a record; {len(named)} shaders named: '
+ ', '.join(f'{v} {k}' for k, v in sorted(by.items())))
+ ', '.join(f'{v} {k}' for k, v in sorted(by.items())) + f'; {compute} compute variants')
return len(recs), mesh
def main():
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
sub = ap.add_subparsers(dest='cmd', required=True)
m = sub.add_parser('merge', help='fold game folders into a list file')
m = sub.add_parser('merge', help='fold game folders, and lists made before, into a list file')
m.add_argument('out', type=Path)
m.add_argument('dirs', nargs='+', type=Path)
m.add_argument('--mesh-only', action='store_true', help='keep the mesh pipelines only')

View File

@@ -291,8 +291,9 @@ def main():
recs += [(LIST_PRIORITY, r) for r in extra]
(out / SEED_KEYS).write_bytes(plist[1])
n_keys = len(pipeline_list.read_keys(plist[1])[3])
print(f'pipeline list: {n_keys} pipelines; of the {len(plist[0])} shader variants they name, {len(extra)} join the '
f'seed, {absent} are of shaders {a.version} does not have, the others are in it already', flush=True)
print(f'pipeline list: {n_keys} pipelines; of its {len(plist[0])} shader variants (the ones they name and the '
f'compute variants), {len(extra)} join the seed, {absent} are of shaders {a.version} does not have, the '
'others are in it already', flush=True)
recs.sort(key=lambda r: r[0]) # stable: list order within a priority
with open(out / 'seed-programs.bin', 'wb') as f:
f.write(b'NX2NSPRG' + struct.pack('<II', 1, 0))