mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-18 21:49:57 -04:00
Wait 8 ms once for a pipeline met for the first time, and carry the compute variants in the pipeline list.
A draw that met a new pipeline waited 3 ms for its fast link, in every frame, which is just under what the link takes with the stage libraries in the driver's cache: in a first-visit session on a new install 66 of 87 such draws gave up at the end of the wait and their frame was held back. The wait is now 8 ms for each pipeline, counted from its first miss and shared by all its draws, and 16 ms for a frame, so a pipeline the driver has to compile costs one frame the wait and the frames after it nothing. The pipeline list tool also takes the compute variants a game folder dispatched, which no pipeline record names and which were translated in the middle of play, and reads a list made before as one of its inputs. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
@@ -7,7 +7,11 @@ folder builds its recorded pipelines ahead of play; another install, or the same
|
||||
meets each of them in play first. A mesh pipeline met that way has no stage libraries to be linked from, so its draws
|
||||
are left out of the picture for its whole compile, 0.2 to 1.1 s.
|
||||
|
||||
This tool folds the two files of any number of game folders into one list file. tools/prepare_shaders.py reads the
|
||||
This tool folds the two files of any number of game folders into one list file; a list file made before can stand
|
||||
in for a folder, so a list grows by the folders played since. Beside the shader variants the pipelines name, the list
|
||||
takes the compute variants each folder dispatched: no pipeline record names a compute shader, and a variant with
|
||||
other answers than the shader list's default one was translated in the middle of play, 30 to 40 ms on the frame that
|
||||
first dispatched it. tools/prepare_shaders.py reads the
|
||||
list beside the shared shader list: the shader variants it names go into the seed, so the preparation translates them,
|
||||
and its pipeline records go next to the game as seed-pipeline-keys.bin, which every start of the game counts as
|
||||
recorded pipelines beside its own (runtime/vk_pipeline.c keys_open_seed), so a new install builds them before play,
|
||||
@@ -27,11 +31,11 @@ changes either, and the runtime ignores one that does not match.
|
||||
The list is derived from game data: it stays out of the repository (tools/check_content.py refuses it) and travels
|
||||
with the installer archive as data/pipeline_list.bin, as the shader list does.
|
||||
|
||||
python tools/pipeline_list.py merge OUT.bin GAME_DIR [GAME_DIR ...] [--mesh-only]
|
||||
python tools/pipeline_list.py merge OUT.bin GAME_DIR_OR_LIST [GAME_DIR_OR_LIST ...] [--mesh-only]
|
||||
python tools/pipeline_list.py info FILE_OR_GAME_DIR [...]
|
||||
|
||||
File: "NX2NPLST", version u32, answers bytes u32, keys bytes u32, then a shader-answers.bin image (header and the
|
||||
records the keys name, input id kept) and a pipeline-keys.bin image (header and records).
|
||||
File: "NX2NPLST", version u32, answers bytes u32, keys bytes u32, then a shader-answers.bin image (header, the
|
||||
records the keys name and the compute variants', input id kept) and a pipeline-keys.bin image (header and records).
|
||||
"""
|
||||
import argparse
|
||||
import struct
|
||||
@@ -44,7 +48,7 @@ ANS_MAGIC, ANS_REC = b'NX2NSANS', 4 + 32 + 32 + 147 + 1 # gfx_shader.c: stage,
|
||||
KEYS_MAGIC = b'NX2NPKEY'
|
||||
ID = 32
|
||||
STAGE_NAMES = {0: 'vertex', 4: 'fragment', 5: 'compute', 6: 'task', 7: 'mesh'} # runtime/gfx.h GFX_STAGE_*
|
||||
STAGE_MESH = 7
|
||||
STAGE_COMPUTE, STAGE_MESH = 5, 7
|
||||
|
||||
|
||||
def read_answers(data):
|
||||
@@ -82,15 +86,19 @@ def key_ids(rec):
|
||||
|
||||
|
||||
def read_dir(d):
|
||||
"""A game folder's recorded keys and answers; a missing file gives no records."""
|
||||
"""A game folder's recorded keys and answers; a missing file gives no records. A list file gives what it holds."""
|
||||
d = Path(d)
|
||||
if d.is_file():
|
||||
got = unpack(d.read_bytes())
|
||||
return (read_keys(got[1]), got[0]) if got else ((b'', 0, 0, []), [])
|
||||
kp, ap = d / 'pipeline-keys.bin', d / 'shader-answers.bin'
|
||||
keys = read_keys(kp.read_bytes()) if kp.is_file() else (b'', 0, 0, [])
|
||||
return keys, read_answers(ap.read_bytes()) if ap.is_file() else []
|
||||
|
||||
|
||||
def merge(dirs, mesh_only=False, log=print):
|
||||
"""One list from the folders' files: each key record once, and the answers record of every shader they name."""
|
||||
"""One list from the folders' files (or earlier lists): each key record once, the answers record of every shader
|
||||
they name and, unless mesh_only, of every compute variant a folder dispatched."""
|
||||
header, keys, seen, answers = None, [], set(), {}
|
||||
for d in dirs:
|
||||
(magic, version, rec, recs), ans = read_dir(d)
|
||||
@@ -119,7 +127,13 @@ def merge(dirs, mesh_only=False, log=print):
|
||||
added += 1
|
||||
for i in ids:
|
||||
answers.setdefault(i, by_id.get(i) or answers[i])
|
||||
log(f'{d}: {len(recs)} key records, {added} new to the list, {lacking} without their shaders\' answers')
|
||||
compute = 0
|
||||
for a in ans if not mesh_only else ():
|
||||
if a[0:4] == struct.pack('<I', STAGE_COMPUTE) and a[36:68] not in answers:
|
||||
answers[a[36:68]] = a
|
||||
compute += 1
|
||||
log(f'{d}: {len(recs)} key records, {added} new to the list, {lacking} without their shaders\' answers; '
|
||||
f'{compute} compute variants new to the list')
|
||||
if header is None:
|
||||
return None
|
||||
return header, keys, list(answers.values())
|
||||
@@ -152,16 +166,17 @@ def describe(name, keys_image, answers):
|
||||
for i in named:
|
||||
s = STAGE_NAMES.get(stage.get(i), 'not in the answers')
|
||||
by[s] = by.get(s, 0) + 1
|
||||
compute = sum(1 for a in answers if a[0:4] == struct.pack('<I', STAGE_COMPUTE))
|
||||
print(f'{name}: {len(recs)} pipelines ({mesh} mesh, {task} of them with a task stage), layout {magic.decode("ascii", "replace")} '
|
||||
f'version {version}, {rec} bytes a record; {len(named)} shaders named: '
|
||||
+ ', '.join(f'{v} {k}' for k, v in sorted(by.items())))
|
||||
+ ', '.join(f'{v} {k}' for k, v in sorted(by.items())) + f'; {compute} compute variants')
|
||||
return len(recs), mesh
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
sub = ap.add_subparsers(dest='cmd', required=True)
|
||||
m = sub.add_parser('merge', help='fold game folders into a list file')
|
||||
m = sub.add_parser('merge', help='fold game folders, and lists made before, into a list file')
|
||||
m.add_argument('out', type=Path)
|
||||
m.add_argument('dirs', nargs='+', type=Path)
|
||||
m.add_argument('--mesh-only', action='store_true', help='keep the mesh pipelines only')
|
||||
|
||||
@@ -291,8 +291,9 @@ def main():
|
||||
recs += [(LIST_PRIORITY, r) for r in extra]
|
||||
(out / SEED_KEYS).write_bytes(plist[1])
|
||||
n_keys = len(pipeline_list.read_keys(plist[1])[3])
|
||||
print(f'pipeline list: {n_keys} pipelines; of the {len(plist[0])} shader variants they name, {len(extra)} join the '
|
||||
f'seed, {absent} are of shaders {a.version} does not have, the others are in it already', flush=True)
|
||||
print(f'pipeline list: {n_keys} pipelines; of its {len(plist[0])} shader variants (the ones they name and the '
|
||||
f'compute variants), {len(extra)} join the seed, {absent} are of shaders {a.version} does not have, the '
|
||||
'others are in it already', flush=True)
|
||||
recs.sort(key=lambda r: r[0]) # stable: list order within a priority
|
||||
with open(out / 'seed-programs.bin', 'wb') as f:
|
||||
f.write(b'NX2NSPRG' + struct.pack('<II', 1, 0))
|
||||
|
||||
Reference in New Issue
Block a user