Merge the 0.6.1 fixes from the 0.6.1.x test builds.

This brings in the test branch up to the shader reads, not the texture eviction by disuse still in progress there:
- black frames: a released or taken-back texture keeps its content, and a new texture over a live one of another size gets the region both cover;
- the menu font atlas uploaded through the runtime's checked upload batch;
- the game held to 240 frames a second, which stops the quick menu's crash;
- the FSR 3.1 contexts evicted instead of turning FSR off after a few render scales;
- AMD frame generation kept off while NVIDIA's holds the device;
- less device memory held that only costs the system's commit;
- shaders read from the disk cache off the render thread past a small budget;
- the exit code 70 text;
- pull requests #260, #261, #270 and #271.
The tool lists in make_kit.py and make_launcher.py and the setting names in cfg.c take both sides: romfs_check.py and pack_shader_cache.py, FOG_CULL, SCENE_SHARE and GPA_TWIN_MAX_KB.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
chrg6
2026-11-09 14:39:42 -05:00
29 changed files with 637 additions and 135 deletions

0
tools/fetch_refs.sh Normal file → Executable file
View File

View File

@@ -1367,6 +1367,8 @@ def step_shaders(out, update=False):
break
if update and shader_carry(out):
say(' the shaders prepared before this update stay in use; the game translates them again in the background')
if loose_shaders(out) >= PACK_LOOSE_MIN:
pack_shader_cache(out)
return
cmd = [sys.executable, '-u', str(KIT / 'tools' / 'prepare_shaders.py'), str(out), '--romfs', str(out / 'romfs'),
'--exefs', str(out / 'exefs'), '--no-list', '--exe', EXE_NAME]
@@ -1416,6 +1418,42 @@ def step_shaders(out, update=False):
'meets them. Run the build again to retry this step')
for seed in ('seed-programs.bin', 'seed-answers.bin'): # translated now: the shader cache holds them
(out / seed).unlink(missing_ok=True)
pack_shader_cache(out)
# An update that carries the shaders over packs the cache when it has this many loose translations: the game writes
# each translation it makes in play, and those of its background pass after a translator change (about 42,000), as loose
# files, and reading one the first time cost 5.6 ms on Windows against a fraction of a millisecond from the pack.
PACK_LOOSE_MIN = 1000
def loose_shaders(out):
"""The loose translations (<key>.spv) in the game's shader cache."""
try:
with os.scandir(out / 'shader-cache') as it:
return sum(1 for e in it if e.name.endswith('.spv'))
except OSError:
return 0
def pack_shader_cache(out):
"""The prepared shader cache folded into shader-cache/shader-cache.pack (tools/pack_shader_cache.py): the
preparation leaves four files per shader, about 175,000 for the whole game, which the game looks up one by one at
every start (gfx_shader.c gfx_shader_prewarm, the use masks read after it) and which are slow to scan, copy and
back up. The game reads the pack first and the loose files after it, so a cache left unpacked still works: a
failure here is said and the build goes on."""
tool = KIT / 'tools' / 'pack_shader_cache.py'
if not tool.is_file() or not (out / 'shader-cache').is_dir():
return
t0 = time.monotonic()
r = subprocess.run([sys.executable, '-u', str(tool), str(out / 'shader-cache')], stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, text=True, encoding='utf-8', errors='replace')
for line in r.stdout.splitlines():
say(' ' + line)
if r.returncode:
say(f' packing the shader cache failed (code {r.returncode}); the game reads the loose files instead')
else:
say(f' shader cache packed in {time.monotonic() - t0:.0f} s')
def prompt_art_dir():
@@ -1554,8 +1592,10 @@ def step_key(step, identity, out=None):
f'|cpu {cpu_tag()}')
if step == 'compile':
return compile_key
# shaders: the built game and the translator that prepared them
return f'shaders-1|{compile_key}|{sha256_file(KIT / "tools" / "prepare_shaders.py")}|{sha256_file(KIT / "bin" / "nx2navi_xlate.id")}'
# shaders: the built game and the translator that prepared them, the one xlate_source finds (the kit's bin/, or a
# source checkout's build/xlate/)
return (f'shaders-1|{compile_key}|{sha256_file(KIT / "tools" / "prepare_shaders.py")}'
f'|{sha256_file(xlate_source() / "nx2navi_xlate.id")}')
def main():

View File

@@ -27,7 +27,7 @@ ROOT = Path(__file__).resolve().parents[1]
NAME = 'NX2navi-BuildKit'
TOOLS = ('make_game.py', 'nup.py', 'romfs_index.py', 'check_content.py', 'hostmem.py', 'prepare_shaders.py',
'shader_census.py', 'shader_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py', 'romfs_check.py',
'build_game.py', 'check_isa.py', 'build_shaders.py') # make_game.py imports build_game's vma_include on
'build_game.py', 'check_isa.py', 'build_shaders.py', 'pack_shader_cache.py') # make_game.py imports build_game's vma_include on
# Linux for the VMA header, and build_game.py imports check_isa and build_shaders
TOP = ('LICENSE', 'LICENSE-ADDITIONS', 'THIRD_PARTY_NOTICES.md', 'VERSION', 'nx2navi.ini.example')
SKIP_DIRS = {'__pycache__', '.pytest_cache'}

View File

@@ -76,7 +76,8 @@ PINS = {
'ac87609db3f715404dd41a989c2f1706ea1512f3d86217f938d11c63af8eb505'),
}
TOOLS = ('make_game.py', 'nup.py', 'romfs_index.py', 'check_content.py', 'hostmem.py', 'prepare_shaders.py',
'shader_census.py', 'shader_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py', 'romfs_check.py')
'shader_census.py', 'shader_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py', 'romfs_check.py',
'pack_shader_cache.py')
TOP = ('LICENSE', 'LICENSE-ADDITIONS', 'THIRD_PARTY_NOTICES.md', 'nx2navi.ini.example')
SKIP_DIRS = {'__pycache__', '.pytest_cache', 'native'}
SKIP_SUFFIXES = {'.o', '.obj', '.pyc', '.exe', '.dll', '.a', '.lib'}

View File

@@ -19,6 +19,8 @@ import os
import re
import struct
import sys
from collections import deque
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
MAGIC = b'TKSPACK1'
@@ -26,6 +28,11 @@ PACK = 'shader-cache.pack'
KINDS = {'.spv': 1, '.res.json': 2, '.use': 3, '.in': 4}
NAME = re.compile(r'([0-9a-f]{64})(\.spv|\.res\.json|\.use|\.in)')
USE_SIZE = 20 # gfx_shader.c use_store: magic, version, three masks
# Loose files are read on READERS threads, at most WINDOW ahead of the one being written. On Windows the time goes into
# opening each file (about 3 ms the first time, against 0.04 ms once the system has seen it), which threads overlap:
# read one at a time, the 175,000 files of a launcher's cache took minutes.
READERS = 16
WINDOW = 1024
def loose_files(cache):
@@ -74,6 +81,13 @@ def valid(kind, data):
return len(data) == USE_SIZE if kind == 3 else len(data) > 0
def read_loose(path):
try:
return path.read_bytes()
except OSError:
return None
def pack(cache, keep=False):
cache = Path(cache)
out = cache / PACK
@@ -89,15 +103,23 @@ def pack(cache, keep=False):
index, written, skipped, offset = [], [], 0, 16 + 48 * len(keys)
old = open(out, 'rb') if packed else None
try:
with open(tmp, 'wb') as f:
with open(tmp, 'wb') as f, ThreadPoolExecutor(READERS) as pool:
f.write(bytes(offset)) # header and index, written once the offsets are known
for key in keys:
ahead, todo = deque(), iter(keys)
def submit():
key = next(todo, None)
if key is not None:
ahead.append((key, pool.submit(read_loose, loose[key]) if key in loose else None))
for _ in range(WINDOW):
submit()
while ahead:
key, reading = ahead.popleft()
submit()
data = None
if key in loose: # a valid loose file, else the packed copy, else nothing
try:
data = loose[key].read_bytes()
except OSError:
pass
if reading: # a valid loose file, else the packed copy, else nothing
data = reading.result()
if data is not None and valid(key[0], data):
written.append(loose[key])
else: