mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-18 04:49:57 -04:00
Merge the 0.6.1 fixes from the 0.6.1.x test builds.
This brings in the test branch up to the shader reads, not the texture eviction by disuse still in progress there: - black frames: a released or taken-back texture keeps its content, and a new texture over a live one of another size gets the region both cover; - the menu font atlas uploaded through the runtime's checked upload batch; - the game held to 240 frames a second, which stops the quick menu's crash; - the FSR 3.1 contexts evicted instead of turning FSR off after a few render scales; - AMD frame generation kept off while NVIDIA's holds the device; - less device memory held that only costs the system's commit; - shaders read from the disk cache off the render thread past a small budget; - the exit code 70 text; - pull requests #260, #261, #270 and #271. The tool lists in make_kit.py and make_launcher.py and the setting names in cfg.c take both sides: romfs_check.py and pack_shader_cache.py, FOG_CULL, SCENE_SHARE and GPA_TWIN_MAX_KB. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
0
tools/fetch_refs.sh
Normal file → Executable file
0
tools/fetch_refs.sh
Normal file → Executable file
@@ -1367,6 +1367,8 @@ def step_shaders(out, update=False):
|
||||
break
|
||||
if update and shader_carry(out):
|
||||
say(' the shaders prepared before this update stay in use; the game translates them again in the background')
|
||||
if loose_shaders(out) >= PACK_LOOSE_MIN:
|
||||
pack_shader_cache(out)
|
||||
return
|
||||
cmd = [sys.executable, '-u', str(KIT / 'tools' / 'prepare_shaders.py'), str(out), '--romfs', str(out / 'romfs'),
|
||||
'--exefs', str(out / 'exefs'), '--no-list', '--exe', EXE_NAME]
|
||||
@@ -1416,6 +1418,42 @@ def step_shaders(out, update=False):
|
||||
'meets them. Run the build again to retry this step')
|
||||
for seed in ('seed-programs.bin', 'seed-answers.bin'): # translated now: the shader cache holds them
|
||||
(out / seed).unlink(missing_ok=True)
|
||||
pack_shader_cache(out)
|
||||
|
||||
|
||||
# An update that carries the shaders over packs the cache when it has this many loose translations: the game writes
|
||||
# each translation it makes in play, and those of its background pass after a translator change (about 42,000), as loose
|
||||
# files, and reading one the first time cost 5.6 ms on Windows against a fraction of a millisecond from the pack.
|
||||
PACK_LOOSE_MIN = 1000
|
||||
|
||||
|
||||
def loose_shaders(out):
|
||||
"""The loose translations (<key>.spv) in the game's shader cache."""
|
||||
try:
|
||||
with os.scandir(out / 'shader-cache') as it:
|
||||
return sum(1 for e in it if e.name.endswith('.spv'))
|
||||
except OSError:
|
||||
return 0
|
||||
|
||||
|
||||
def pack_shader_cache(out):
|
||||
"""The prepared shader cache folded into shader-cache/shader-cache.pack (tools/pack_shader_cache.py): the
|
||||
preparation leaves four files per shader, about 175,000 for the whole game, which the game looks up one by one at
|
||||
every start (gfx_shader.c gfx_shader_prewarm, the use masks read after it) and which are slow to scan, copy and
|
||||
back up. The game reads the pack first and the loose files after it, so a cache left unpacked still works: a
|
||||
failure here is said and the build goes on."""
|
||||
tool = KIT / 'tools' / 'pack_shader_cache.py'
|
||||
if not tool.is_file() or not (out / 'shader-cache').is_dir():
|
||||
return
|
||||
t0 = time.monotonic()
|
||||
r = subprocess.run([sys.executable, '-u', str(tool), str(out / 'shader-cache')], stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT, text=True, encoding='utf-8', errors='replace')
|
||||
for line in r.stdout.splitlines():
|
||||
say(' ' + line)
|
||||
if r.returncode:
|
||||
say(f' packing the shader cache failed (code {r.returncode}); the game reads the loose files instead')
|
||||
else:
|
||||
say(f' shader cache packed in {time.monotonic() - t0:.0f} s')
|
||||
|
||||
|
||||
def prompt_art_dir():
|
||||
@@ -1554,8 +1592,10 @@ def step_key(step, identity, out=None):
|
||||
f'|cpu {cpu_tag()}')
|
||||
if step == 'compile':
|
||||
return compile_key
|
||||
# shaders: the built game and the translator that prepared them
|
||||
return f'shaders-1|{compile_key}|{sha256_file(KIT / "tools" / "prepare_shaders.py")}|{sha256_file(KIT / "bin" / "nx2navi_xlate.id")}'
|
||||
# shaders: the built game and the translator that prepared them, the one xlate_source finds (the kit's bin/, or a
|
||||
# source checkout's build/xlate/)
|
||||
return (f'shaders-1|{compile_key}|{sha256_file(KIT / "tools" / "prepare_shaders.py")}'
|
||||
f'|{sha256_file(xlate_source() / "nx2navi_xlate.id")}')
|
||||
|
||||
|
||||
def main():
|
||||
|
||||
@@ -27,7 +27,7 @@ ROOT = Path(__file__).resolve().parents[1]
|
||||
NAME = 'NX2navi-BuildKit'
|
||||
TOOLS = ('make_game.py', 'nup.py', 'romfs_index.py', 'check_content.py', 'hostmem.py', 'prepare_shaders.py',
|
||||
'shader_census.py', 'shader_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py', 'romfs_check.py',
|
||||
'build_game.py', 'check_isa.py', 'build_shaders.py') # make_game.py imports build_game's vma_include on
|
||||
'build_game.py', 'check_isa.py', 'build_shaders.py', 'pack_shader_cache.py') # make_game.py imports build_game's vma_include on
|
||||
# Linux for the VMA header, and build_game.py imports check_isa and build_shaders
|
||||
TOP = ('LICENSE', 'LICENSE-ADDITIONS', 'THIRD_PARTY_NOTICES.md', 'VERSION', 'nx2navi.ini.example')
|
||||
SKIP_DIRS = {'__pycache__', '.pytest_cache'}
|
||||
|
||||
@@ -76,7 +76,8 @@ PINS = {
|
||||
'ac87609db3f715404dd41a989c2f1706ea1512f3d86217f938d11c63af8eb505'),
|
||||
}
|
||||
TOOLS = ('make_game.py', 'nup.py', 'romfs_index.py', 'check_content.py', 'hostmem.py', 'prepare_shaders.py',
|
||||
'shader_census.py', 'shader_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py', 'romfs_check.py')
|
||||
'shader_census.py', 'shader_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py', 'romfs_check.py',
|
||||
'pack_shader_cache.py')
|
||||
TOP = ('LICENSE', 'LICENSE-ADDITIONS', 'THIRD_PARTY_NOTICES.md', 'nx2navi.ini.example')
|
||||
SKIP_DIRS = {'__pycache__', '.pytest_cache', 'native'}
|
||||
SKIP_SUFFIXES = {'.o', '.obj', '.pyc', '.exe', '.dll', '.a', '.lib'}
|
||||
|
||||
@@ -19,6 +19,8 @@ import os
|
||||
import re
|
||||
import struct
|
||||
import sys
|
||||
from collections import deque
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
MAGIC = b'TKSPACK1'
|
||||
@@ -26,6 +28,11 @@ PACK = 'shader-cache.pack'
|
||||
KINDS = {'.spv': 1, '.res.json': 2, '.use': 3, '.in': 4}
|
||||
NAME = re.compile(r'([0-9a-f]{64})(\.spv|\.res\.json|\.use|\.in)')
|
||||
USE_SIZE = 20 # gfx_shader.c use_store: magic, version, three masks
|
||||
# Loose files are read on READERS threads, at most WINDOW ahead of the one being written. On Windows the time goes into
|
||||
# opening each file (about 3 ms the first time, against 0.04 ms once the system has seen it), which threads overlap:
|
||||
# read one at a time, the 175,000 files of a launcher's cache took minutes.
|
||||
READERS = 16
|
||||
WINDOW = 1024
|
||||
|
||||
|
||||
def loose_files(cache):
|
||||
@@ -74,6 +81,13 @@ def valid(kind, data):
|
||||
return len(data) == USE_SIZE if kind == 3 else len(data) > 0
|
||||
|
||||
|
||||
def read_loose(path):
|
||||
try:
|
||||
return path.read_bytes()
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
def pack(cache, keep=False):
|
||||
cache = Path(cache)
|
||||
out = cache / PACK
|
||||
@@ -89,15 +103,23 @@ def pack(cache, keep=False):
|
||||
index, written, skipped, offset = [], [], 0, 16 + 48 * len(keys)
|
||||
old = open(out, 'rb') if packed else None
|
||||
try:
|
||||
with open(tmp, 'wb') as f:
|
||||
with open(tmp, 'wb') as f, ThreadPoolExecutor(READERS) as pool:
|
||||
f.write(bytes(offset)) # header and index, written once the offsets are known
|
||||
for key in keys:
|
||||
ahead, todo = deque(), iter(keys)
|
||||
|
||||
def submit():
|
||||
key = next(todo, None)
|
||||
if key is not None:
|
||||
ahead.append((key, pool.submit(read_loose, loose[key]) if key in loose else None))
|
||||
|
||||
for _ in range(WINDOW):
|
||||
submit()
|
||||
while ahead:
|
||||
key, reading = ahead.popleft()
|
||||
submit()
|
||||
data = None
|
||||
if key in loose: # a valid loose file, else the packed copy, else nothing
|
||||
try:
|
||||
data = loose[key].read_bytes()
|
||||
except OSError:
|
||||
pass
|
||||
if reading: # a valid loose file, else the packed copy, else nothing
|
||||
data = reading.result()
|
||||
if data is not None and valid(key[0], data):
|
||||
written.append(loose[key])
|
||||
else:
|
||||
|
||||
Reference in New Issue
Block a user