Check every player's game files during setup, and stand in for damaged textures.

A dump can hold files whose first pages are zeros, with a container checksum that passes because the damage was packed along with them. The game then opens such a file over and over and gives up on whatever needs it: one damaged normal map left the Great Fairy Fountain without walls.

The game build's new last step, tools/romfs_check.py, decompresses every file of the game data once, in worker threads and never while the game runs, and writes romfs-check.json beside the game. A damaged texture gets a stand-in, an undamaged texture of the same kind renamed to it, in an ordinary mod folder the Mods menu can turn off, so the game can load what needs it. Other damaged files are reported, since only a clean dump restores them.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
epicmeme546
2026-11-09 14:16:19 -05:00
parent 7b1bb4edf3
commit 91003c3227
9 changed files with 333 additions and 23 deletions

View File

@@ -24,7 +24,10 @@ Steps, each skipped when a run before finished it for the same inputs (state in
from it into <out>/romfs-patches-* (the Controls menu's Button prompts loads them), the keys as the
bind_ lines of <out>/nx2navi.ini give them. Its key names everything it reads (the RomFS, the art, the
tools, those lines), so it runs again only when one of them changed or a folder is gone, and running
it makes no other step run again. It is last: a failure here leaves the game and its shaders complete.
it makes no other step run again. A failure here leaves the game and its shaders complete.
8. scan tools/romfs_check.py decompresses every file of the game data once, writes <out>/romfs-check.json and
makes stand-ins for damaged textures in <out>/mods. Its key names the game files and the tool, and it
is last: nothing of it runs while the game does.
Without art the step is skipped and the game shows its own prompts, as before
The exefs and romfs steps depend only on the game files; lift and compile depend on this kit's code too. A newer kit
@@ -82,14 +85,15 @@ EXEFS_FILES = ('main', 'main.npdm', 'sdk', 'subsdk0', 'rtld')
MODULES = ('main', 'sdk', 'subsdk0', 'rtld')
ALIGN = 16
STATE_VERSION = 2
STEPS = ('exefs', 'romfs', 'lift', 'compile', 'shaders', 'prompts')
STEPS = ('exefs', 'romfs', 'lift', 'compile', 'shaders', 'prompts', 'scan')
# Steps whose key names every input they read rather than the step before them: a step before them that runs does
# not make them run again, and they make no step after them run again.
OWN_KEY_STEPS = ('prompts',)
OWN_KEY_STEPS = ('prompts', 'scan')
# The Python packages tools/pad_prompts.py imports (pip name: import name), checked before a build with button art.
PROMPT_PACKAGES = {'zstandard': 'zstandard', 'fonttools': 'fontTools', 'pillow': 'PIL'}
# The line tools/pad_prompts.py prints for each overlay folder it wrote.
PROMPTS_WRITTEN = re.compile(r'^.+: \d+ files, the font grew by -?\d+ bytes$')
SCAN_CHECKED = re.compile(r'^checked (\d+) of (\d+)$') # tools/romfs_check.py's progress lines
# Bring-up fixes in the lifted SDK, each anchored on exact generated text that must occur once.
LIFT_FIXES = [
@@ -1502,6 +1506,30 @@ def cpu_tag():
return 'any'
def step_scan(out):
"""Every file of the game data decompressed once, and stand-ins made for the damaged textures: tools/romfs_check.py,
in worker threads here, so nothing of it runs while the game does."""
proc = subprocess.Popen(py_cmd(KIT / 'tools' / 'romfs_check.py', '--game', out),
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding='utf-8',
errors='replace', bufsize=1)
emit('progress', step='scan', fraction=0.0, detail='Checking the game files')
try:
for line in proc.stdout:
line = line.rstrip('\n')
m = SCAN_CHECKED.match(line)
if m:
emit('progress', step='scan', fraction=round(int(m[1]) / max(int(m[2]), 1), 4),
detail=f'Checked {m[1]} of {m[2]} files')
else:
say(f' {line}')
finally:
rc = proc.wait()
proc.stdout.close()
if rc:
fail('checking the game files failed (the lines above say why); the game still runs. Run the build again to '
'retry this step')
def step_key(step, identity, out=None):
"""What a step's output depends on. exefs and romfs: the game files only; lift: the unpacked ExeFS and the
lifter; compile: the lift, the runtime sources, this script and the processor (cpu_tag); prompts: the game files,
@@ -1509,6 +1537,10 @@ def step_key(step, identity, out=None):
is still there."""
if step in ('exefs', 'romfs'):
return f'{step}-1|{identity}'
if step == 'scan':
written = 'written' if (out / 'romfs-check.json').is_file() else 'missing'
tool = KIT / 'tools' / 'romfs_check.py'
return f'scan-1|{identity}|{sha256_file(tool) if tool.is_file() else "no tool"}|{written}'
if step == 'prompts':
tools = tree_digest(KIT / 'tools', ('pad_prompts.py', 'prompt_art.py', 'button_icons.py'))
binds = sha256_bytes(json.dumps(bind_lines(out)).encode())
@@ -1600,7 +1632,8 @@ def main():
runs = {'exefs': lambda: step_exefs(src, out), 'romfs': lambda: step_romfs(src, out, min(a.jobs, 8)),
'lift': lambda: step_lift(out, cache),
'compile': lambda: step_compile(out, cache, msys, a.jobs, romfs_job, a.rebuild and not a.retune),
'shaders': lambda: step_shaders(out, a.rebuild), 'prompts': lambda: step_prompts(out)}
'shaders': lambda: step_shaders(out, a.rebuild), 'prompts': lambda: step_prompts(out),
'scan': lambda: step_scan(out)}
state_lock = threading.Lock() # the RomFS step records itself from its own thread
def save():
@@ -1642,7 +1675,7 @@ def main():
state['steps'].pop(later, None)
say(f'{name}:')
emit('step', step=name, state='start')
if name in ('shaders', 'prompts') and romfs_job: # they read the RomFS image
if name in ('shaders', 'prompts', 'scan') and romfs_job: # they read the RomFS image
romfs_job.join()
if name == 'romfs' and a.until != 'romfs':
# The RomFS image is mostly disk work, the lift and the compile are processor work: they run side

View File

@@ -26,7 +26,7 @@ from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
NAME = 'NX2navi-BuildKit'
TOOLS = ('make_game.py', 'nup.py', 'romfs_index.py', 'check_content.py', 'hostmem.py', 'prepare_shaders.py',
'shader_census.py', 'shader_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py',
'shader_census.py', 'shader_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py', 'romfs_check.py',
'build_game.py', 'check_isa.py', 'build_shaders.py') # make_game.py imports build_game's vma_include on
# Linux for the VMA header, and build_game.py imports check_isa and build_shaders
TOP = ('LICENSE', 'LICENSE-ADDITIONS', 'THIRD_PARTY_NOTICES.md', 'VERSION', 'nx2navi.ini.example')

View File

@@ -76,7 +76,7 @@ PINS = {
'ac87609db3f715404dd41a989c2f1706ea1512f3d86217f938d11c63af8eb505'),
}
TOOLS = ('make_game.py', 'nup.py', 'romfs_index.py', 'check_content.py', 'hostmem.py', 'prepare_shaders.py',
'shader_census.py', 'shader_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py')
'shader_census.py', 'shader_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py', 'romfs_check.py')
TOP = ('LICENSE', 'LICENSE-ADDITIONS', 'THIRD_PARTY_NOTICES.md', 'nx2navi.ini.example')
SKIP_DIRS = {'__pycache__', '.pytest_cache', 'native'}
SKIP_SUFFIXES = {'.o', '.obj', '.pyc', '.exe', '.dll', '.a', '.lib'}

186
tools/romfs_check.py Normal file
View File

@@ -0,0 +1,186 @@
"""Checks every file of a player's packed game data, and stands in for the damaged textures it can.
python tools/romfs_check.py --game FOLDER [--romfs FOLDER] [--mods FOLDER] [--jobs N]
A dump can hold files whose first pages are zeros, with a container checksum that still passes because the damage was
packed along with them. The game then cannot decompress such a file, opens it over and over and gives up on everything
that needs it: one damaged normal map left a whole room without walls. Nothing during play looks for this, so the game
build runs this once for each copy of the game data (tools/make_game.py step_scan).
Every .zs file of <game>/romfs (index.json and romfs.bin, as tools/make_game.py writes them; --romfs names another) is
decompressed in full, with the dictionary its frame names from Pack/ZsDic.pack.zs. A file of any other kind that is
nothing but zeros is reported too. The result goes to <game>/romfs-check.json.
A damaged Tex/<name>.bntx.zs is replaced by a stand-in: an undamaged texture of the same kind, the closest name with the
same length, with the name inside it changed, so the game loads it and what uses it appears. It is another texture, not
the original, and only a clean dump restores the original. The stand-ins are an ordinary mod folder (--mods, default
<game>/mods) named STANDIN_MOD that the Mods menu can turn off; it is removed again when nothing is damaged. They are
built from the player's own files on the player's machine and never enter the repository.
Needs zstandard (tools/requirements-launcher.txt).
"""
import argparse
import concurrent.futures
import json
import os
import re
import shutil
import struct
import sys
import threading
from pathlib import Path
import zstandard
STANDIN_MOD = 'Damaged file stand-ins'
DICTIONARIES = 'Pack/ZsDic.pack.zs'
ZSTD_MAGIC = b'\x28\xb5\x2f\xfd'
PIECE = 1 << 22
TEXTURE = re.compile(r'Tex/([^/]+)\.bntx\.zs')
def sarc_members(blob):
"""Name -> (start, end) of the files of a SARC archive."""
header = struct.unpack_from('<H', blob, 4)[0]
data = struct.unpack_from('<I', blob, 12)[0]
count = struct.unpack_from('<H', blob, header + 6)[0]
names = header + 12 + 16 * count + 8
out = {}
for i in range(count):
_, attr, start, end = struct.unpack_from('<IIII', blob, header + 12 + 16 * i)
name = blob[names + (attr & 0xffff) * 4:].split(b'\0', 1)[0].decode('utf-8', 'replace')
out[name] = (data + start, data + end)
return out
class Image:
"""The packed game data; each thread reads through its own handle."""
def __init__(self, folder):
self.path = Path(folder) / 'romfs.bin'
self.index = json.loads((Path(folder) / 'index.json').read_text(encoding='utf-8'))
self.files = self.index['files']
self.by_path = {f['path']: f for f in self.files}
self.local = threading.local()
self.dicts = {}
entry = self.by_path.get(DICTIONARIES)
if entry:
pack = zstandard.ZstdDecompressor().decompressobj().decompress(self.read(entry))
for start, end in sarc_members(pack).values():
d = zstandard.ZstdCompressionDict(pack[start:end])
self.dicts[d.dict_id()] = d
def read(self, entry):
f = getattr(self.local, 'f', None)
if f is None:
f = self.local.f = open(self.path, 'rb')
f.seek(entry['offset'])
return f.read(entry['size'])
def decompressor(self, dict_id):
"""One per thread and dictionary: a decompressor is not for two threads at once."""
cache = getattr(self.local, 'z', None)
if cache is None:
cache = self.local.z = {}
if dict_id not in cache:
d = self.dicts[dict_id] if dict_id else None
cache[dict_id] = zstandard.ZstdDecompressor(dict_data=d) if d else zstandard.ZstdDecompressor()
return cache[dict_id]
def decompress(self, entry):
data = self.read(entry)
z = self.decompressor(zstandard.get_frame_parameters(data).dict_id)
return z.decompressobj().decompress(data)
def problem(image, entry):
"""Why the file is damaged, or None. A frame naming a dictionary the game does not have cannot be judged: None."""
data = image.read(entry)
if len(data) != entry['size']:
return 'shorter than the index says'
if not entry['path'].endswith('.zs'):
return 'nothing but zeros' if len(data) >= 16 and not any(data) else None
if data[:4] != ZSTD_MAGIC:
return 'no compressed frame at its start'
try:
params = zstandard.get_frame_parameters(data)
if params.dict_id and params.dict_id not in image.dicts:
return None
out = image.decompressor(params.dict_id).decompressobj()
size = 0
for at in range(0, len(data), PIECE):
size += len(out.decompress(data[at:at + PIECE]))
except zstandard.ZstdError as e:
return f'does not decompress ({e})'
if params.content_size and size != params.content_size:
return f'decompresses to {size} bytes, not {params.content_size}'
return None
def donors(image, damaged, name):
"""Undamaged textures whose names have this one's length, most alike first; the same last word is worth most."""
words = name.split('_')
found = []
for path in image.by_path:
m = TEXTURE.fullmatch(path)
if not m or path in damaged or len(m[1]) != len(name):
continue
other = m[1].split('_')
found.append((other[-1] != words[-1], -len(set(other) & set(words)), path))
return [path for _, _, path in sorted(found)]
def stand_in(image, damaged, path):
"""A compressed stand-in for the damaged texture at path, or None when no undamaged texture can be renamed to it."""
name = TEXTURE.fullmatch(path)[1]
for donor in donors(image, damaged, name):
try:
blob = image.decompress(image.by_path[donor])
except zstandard.ZstdError:
continue
old = TEXTURE.fullmatch(donor)[1].encode()
if blob[:4] == b'BNTX' and blob.count(old) == 1:
return zstandard.ZstdCompressor(level=19).compress(blob.replace(old, name.encode())), donor
return None
def main():
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
ap.add_argument('--game', type=Path, required=True, help='the game folder')
ap.add_argument('--romfs', type=Path, help='the folder of index.json and romfs.bin (default <game>/romfs)')
ap.add_argument('--mods', type=Path, help='the mods folder (default <game>/mods)')
ap.add_argument('--jobs', type=int, default=min(os.cpu_count() or 4, 12))
a = ap.parse_args()
image = Image(a.romfs or a.game / 'romfs')
total = len(image.files)
damaged = {}
with concurrent.futures.ThreadPoolExecutor(a.jobs) as pool:
for n, (entry, why) in enumerate(zip(image.files, pool.map(lambda e: problem(image, e), image.files)), 1):
if why:
damaged[entry['path']] = why
print(f'damaged: {entry["path"]}: {why}', flush=True)
if n % 5000 == 0 or n == total:
print(f'checked {n} of {total}', flush=True)
mod = (a.mods or a.game / 'mods') / STANDIN_MOD
shutil.rmtree(mod, ignore_errors=True)
repaired, left = {}, []
for path in sorted(damaged):
made = stand_in(image, damaged, path) if TEXTURE.fullmatch(path) else None
if not made:
left.append(path)
continue
target = mod / 'romfs' / path
target.parent.mkdir(parents=True, exist_ok=True)
target.write_bytes(made[0])
repaired[path] = made[1]
print(f'stand-in: {path} from {made[1]}', flush=True)
report = {'status': 'pass', 'files': total, 'damaged': damaged, 'repaired': repaired, 'not_repaired': left,
'romfs_sha256': image.index.get('romfs_sha256')}
(a.game / 'romfs-check.json').write_text(json.dumps(report, indent=1) + '\n', encoding='utf-8')
print(f'{total} files checked, {len(damaged)} damaged, {len(repaired)} given a stand-in, {len(left)} left as they are',
flush=True)
return 0
if __name__ == '__main__':
sys.exit(main())