mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-17 02:49:57 -04:00
Merge main into the test line: the setup game-file check and the log of game files that keep failing to load.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -110,7 +110,13 @@ A path the Windows ANSI code page cannot hold breaks the MSYS2 tools, so the too
|
||||
`nx2navi.ini`'s `bind_` lines, so Rebuild the Game after rebinding keys draws them again; the step's key names
|
||||
all it reads (the RomFS, the art, the tools, those lines), so it runs again only when one changed or a folder is
|
||||
gone, and runs no other step again. Without art the row is not shown and the game keeps its own prompts.
|
||||
5. The game's Cetra Display is decoded for the window.
|
||||
5. **Check the Game Files**: make_game.py's last step runs `tools/romfs_check.py`, once for each copy of the game data
|
||||
(about 4 minutes, in worker threads, never while the game runs). It decompresses every file, writes
|
||||
`romfs-check.json` beside the game, and for each damaged texture puts a stand-in, another texture of the same kind
|
||||
under the damaged one's name, in `mods/Damaged file stand-ins`, so the game can load what needs it. A damaged file
|
||||
of any other kind is only reported: only a clean dump restores it. In play the game logs a file it keeps opening
|
||||
and cannot load (runtime/romfs.c `open_note`).
|
||||
6. The game's Cetra Display is decoded for the window.
|
||||
|
||||
## Updating
|
||||
|
||||
|
||||
@@ -29,13 +29,15 @@ STEPS = {
|
||||
'play stops for them far less often.'),
|
||||
'prompts': ('Draw the Button Prompts', 'Drawing the Xbox, PlayStation and keyboard and mouse button prompts the '
|
||||
"game's Controls menu offers under Button prompts."),
|
||||
'scan': ('Check the Game Files', 'Decompressing every game file once to find damage, and standing in for damaged '
|
||||
'textures so the world still loads.'),
|
||||
'ready': ('Ready to Play', 'The game is built.'),
|
||||
'download': ('Download {ver}', 'Downloading the new version of NX2navi.'),
|
||||
'verify': ('Verify the Download', 'Checking the download against its checksum.'),
|
||||
'swap': ('Swap In the New Build', 'Linking the new build. It replaces the old one only once it is complete.'),
|
||||
}
|
||||
WEIGHTS = {'tools': 12, 'tools_update': 4, 'check': 1, 'exefs': 2, 'romfs': 14, 'lift': 14, 'compile': 55,
|
||||
'compile_changed': 55, 'shaders': 18, 'prompts': 4, 'ready': 0, 'download': 8, 'verify': 1, 'swap': 6}
|
||||
'compile_changed': 55, 'shaders': 18, 'prompts': 4, 'scan': 5, 'ready': 0, 'download': 8, 'verify': 1, 'swap': 6}
|
||||
|
||||
|
||||
def prompt_rows(ctx):
|
||||
@@ -296,14 +298,14 @@ class InstallJob(Job):
|
||||
|
||||
def __init__(self, ctx, dump, game_dir):
|
||||
super().__init__(ctx, ['tools', 'check', 'exefs', 'romfs', 'lift', 'compile', 'shaders', *prompt_rows(ctx),
|
||||
'ready'])
|
||||
'scan', 'ready'])
|
||||
self.dump, self.game_dir = str(dump), str(game_dir)
|
||||
|
||||
def run(self):
|
||||
self.ensure_tools('tools')
|
||||
self.make_game(build.install_args(self.ctx.layout, self.dump, self.game_dir),
|
||||
{s: s for s in ('check', 'exefs', 'romfs', 'lift', 'compile', 'shaders', 'prompts')})
|
||||
for sid in ('shaders', 'prompts'):
|
||||
{s: s for s in ('check', 'exefs', 'romfs', 'lift', 'compile', 'shaders', 'prompts', 'scan')})
|
||||
for sid in ('shaders', 'prompts', 'scan'):
|
||||
if self.step(sid) and self.step(sid).state == PENDING:
|
||||
self.skip(sid)
|
||||
self.finish('ready', 'Ready')
|
||||
@@ -319,7 +321,7 @@ class RebuildJob(Job):
|
||||
retune = True # Rebuild the Game: the game's code for this processor's level (build.rebuild_args)
|
||||
|
||||
def __init__(self, ctx, game_dir, dump=''):
|
||||
super().__init__(ctx, ['tools_update', 'lift', 'compile_changed', 'swap', 'shaders', *prompt_rows(ctx)])
|
||||
super().__init__(ctx, ['tools_update', 'lift', 'compile_changed', 'swap', 'shaders', *prompt_rows(ctx), 'scan'])
|
||||
self.game_dir, self.dump = str(game_dir), dump
|
||||
|
||||
def run(self):
|
||||
@@ -333,9 +335,9 @@ class RebuildJob(Job):
|
||||
args = build.install_args(layout, self.dump, self.game_dir)
|
||||
else:
|
||||
raise build.BuildError('the game folder has no earlier build: install the game again')
|
||||
self.make_game(args, {'lift': 'lift', 'compile': 'compile_changed', 'shaders': 'shaders', 'prompts': 'prompts'},
|
||||
self.make_game(args, {'lift': 'lift', 'compile': 'compile_changed', 'shaders': 'shaders', 'prompts': 'prompts', 'scan': 'scan'},
|
||||
link_step='swap')
|
||||
for sid in ('lift', 'compile_changed', 'swap', 'shaders', 'prompts'):
|
||||
for sid in ('lift', 'compile_changed', 'swap', 'shaders', 'prompts', 'scan'):
|
||||
if self.step(sid) and self.step(sid).state == PENDING:
|
||||
self.skip(sid)
|
||||
|
||||
@@ -378,7 +380,7 @@ class ContinueUpdateJob(RebuildJob):
|
||||
|
||||
def __init__(self, ctx, game_dir, old, dump=''):
|
||||
Job.__init__(self, ctx, ['download', 'verify', 'tools_update', 'lift', 'compile_changed', 'swap', 'shaders',
|
||||
*prompt_rows(ctx)], ctx.layout.version)
|
||||
*prompt_rows(ctx), 'scan'], ctx.layout.version)
|
||||
self.game_dir, self.dump, self.old = str(game_dir), dump, old
|
||||
self.title = f'Updating to {versions.display(ctx.layout.version)}'
|
||||
for sid, value in (('download', 'Done'), ('verify', 'Checksum OK')):
|
||||
@@ -389,7 +391,7 @@ class ContinueUpdateJob(RebuildJob):
|
||||
RebuildJob.run(self)
|
||||
else:
|
||||
self.ensure_tools('tools_update')
|
||||
for sid in ('lift', 'compile_changed', 'swap', 'shaders', 'prompts'):
|
||||
for sid in ('lift', 'compile_changed', 'swap', 'shaders', 'prompts', 'scan'):
|
||||
self.skip(sid, 'No game yet')
|
||||
updater.prune(self.ctx.layout)
|
||||
|
||||
|
||||
@@ -1124,7 +1124,7 @@ class App:
|
||||
return {'tools': 'Gathering Build Tools', 'tools_update': 'Updating Build Tools', 'check': 'Inspecting Your Game',
|
||||
'exefs': 'Unpacking the Program', 'romfs': 'Packing the Game Data', 'lift': 'Translating the Code',
|
||||
'compile': 'Compiling', 'compile_changed': 'Compiling Changed Files', 'shaders': 'Preparing the Shaders',
|
||||
'prompts': 'Drawing the Button Prompts', 'ready': 'Ready to Play',
|
||||
'prompts': 'Drawing the Button Prompts', 'scan': 'Checking the Game Files', 'ready': 'Ready to Play',
|
||||
'download': 'Downloading', 'verify': 'Verifying', 'swap': 'Swapping In the New Build'}.get(st.id,
|
||||
st.label)
|
||||
|
||||
|
||||
@@ -85,6 +85,8 @@ static uint64_t handles[NH]; // file index + 1; 0 is free
|
||||
static uint32_t handle_ovl[NH]; // overlay slot + 1 of an open handle, 0 = original
|
||||
static PlatFile *handle_file[NH]; // the handle's open overlay file, when handle_ovl is set
|
||||
static Overlay *overlays; // sorted by RomFS index after overlay_init
|
||||
static uint32_t *open_ms; // open_note: per file, the last open and the opens in a row
|
||||
static uint16_t *open_n;
|
||||
static size_t noverlay, overlay_cap;
|
||||
static char **added; // RomFS paths of the added files, sorted; added[k] is index file_count + k
|
||||
static size_t nadded;
|
||||
@@ -1017,6 +1019,7 @@ static void init_once(void *param) {
|
||||
}
|
||||
}
|
||||
overlay_init();
|
||||
open_ms = calloc(file_count, sizeof *open_ms), open_n = calloc(file_count, sizeof *open_n); // open_note
|
||||
rt_log("RomFS ready: %llu files, %zu overlays, %zu added, %.2f s", (unsigned long long)file_count, noverlay,
|
||||
nadded, now_s() - t0);
|
||||
#endif
|
||||
@@ -1124,6 +1127,23 @@ static int movie_skipped(const char *p) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
// A file the game cannot load, damaged or refused for any other reason: its loader opens it again and again and in the
|
||||
// end does without it, and whatever needs it stays out of the world. tools/romfs_check.py finds damaged files before
|
||||
// play; this names the ones that still fail, in the log. One timestamp and one count per file, read and written only
|
||||
// here, where a file is opened (a few thousand times a session), without a lock: a race at worst logs a line twice.
|
||||
#define RETRY_OPENS 24 // opens each less than RETRY_GAP_MS after the last; a file reloaded in play stays far below
|
||||
#define RETRY_GAP_MS 3000
|
||||
|
||||
static void open_note(uint64_t i, const char *path) {
|
||||
if (!open_n || i >= file_count) return;
|
||||
uint32_t now = (uint32_t)(plat_now_ns() / 1000000);
|
||||
if (now - open_ms[i] > RETRY_GAP_MS) open_n[i] = 0;
|
||||
open_ms[i] = now;
|
||||
if (open_n[i] < RETRY_OPENS && ++open_n[i] == RETRY_OPENS)
|
||||
rt_log("RomFS: the game keeps opening %s and cannot load it; what needs it stays out of the world "
|
||||
"(romfs-check.json lists the files found damaged)", path);
|
||||
}
|
||||
|
||||
uint32_t romfs_open(uint64_t *out, const char *path, int mode) {
|
||||
fs_trace("rom", path);
|
||||
if (!out || mode != FS_OPEN_READ) return FS_R_UNSUPPORTED_OPEN;
|
||||
@@ -1137,6 +1157,7 @@ uint32_t romfs_open(uint64_t *out, const char *path, int mode) {
|
||||
}
|
||||
uint64_t i = file_index(p);
|
||||
if (i == UINT64_MAX) return FS_R_PATH_NOT_FOUND;
|
||||
open_note(i, p);
|
||||
uint32_t ov = overlay_slot(i);
|
||||
PlatFile *f = NULL;
|
||||
if (ov) {
|
||||
|
||||
@@ -1512,9 +1512,9 @@ class MakeGameTest(Tmp):
|
||||
sys.path.insert(0, str(self.layout.pydeps)) # the fake zbic
|
||||
self.addCleanup(sys.path.remove, str(self.layout.pydeps))
|
||||
with mock.patch.multiple(self.mg, KIT=kit, PROMPT_PACKAGES={}, EVENTS=False, step_romfs=fake('romfs'),
|
||||
step_lift=fake('lift'), step_compile=fake('compile'), step_shaders=fake('shaders')):
|
||||
step_lift=fake('lift'), step_compile=fake('compile'), step_scan=fake('scan'), step_shaders=fake('shaders')):
|
||||
others, ev, runs = build()
|
||||
self.assertEqual((others, runs), (['compile', 'lift', 'romfs', 'shaders'], 1))
|
||||
self.assertEqual((others, runs), (['compile', 'lift', 'romfs', 'scan', 'shaders'], 1))
|
||||
self.assertEqual(states(ev), ['start', 'done'])
|
||||
fractions = [e['fraction'] for e in ev if e['ev'] == 'progress']
|
||||
self.assertEqual(fractions, [0.0, 0.2, 0.4, 0.6, 0.8, 1.0])
|
||||
@@ -1892,7 +1892,7 @@ class JobTest(Tmp):
|
||||
ctx = self.make_ctx('ok')
|
||||
j = jobs.InstallJob(ctx, self.tmp / 'g.nup', self.tmp / 'game').run_inline()
|
||||
self.assertEqual(j.result, 'ok', j.error)
|
||||
self.assertEqual([s.state for s in j.steps], [jobs.SKIPPED] + [jobs.DONE] * 7) # tools ... shaders, ready
|
||||
self.assertEqual([s.state for s in j.steps], [jobs.SKIPPED] + [jobs.DONE] * 6 + [jobs.SKIPPED, jobs.DONE]) # tools ... shaders, scan (the fake build has no such step), ready
|
||||
self.assertEqual(j.step('check').value, 'Version 1.1.1')
|
||||
self.assertTrue(any('working on lift' in ln for ln in j.lines))
|
||||
self.assertEqual(j.overall(), 1.0)
|
||||
@@ -1905,9 +1905,9 @@ class JobTest(Tmp):
|
||||
(ctx.layout.app / 'prompt-art').mkdir()
|
||||
j = jobs.InstallJob(ctx, self.tmp / 'g.nup', self.tmp / 'game').run_inline()
|
||||
self.assertEqual(j.result, 'ok', j.error)
|
||||
self.assertEqual([s.id for s in j.steps][-3:], ['shaders', 'prompts', 'ready'])
|
||||
self.assertEqual([s.id for s in j.steps][-4:], ['shaders', 'prompts', 'scan', 'ready'])
|
||||
self.assertEqual(j.step('prompts').state, jobs.DONE)
|
||||
self.assertEqual([s.id for s in jobs.RebuildJob(ctx, self.tmp / 'game').steps][-2:], ['shaders', 'prompts'])
|
||||
self.assertEqual([s.id for s in jobs.RebuildJob(ctx, self.tmp / 'game').steps][-3:], ['shaders', 'prompts', 'scan'])
|
||||
|
||||
def test_failed_job(self):
|
||||
self.patch_tools()
|
||||
|
||||
83
tests/romfs_check_test.py
Normal file
83
tests/romfs_check_test.py
Normal file
@@ -0,0 +1,83 @@
|
||||
"""tools/romfs_check.py on a small made-up game: damaged files are found, a damaged texture gets a stand-in with its own
|
||||
name inside, and a clean copy leaves nothing behind. No game data."""
|
||||
import json
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
import zstandard
|
||||
|
||||
TOOL = Path(__file__).resolve().parent.parent / 'tools' / 'romfs_check.py'
|
||||
|
||||
|
||||
def sarc(files):
|
||||
"""A SARC archive: little endian, the file table, then the names, then the data."""
|
||||
names, data, nodes = b'', b'', b''
|
||||
for name, body in files.items():
|
||||
nodes += struct.pack('<IIII', 0, 0x01000000 | len(names) // 4, len(data), len(data) + len(body))
|
||||
names += (name.encode() + b'\0').ljust((len(name) + 4) & ~3, b'\0')
|
||||
data += body
|
||||
sfat = b'SFAT' + struct.pack('<HHI', 12, len(files), 101) + nodes
|
||||
sfnt = b'SFNT' + struct.pack('<HH', 8, 0) + names
|
||||
offset = 0x14 + len(sfat) + len(sfnt)
|
||||
head = b'SARC' + struct.pack('<HHIIHH', 0x14, 0xFEFF, offset + len(data), offset, 0x100, 0)
|
||||
return head + sfat + sfnt + data
|
||||
|
||||
|
||||
def texture(name):
|
||||
return b'BNTX\0\0\0\0' + name.encode() + b'\0' + bytes(range(256)) * 8
|
||||
|
||||
|
||||
class RomfsCheck(unittest.TestCase):
|
||||
def build(self, game, files):
|
||||
z = zstandard.ZstdCompressor(level=3)
|
||||
rom = game / 'romfs'
|
||||
rom.mkdir(parents=True)
|
||||
entries, image = [], b''
|
||||
for path, body in files.items():
|
||||
entries.append({'path': path, 'offset': len(image), 'size': len(body)})
|
||||
image += body
|
||||
(rom / 'romfs.bin').write_bytes(image)
|
||||
(rom / 'index.json').write_text(json.dumps({'status': 'pass', 'romfs_size': len(image), 'files': entries}))
|
||||
return z
|
||||
|
||||
def run_tool(self, game):
|
||||
r = subprocess.run([sys.executable, str(TOOL), '--game', str(game), '--jobs', '2'], capture_output=True, text=True)
|
||||
self.assertEqual(r.returncode, 0, r.stderr)
|
||||
return json.loads((game / 'romfs-check.json').read_text())
|
||||
|
||||
def test_damaged_files_and_stand_in(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
game = Path(tmp)
|
||||
z = zstandard.ZstdCompressor(level=3)
|
||||
good = z.compress(texture('Cave_Floor_B_Nrm'))
|
||||
bad = bytes(16) + z.compress(texture('Cave_Floor_A_Nrm'))
|
||||
self.build(game, {'Tex/Cave_Floor_B_Nrm.bntx.zs': good, 'Tex/Cave_Floor_A_Nrm.bntx.zs': bad,
|
||||
'Model/Cave.bfres.zs': bytes(64), 'Param/Zero.bgyml': bytes(32),
|
||||
'Param/Fine.bgyml': b'YB' + bytes(30)})
|
||||
report = self.run_tool(game)
|
||||
self.assertEqual(sorted(report['damaged']),
|
||||
['Model/Cave.bfres.zs', 'Param/Zero.bgyml', 'Tex/Cave_Floor_A_Nrm.bntx.zs'])
|
||||
self.assertEqual(report['repaired'], {'Tex/Cave_Floor_A_Nrm.bntx.zs': 'Tex/Cave_Floor_B_Nrm.bntx.zs'})
|
||||
self.assertEqual(sorted(report['not_repaired']), ['Model/Cave.bfres.zs', 'Param/Zero.bgyml'])
|
||||
made = game / 'mods' / 'Damaged file stand-ins' / 'romfs' / 'Tex' / 'Cave_Floor_A_Nrm.bntx.zs'
|
||||
blob = zstandard.ZstdDecompressor().decompressobj().decompress(made.read_bytes())
|
||||
self.assertIn(b'Cave_Floor_A_Nrm\0', blob)
|
||||
self.assertNotIn(b'Cave_Floor_B_Nrm', blob)
|
||||
|
||||
def test_clean_game_leaves_no_mod(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
game = Path(tmp)
|
||||
z = zstandard.ZstdCompressor(level=3)
|
||||
self.build(game, {'Tex/A.bntx.zs': z.compress(texture('A'))})
|
||||
stale = game / 'mods' / 'Damaged file stand-ins' / 'romfs'
|
||||
stale.mkdir(parents=True)
|
||||
self.assertEqual(self.run_tool(game)['damaged'], {})
|
||||
self.assertFalse(stale.parent.exists())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -24,7 +24,10 @@ Steps, each skipped when a run before finished it for the same inputs (state in
|
||||
from it into <out>/romfs-patches-* (the Controls menu's Button prompts loads them), the keys as the
|
||||
bind_ lines of <out>/nx2navi.ini give them. Its key names everything it reads (the RomFS, the art, the
|
||||
tools, those lines), so it runs again only when one of them changed or a folder is gone, and running
|
||||
it makes no other step run again. It is last: a failure here leaves the game and its shaders complete.
|
||||
it makes no other step run again. A failure here leaves the game and its shaders complete.
|
||||
8. scan tools/romfs_check.py decompresses every file of the game data once, writes <out>/romfs-check.json and
|
||||
makes stand-ins for damaged textures in <out>/mods. Its key names the game files and the tool, and it
|
||||
is last: nothing of it runs while the game does.
|
||||
Without art the step is skipped and the game shows its own prompts, as before
|
||||
|
||||
The exefs and romfs steps depend only on the game files; lift and compile depend on this kit's code too. A newer kit
|
||||
@@ -82,14 +85,15 @@ EXEFS_FILES = ('main', 'main.npdm', 'sdk', 'subsdk0', 'rtld')
|
||||
MODULES = ('main', 'sdk', 'subsdk0', 'rtld')
|
||||
ALIGN = 16
|
||||
STATE_VERSION = 2
|
||||
STEPS = ('exefs', 'romfs', 'lift', 'compile', 'shaders', 'prompts')
|
||||
STEPS = ('exefs', 'romfs', 'lift', 'compile', 'shaders', 'prompts', 'scan')
|
||||
# Steps whose key names every input they read rather than the step before them: a step before them that runs does
|
||||
# not make them run again, and they make no step after them run again.
|
||||
OWN_KEY_STEPS = ('prompts',)
|
||||
OWN_KEY_STEPS = ('prompts', 'scan')
|
||||
# The Python packages tools/pad_prompts.py imports (pip name: import name), checked before a build with button art.
|
||||
PROMPT_PACKAGES = {'zstandard': 'zstandard', 'fonttools': 'fontTools', 'pillow': 'PIL'}
|
||||
# The line tools/pad_prompts.py prints for each overlay folder it wrote.
|
||||
PROMPTS_WRITTEN = re.compile(r'^.+: \d+ files, the font grew by -?\d+ bytes$')
|
||||
SCAN_CHECKED = re.compile(r'^checked (\d+) of (\d+)$') # tools/romfs_check.py's progress lines
|
||||
|
||||
# Bring-up fixes in the lifted SDK, each anchored on exact generated text that must occur once.
|
||||
LIFT_FIXES = [
|
||||
@@ -1540,6 +1544,30 @@ def cpu_tag():
|
||||
return 'any'
|
||||
|
||||
|
||||
def step_scan(out):
|
||||
"""Every file of the game data decompressed once, and stand-ins made for the damaged textures: tools/romfs_check.py,
|
||||
in worker threads here, so nothing of it runs while the game does."""
|
||||
proc = subprocess.Popen(py_cmd(KIT / 'tools' / 'romfs_check.py', '--game', out),
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding='utf-8',
|
||||
errors='replace', bufsize=1)
|
||||
emit('progress', step='scan', fraction=0.0, detail='Checking the game files')
|
||||
try:
|
||||
for line in proc.stdout:
|
||||
line = line.rstrip('\n')
|
||||
m = SCAN_CHECKED.match(line)
|
||||
if m:
|
||||
emit('progress', step='scan', fraction=round(int(m[1]) / max(int(m[2]), 1), 4),
|
||||
detail=f'Checked {m[1]} of {m[2]} files')
|
||||
else:
|
||||
say(f' {line}')
|
||||
finally:
|
||||
rc = proc.wait()
|
||||
proc.stdout.close()
|
||||
if rc:
|
||||
fail('checking the game files failed (the lines above say why); the game still runs. Run the build again to '
|
||||
'retry this step')
|
||||
|
||||
|
||||
def step_key(step, identity, out=None):
|
||||
"""What a step's output depends on. exefs and romfs: the game files only; lift: the unpacked ExeFS and the
|
||||
lifter; compile: the lift, the runtime sources, this script and the processor (cpu_tag); prompts: the game files,
|
||||
@@ -1547,6 +1575,10 @@ def step_key(step, identity, out=None):
|
||||
is still there."""
|
||||
if step in ('exefs', 'romfs'):
|
||||
return f'{step}-1|{identity}'
|
||||
if step == 'scan':
|
||||
written = 'written' if (out / 'romfs-check.json').is_file() else 'missing'
|
||||
tool = KIT / 'tools' / 'romfs_check.py'
|
||||
return f'scan-1|{identity}|{sha256_file(tool) if tool.is_file() else "no tool"}|{written}'
|
||||
if step == 'prompts':
|
||||
tools = tree_digest(KIT / 'tools', ('pad_prompts.py', 'prompt_art.py', 'button_icons.py'))
|
||||
binds = sha256_bytes(json.dumps(bind_lines(out)).encode())
|
||||
@@ -1640,7 +1672,8 @@ def main():
|
||||
runs = {'exefs': lambda: step_exefs(src, out), 'romfs': lambda: step_romfs(src, out, min(a.jobs, 8)),
|
||||
'lift': lambda: step_lift(out, cache),
|
||||
'compile': lambda: step_compile(out, cache, msys, a.jobs, romfs_job, a.rebuild and not a.retune),
|
||||
'shaders': lambda: step_shaders(out, a.rebuild), 'prompts': lambda: step_prompts(out)}
|
||||
'shaders': lambda: step_shaders(out, a.rebuild), 'prompts': lambda: step_prompts(out),
|
||||
'scan': lambda: step_scan(out)}
|
||||
state_lock = threading.Lock() # the RomFS step records itself from its own thread
|
||||
|
||||
def save():
|
||||
@@ -1682,7 +1715,7 @@ def main():
|
||||
state['steps'].pop(later, None)
|
||||
say(f'{name}:')
|
||||
emit('step', step=name, state='start')
|
||||
if name in ('shaders', 'prompts') and romfs_job: # they read the RomFS image
|
||||
if name in ('shaders', 'prompts', 'scan') and romfs_job: # they read the RomFS image
|
||||
romfs_job.join()
|
||||
if name == 'romfs' and a.until != 'romfs':
|
||||
# The RomFS image is mostly disk work, the lift and the compile are processor work: they run side
|
||||
|
||||
@@ -26,7 +26,7 @@ from pathlib import Path
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
NAME = 'NX2navi-BuildKit'
|
||||
TOOLS = ('make_game.py', 'nup.py', 'romfs_index.py', 'check_content.py', 'hostmem.py', 'prepare_shaders.py',
|
||||
'shader_census.py', 'shader_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py',
|
||||
'shader_census.py', 'shader_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py', 'romfs_check.py',
|
||||
'build_game.py', 'check_isa.py', 'build_shaders.py', 'pack_shader_cache.py') # make_game.py imports build_game's vma_include on
|
||||
# Linux for the VMA header, and build_game.py imports check_isa and build_shaders
|
||||
TOP = ('LICENSE', 'LICENSE-ADDITIONS', 'THIRD_PARTY_NOTICES.md', 'VERSION', 'nx2navi.ini.example')
|
||||
|
||||
@@ -76,7 +76,7 @@ PINS = {
|
||||
'ac87609db3f715404dd41a989c2f1706ea1512f3d86217f938d11c63af8eb505'),
|
||||
}
|
||||
TOOLS = ('make_game.py', 'nup.py', 'romfs_index.py', 'check_content.py', 'hostmem.py', 'prepare_shaders.py',
|
||||
'shader_census.py', 'shader_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py',
|
||||
'shader_census.py', 'shader_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py', 'romfs_check.py',
|
||||
'pack_shader_cache.py')
|
||||
TOP = ('LICENSE', 'LICENSE-ADDITIONS', 'THIRD_PARTY_NOTICES.md', 'nx2navi.ini.example')
|
||||
SKIP_DIRS = {'__pycache__', '.pytest_cache', 'native'}
|
||||
|
||||
186
tools/romfs_check.py
Normal file
186
tools/romfs_check.py
Normal file
@@ -0,0 +1,186 @@
|
||||
"""Checks every file of a player's packed game data, and stands in for the damaged textures it can.
|
||||
|
||||
python tools/romfs_check.py --game FOLDER [--romfs FOLDER] [--mods FOLDER] [--jobs N]
|
||||
|
||||
A dump can hold files whose first pages are zeros, with a container checksum that still passes because the damage was
|
||||
packed along with them. The game then cannot decompress such a file, opens it over and over and gives up on everything
|
||||
that needs it: one damaged normal map left a whole room without walls. Nothing during play looks for this, so the game
|
||||
build runs this once for each copy of the game data (tools/make_game.py step_scan).
|
||||
|
||||
Every .zs file of <game>/romfs (index.json and romfs.bin, as tools/make_game.py writes them; --romfs names another) is
|
||||
decompressed in full, with the dictionary its frame names from Pack/ZsDic.pack.zs. A file of any other kind that is
|
||||
nothing but zeros is reported too. The result goes to <game>/romfs-check.json.
|
||||
|
||||
A damaged Tex/<name>.bntx.zs is replaced by a stand-in: an undamaged texture of the same kind, the closest name with the
|
||||
same length, with the name inside it changed, so the game loads it and what uses it appears. It is another texture, not
|
||||
the original, and only a clean dump restores the original. The stand-ins are an ordinary mod folder (--mods, default
|
||||
<game>/mods) named STANDIN_MOD that the Mods menu can turn off; it is removed again when nothing is damaged. They are
|
||||
built from the player's own files on the player's machine and never enter the repository.
|
||||
|
||||
Needs zstandard (tools/requirements-launcher.txt).
|
||||
"""
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import struct
|
||||
import sys
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
import zstandard
|
||||
|
||||
STANDIN_MOD = 'Damaged file stand-ins'
|
||||
DICTIONARIES = 'Pack/ZsDic.pack.zs'
|
||||
ZSTD_MAGIC = b'\x28\xb5\x2f\xfd'
|
||||
PIECE = 1 << 22
|
||||
TEXTURE = re.compile(r'Tex/([^/]+)\.bntx\.zs')
|
||||
|
||||
|
||||
def sarc_members(blob):
|
||||
"""Name -> (start, end) of the files of a SARC archive."""
|
||||
header = struct.unpack_from('<H', blob, 4)[0]
|
||||
data = struct.unpack_from('<I', blob, 12)[0]
|
||||
count = struct.unpack_from('<H', blob, header + 6)[0]
|
||||
names = header + 12 + 16 * count + 8
|
||||
out = {}
|
||||
for i in range(count):
|
||||
_, attr, start, end = struct.unpack_from('<IIII', blob, header + 12 + 16 * i)
|
||||
name = blob[names + (attr & 0xffff) * 4:].split(b'\0', 1)[0].decode('utf-8', 'replace')
|
||||
out[name] = (data + start, data + end)
|
||||
return out
|
||||
|
||||
|
||||
class Image:
|
||||
"""The packed game data; each thread reads through its own handle."""
|
||||
|
||||
def __init__(self, folder):
|
||||
self.path = Path(folder) / 'romfs.bin'
|
||||
self.index = json.loads((Path(folder) / 'index.json').read_text(encoding='utf-8'))
|
||||
self.files = self.index['files']
|
||||
self.by_path = {f['path']: f for f in self.files}
|
||||
self.local = threading.local()
|
||||
self.dicts = {}
|
||||
entry = self.by_path.get(DICTIONARIES)
|
||||
if entry:
|
||||
pack = zstandard.ZstdDecompressor().decompressobj().decompress(self.read(entry))
|
||||
for start, end in sarc_members(pack).values():
|
||||
d = zstandard.ZstdCompressionDict(pack[start:end])
|
||||
self.dicts[d.dict_id()] = d
|
||||
|
||||
def read(self, entry):
|
||||
f = getattr(self.local, 'f', None)
|
||||
if f is None:
|
||||
f = self.local.f = open(self.path, 'rb')
|
||||
f.seek(entry['offset'])
|
||||
return f.read(entry['size'])
|
||||
|
||||
def decompressor(self, dict_id):
|
||||
"""One per thread and dictionary: a decompressor is not for two threads at once."""
|
||||
cache = getattr(self.local, 'z', None)
|
||||
if cache is None:
|
||||
cache = self.local.z = {}
|
||||
if dict_id not in cache:
|
||||
d = self.dicts[dict_id] if dict_id else None
|
||||
cache[dict_id] = zstandard.ZstdDecompressor(dict_data=d) if d else zstandard.ZstdDecompressor()
|
||||
return cache[dict_id]
|
||||
|
||||
def decompress(self, entry):
|
||||
data = self.read(entry)
|
||||
z = self.decompressor(zstandard.get_frame_parameters(data).dict_id)
|
||||
return z.decompressobj().decompress(data)
|
||||
|
||||
|
||||
def problem(image, entry):
|
||||
"""Why the file is damaged, or None. A frame naming a dictionary the game does not have cannot be judged: None."""
|
||||
data = image.read(entry)
|
||||
if len(data) != entry['size']:
|
||||
return 'shorter than the index says'
|
||||
if not entry['path'].endswith('.zs'):
|
||||
return 'nothing but zeros' if len(data) >= 16 and not any(data) else None
|
||||
if data[:4] != ZSTD_MAGIC:
|
||||
return 'no compressed frame at its start'
|
||||
try:
|
||||
params = zstandard.get_frame_parameters(data)
|
||||
if params.dict_id and params.dict_id not in image.dicts:
|
||||
return None
|
||||
out = image.decompressor(params.dict_id).decompressobj()
|
||||
size = 0
|
||||
for at in range(0, len(data), PIECE):
|
||||
size += len(out.decompress(data[at:at + PIECE]))
|
||||
except zstandard.ZstdError as e:
|
||||
return f'does not decompress ({e})'
|
||||
if params.content_size and size != params.content_size:
|
||||
return f'decompresses to {size} bytes, not {params.content_size}'
|
||||
return None
|
||||
|
||||
|
||||
def donors(image, damaged, name):
|
||||
"""Undamaged textures whose names have this one's length, most alike first; the same last word is worth most."""
|
||||
words = name.split('_')
|
||||
found = []
|
||||
for path in image.by_path:
|
||||
m = TEXTURE.fullmatch(path)
|
||||
if not m or path in damaged or len(m[1]) != len(name):
|
||||
continue
|
||||
other = m[1].split('_')
|
||||
found.append((other[-1] != words[-1], -len(set(other) & set(words)), path))
|
||||
return [path for _, _, path in sorted(found)]
|
||||
|
||||
|
||||
def stand_in(image, damaged, path):
|
||||
"""A compressed stand-in for the damaged texture at path, or None when no undamaged texture can be renamed to it."""
|
||||
name = TEXTURE.fullmatch(path)[1]
|
||||
for donor in donors(image, damaged, name):
|
||||
try:
|
||||
blob = image.decompress(image.by_path[donor])
|
||||
except zstandard.ZstdError:
|
||||
continue
|
||||
old = TEXTURE.fullmatch(donor)[1].encode()
|
||||
if blob[:4] == b'BNTX' and blob.count(old) == 1:
|
||||
return zstandard.ZstdCompressor(level=19).compress(blob.replace(old, name.encode())), donor
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument('--game', type=Path, required=True, help='the game folder')
|
||||
ap.add_argument('--romfs', type=Path, help='the folder of index.json and romfs.bin (default <game>/romfs)')
|
||||
ap.add_argument('--mods', type=Path, help='the mods folder (default <game>/mods)')
|
||||
ap.add_argument('--jobs', type=int, default=min(os.cpu_count() or 4, 12))
|
||||
a = ap.parse_args()
|
||||
image = Image(a.romfs or a.game / 'romfs')
|
||||
total = len(image.files)
|
||||
damaged = {}
|
||||
with concurrent.futures.ThreadPoolExecutor(a.jobs) as pool:
|
||||
for n, (entry, why) in enumerate(zip(image.files, pool.map(lambda e: problem(image, e), image.files)), 1):
|
||||
if why:
|
||||
damaged[entry['path']] = why
|
||||
print(f'damaged: {entry["path"]}: {why}', flush=True)
|
||||
if n % 5000 == 0 or n == total:
|
||||
print(f'checked {n} of {total}', flush=True)
|
||||
mod = (a.mods or a.game / 'mods') / STANDIN_MOD
|
||||
shutil.rmtree(mod, ignore_errors=True)
|
||||
repaired, left = {}, []
|
||||
for path in sorted(damaged):
|
||||
made = stand_in(image, damaged, path) if TEXTURE.fullmatch(path) else None
|
||||
if not made:
|
||||
left.append(path)
|
||||
continue
|
||||
target = mod / 'romfs' / path
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_bytes(made[0])
|
||||
repaired[path] = made[1]
|
||||
print(f'stand-in: {path} from {made[1]}', flush=True)
|
||||
report = {'status': 'pass', 'files': total, 'damaged': damaged, 'repaired': repaired, 'not_repaired': left,
|
||||
'romfs_sha256': image.index.get('romfs_sha256')}
|
||||
(a.game / 'romfs-check.json').write_text(json.dumps(report, indent=1) + '\n', encoding='utf-8')
|
||||
print(f'{total} files checked, {len(damaged)} damaged, {len(repaired)} given a stand-in, {len(left)} left as they are',
|
||||
flush=True)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user