Unpack the texture archive in pieces when drawing the button prompts.

pad_prompts.py gave each packed game file to the decompressor in one
call, which returns one bytes object grown by copying. On made-up data
of the size of the interface texture archive, 69 MB packed and 244 MB
unpacked, that takes 21 s on a fast PC; players' builds waited 44 to 52 s
before the step's first folder, most of its minute. In 64 KiB pieces
joined once the same bytes take 0.3 s.

The prompts step's key held the digest of the three scripts' bytes, so
this edit would have drawn every install's prompts again, as two of the
last four releases did. It now holds a constant changed by hand, whose
value is the scripts' digest in 0.7.0 and 0.7.1, and a test fails when
the scripts' logic changes, to ask whether the constant must change.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
chrg6
2026-11-10 15:59:45 -05:00
parent e678e1f38e
commit 62a02da381
4 changed files with 150 additions and 9 deletions

View File

@@ -112,6 +112,12 @@ SCAN_CHECKED = re.compile(r'^checked (\d+) of (\d+)$') # tools/romfs_check.py'
# check, so the key of every install that passed it stays what it is. tests/launcher_test.py
# test_the_file_checks_logic_is_the_recorded_one fails when the script's logic changes, to ask this question.
SCAN_TOOL = '55ea0c150aaf9dac93a2e6fd3b4ca11e1c572b5eee8b6bd74960535964384634'
# The same for the tools that draw the button prompts (tools/pad_prompts.py, prompt_art.py, button_icons.py) in the
# prompts step's key, which held the digest of the three scripts' bytes: two of four releases changed a script, and
# each time every install drew its prompts again, a minute. Changed by hand when a change to the scripts can change
# what they write. Its value is their digest in 0.7.0 and 0.7.1, so the key of an install made by those stays what it
# is. tests/launcher_test.py test_the_prompt_tools_logic_is_the_recorded_one asks the question when the logic changes.
PROMPT_TOOLS = 'ad59f2c4533f6ea2630b916ef5f3695f72daffe9b2b7663eea4168a4f08e4249'
# Bring-up fixes in the lifted SDK, each anchored on exact generated text that must occur once.
LIFT_FIXES = [
@@ -2110,8 +2116,8 @@ def step_scan(out):
def step_key(step, identity, out=None):
"""What a step's output depends on. exefs and romfs: the game files only; lift: the unpacked ExeFS and the
lifter; compile: the lift, the runtime sources, this script and the processor (cpu_tag); prompts: the game files,
the kit's button art, the tools that draw it, the bind_ lines of nx2navi.ini, and whether every folder it writes
is still there; scan: the game files, the check's version (SCAN_TOOL) and whether its report is still there."""
the kit's button art, the version of the tools that draw it (PROMPT_TOOLS), the bind_ lines of nx2navi.ini, and
whether every folder it writes is still there; scan: the game files, the check's version (SCAN_TOOL) and whether its report is still there."""
if step in ('exefs', 'romfs'):
return f'{step}-1|{identity}'
if step == 'scan':
@@ -2119,10 +2125,9 @@ def step_key(step, identity, out=None):
tool = KIT / 'tools' / 'romfs_check.py'
return f'scan-1|{identity}|{SCAN_TOOL if tool.is_file() else "no tool"}|{written}'
if step == 'prompts':
tools = tree_digest(KIT / 'tools', ('pad_prompts.py', 'prompt_art.py', 'button_icons.py'))
binds = sha256_bytes(json.dumps(bind_lines(out)).encode())
written = all((out / f / 'manifest.tsv').is_file() for f in prompt_art.OVERLAYS)
return (f'prompts-1|{identity}|{tree_digest(prompt_art_dir(), ("**/*.png",))}|{tools}|{binds}'
return (f'prompts-1|{identity}|{tree_digest(prompt_art_dir(), ("**/*.png",))}|{PROMPT_TOOLS}|{binds}'
f'|{"written" if written else "missing"}')
lift = f'lift|{exefs_digest(out)}|{lift_digest()}'
if step == 'lift':

View File

@@ -205,7 +205,7 @@ class Romfs:
def dictionary(self, dict_id):
if self.dicts is None:
self.dicts = {}
pack = zstandard.ZstdDecompressor().decompressobj().decompress(self.raw(DICTIONARIES))
pack = unpack(self.raw(DICTIONARIES))
for lo, hi in sarc_members(pack).values():
d = zstandard.ZstdCompressionDict(pack[lo:hi])
self.dicts[d.dict_id()] = d
@@ -218,7 +218,27 @@ class Romfs:
return raw, None
dict_id = zstandard.get_frame_parameters(raw).dict_id
d = self.dictionary(dict_id) if dict_id else None
return zstandard.ZstdDecompressor(dict_data=d).decompressobj().decompress(raw), d
return unpack(raw, d), d
# How much of a packed file unpack gives the decompressor at a time.
PIECE = 1 << 16
def unpack(raw, dictionary=None):
"""The contents of a zstd frame, with the dictionary it was packed with; bytes after the frame are left out.
The file goes to the decompressor in pieces of PIECE bytes and the pieces' outputs are joined once. Given the
whole file in one call, the library returns one bytes object that it grows by copying as the output comes: made-up
data of the size of the interface texture archive, 69 MB packed and 244 MB unpacked, took 21 s that way on a
fast PC against 0.3 s in pieces. Players' builds waited 44 to 52 s before this tool's first folder, most of
its minute."""
out, parts = zstandard.ZstdDecompressor(dict_data=dictionary).decompressobj(), []
for at in range(0, len(raw), PIECE):
parts.append(out.decompress(raw[at:at + PIECE]))
if out.eof:
break
return b''.join(parts)
def sarc_members(data):