diff --git a/docs/launcher.md b/docs/launcher.md index b6f92a1a..a00fb0f9 100644 --- a/docs/launcher.md +++ b/docs/launcher.md @@ -154,10 +154,13 @@ A path the Windows ANSI code page cannot hold breaks the MSYS2 tools, so the too and writes the Xbox, PlayStation, Steam Deck and keyboard and mouse prompts into `romfs-patches-*` beside the game (about 30 s), which the game's Controls > Button prompts loads. The keyboard and mouse prompts show the keys of `nx2navi.ini`'s `bind_` lines, so Rebuild the Game after rebinding keys draws them again; the step's key names - all it reads (the RomFS, the art, the tools, those lines), so it runs again only when one changed or a folder is - gone, and runs no other step again. Without art the row is not shown and the game keeps its own prompts. + all it reads (the RomFS, the art, those lines) and the version of the tools (`make_game.py` `PROMPT_TOOLS`, changed + by hand when a change to them changes what they write: an edit of a script alone does not draw the prompts + again), so it runs again only when one changed or a folder is gone, and runs no other step again. Without art + the row is not shown and the game keeps its own prompts. 5. **Check the Game Files**: make_game.py's last step runs `tools/romfs_check.py`, once for each copy of the game data - (about 4 minutes, in worker threads, never while the game runs). It decompresses every file, writes + (about 4 minutes on players' PCs up to 0.7.1, in worker threads, never while the game runs; its key holds the + check's version, `SCAN_TOOL`, in the same way). It decompresses every file, writes `romfs-check.json` beside the game, and for each damaged texture puts a stand-in, another texture of the same kind under the damaged one's name, in `mods/Damaged file stand-ins`, so the game can load what needs it. A damaged file of any other kind is only reported: only a clean dump restores it. In play the game logs a file it keeps opening diff --git a/tests/launcher_test.py b/tests/launcher_test.py index a6495f53..00e24358 100644 --- a/tests/launcher_test.py +++ b/tests/launcher_test.py @@ -3024,6 +3024,119 @@ class FileCheckTest(Tmp): self.assertNotEqual(logic_digest(a), logic_digest(c)) +# make_game.PROMPT_TOOLS, and the logic_digest of the three scripts that draw the button prompts when that value was +# last looked at, as SCAN_LOGIC for the file check: a change to their logic fails +# PromptToolsTest.test_the_prompt_tools_logic_is_the_recorded_one. If the change can alter what the tools write, give +# PROMPT_TOOLS a new value (every install then draws its prompts again) and record both here; if it cannot, record +# only the new digests. +PROMPT_LOGIC = ('ad59f2c4533f6ea2630b916ef5f3695f72daffe9b2b7663eea4168a4f08e4249', + {'pad_prompts.py': '1b008475289e0dabfefb8c61969c719c6faa79bbfc8504593e5fe1e79d4d4d33', + 'prompt_art.py': '87edacabcddb4952f5862373815d7d46a05c67f350e0734025ae4d2f83b12038', + 'button_icons.py': '3804e371a030965bc9eb4226f333b611651f25efaa67ee523b404edc3257818a'}) + + +def load_pad_prompts(): + """tools/pad_prompts.py as a module. Where fontTools is not installed, the names the tool and + tools/button_icons.py import from it get empty stand-ins for the import: the tests here use the tool's reading + of the game files only, which needs none of them. The module is not left in sys.modules.""" + import types + stubs = {} + try: + import fontTools # noqa: F401 + except ImportError: + for name, classes in (('fontTools', ()), ('fontTools.pens', ()), ('fontTools.ttLib', ('TTFont',)), + ('fontTools.pens.basePen', ('BasePen',)), ('fontTools.pens.boundsPen', ('BoundsPen',)), + ('fontTools.pens.recordingPen', ('RecordingPen',)), + ('fontTools.pens.reverseContourPen', ('ReverseContourPen',)), + ('fontTools.pens.t2CharStringPen', ('T2CharStringPen',)), + ('fontTools.pens.transformPen', ('TransformPen',))): + stubs[name] = types.ModuleType(name) + for cls in classes: + setattr(stubs[name], cls, type(cls, (), {})) + with mock.patch.dict(sys.modules, stubs): + for name in ('pad_prompts', 'button_icons'): + sys.modules.pop(name, None) + import pad_prompts + return pad_prompts + + +class PromptToolsTest(Tmp): + """tools/pad_prompts.py reads the game's files in pieces: the same bytes as in one call, without the minutes one + call took on the 244 MB texture archive. And the prompts step's key no longer follows the scripts' bytes.""" + + def setUp(self): + super().setUp() + try: + import zstandard # noqa: F401 + self.pp = load_pad_prompts() + except ImportError as e: + self.skipTest(f'needs zstandard and Pillow: {e}') + + def test_a_file_unpacks_to_the_same_bytes_in_pieces(self): + import random + import zstandard + pp = self.pp + seven = bytes(i % 7 for i in range(256)) + big = random.Random(5).randbytes(24 << 20).translate(seven) # packs to several pieces + small = b'a small file ' * 100 + samples = [bytes([65 + i % 7]) * 40 + f'sample {i} of the dictionary'.encode() * 3 for i in range(400)] + dic = zstandard.train_dictionary(4096, samples) + z = zstandard.ZstdCompressor(level=3) + with_dict = zstandard.ZstdCompressor(level=3, dict_data=dic).compress(b''.join(samples[:50])) + no_size = zstandard.ZstdCompressor(level=3, write_content_size=False).compress(big[:3 << 20]) + + def whole(raw, d=None): # as the tool read a file before: the whole file in one call + return zstandard.ZstdDecompressor(dict_data=d).decompressobj().decompress(raw) + packed = z.compress(big) + self.assertGreater(len(packed), 4 * pp.PIECE) + for name, raw, d in (('large', packed, None), ('small', z.compress(small), None), ('empty', z.compress(b''), None), + ('with a dictionary', with_dict, dic), ('no size in its frame', no_size, None), + ('bytes after the frame', z.compress(small) + bytes(300 << 10), None), + ('two frames', z.compress(small) + packed, None)): + self.assertEqual(pp.unpack(raw, d), whole(raw, d), name) + with mock.patch.object(pp, 'PIECE', 1000): + self.assertEqual(pp.unpack(raw, d), whole(raw, d), name) + self.assertEqual(pp.unpack(packed), big) + with self.assertRaises(zstandard.ZstdError): + pp.unpack(packed[:64] + bytes(300 << 10) + packed[64 + (300 << 10):]) + # the tool's reader of extracted game files: plain files, packed ones, and one packed with a dictionary of + # the game's dictionary archive + root = self.tmp / 'romfs' + for path, data in (('Font2/plain.bin', small), ('UI/big.bntx.zs', packed), ('UI/dict.bntx.zs', with_dict), + (pp.DICTIONARIES, z.compress(sarc({'zs.zsdic': dic.as_bytes()})))): + (root / path).parent.mkdir(parents=True, exist_ok=True) + (root / path).write_bytes(data) + romfs = pp.Romfs(root) + self.assertEqual(romfs.load('Font2/plain.bin'), (small, None)) + self.assertEqual(romfs.load('UI/big.bntx.zs'), (big, None)) + content, used = romfs.load('UI/dict.bntx.zs') + self.assertEqual((content, used.dict_id()), (b''.join(samples[:50]), dic.dict_id())) + + def test_the_prompts_key_is_the_one_installs_have(self): + import make_game + kit = self.tmp / 'kit' + art = kit / 'prompt-art' / 'XGamepad' / 'Default' + art.mkdir(parents=True) + (art / 'T_X_A_White.png').write_bytes(b'picture') + out = self.tmp / 'game' + out.mkdir() + ident = 'nup C:\\dumps\\game.nup 28000000000 1760000000' + released = 'ad59f2c4533f6ea2630b916ef5f3695f72daffe9b2b7663eea4168a4f08e4249' # the scripts of 0.7.1 + with mock.patch.object(make_game, 'KIT', kit): + pictures = make_game.tree_digest(kit / 'prompt-art', ('**/*.png',)) + binds = make_game.sha256_bytes(b'[]') + self.assertEqual(make_game.step_key('prompts', ident, out), + f'prompts-1|{ident}|{pictures}|{released}|{binds}|missing') + + def test_the_prompt_tools_logic_is_the_recorded_one(self): + import make_game + now = (make_game.PROMPT_TOOLS, {name: logic_digest(ROOT / 'tools' / name) for name in PROMPT_LOGIC[1]}) + self.assertEqual(now, PROMPT_LOGIC, + 'The logic of the tools that draw the button prompts changed. If the change can alter what ' + 'they write, give make_game.PROMPT_TOOLS a new value, so that installs draw their prompts ' + 'again; either way record the new pair in PROMPT_LOGIC (tests/launcher_test.py).') + + # ---------------------------------------------------------------- jobs FAKE_MAKE_GAME = r''' diff --git a/tools/make_game.py b/tools/make_game.py index 0ab1ebad..174c0c9f 100644 --- a/tools/make_game.py +++ b/tools/make_game.py @@ -112,6 +112,12 @@ SCAN_CHECKED = re.compile(r'^checked (\d+) of (\d+)$') # tools/romfs_check.py' # check, so the key of every install that passed it stays what it is. tests/launcher_test.py # test_the_file_checks_logic_is_the_recorded_one fails when the script's logic changes, to ask this question. SCAN_TOOL = '55ea0c150aaf9dac93a2e6fd3b4ca11e1c572b5eee8b6bd74960535964384634' +# The same for the tools that draw the button prompts (tools/pad_prompts.py, prompt_art.py, button_icons.py) in the +# prompts step's key, which held the digest of the three scripts' bytes: two of four releases changed a script, and +# each time every install drew its prompts again, a minute. Changed by hand when a change to the scripts can change +# what they write. Its value is their digest in 0.7.0 and 0.7.1, so the key of an install made by those stays what it +# is. tests/launcher_test.py test_the_prompt_tools_logic_is_the_recorded_one asks the question when the logic changes. +PROMPT_TOOLS = 'ad59f2c4533f6ea2630b916ef5f3695f72daffe9b2b7663eea4168a4f08e4249' # Bring-up fixes in the lifted SDK, each anchored on exact generated text that must occur once. LIFT_FIXES = [ @@ -2110,8 +2116,8 @@ def step_scan(out): def step_key(step, identity, out=None): """What a step's output depends on. exefs and romfs: the game files only; lift: the unpacked ExeFS and the lifter; compile: the lift, the runtime sources, this script and the processor (cpu_tag); prompts: the game files, - the kit's button art, the tools that draw it, the bind_ lines of nx2navi.ini, and whether every folder it writes - is still there; scan: the game files, the check's version (SCAN_TOOL) and whether its report is still there.""" + the kit's button art, the version of the tools that draw it (PROMPT_TOOLS), the bind_ lines of nx2navi.ini, and + whether every folder it writes is still there; scan: the game files, the check's version (SCAN_TOOL) and whether its report is still there.""" if step in ('exefs', 'romfs'): return f'{step}-1|{identity}' if step == 'scan': @@ -2119,10 +2125,9 @@ def step_key(step, identity, out=None): tool = KIT / 'tools' / 'romfs_check.py' return f'scan-1|{identity}|{SCAN_TOOL if tool.is_file() else "no tool"}|{written}' if step == 'prompts': - tools = tree_digest(KIT / 'tools', ('pad_prompts.py', 'prompt_art.py', 'button_icons.py')) binds = sha256_bytes(json.dumps(bind_lines(out)).encode()) written = all((out / f / 'manifest.tsv').is_file() for f in prompt_art.OVERLAYS) - return (f'prompts-1|{identity}|{tree_digest(prompt_art_dir(), ("**/*.png",))}|{tools}|{binds}' + return (f'prompts-1|{identity}|{tree_digest(prompt_art_dir(), ("**/*.png",))}|{PROMPT_TOOLS}|{binds}' f'|{"written" if written else "missing"}') lift = f'lift|{exefs_digest(out)}|{lift_digest()}' if step == 'lift': diff --git a/tools/pad_prompts.py b/tools/pad_prompts.py index 4eae7e09..7cee1d02 100644 --- a/tools/pad_prompts.py +++ b/tools/pad_prompts.py @@ -205,7 +205,7 @@ class Romfs: def dictionary(self, dict_id): if self.dicts is None: self.dicts = {} - pack = zstandard.ZstdDecompressor().decompressobj().decompress(self.raw(DICTIONARIES)) + pack = unpack(self.raw(DICTIONARIES)) for lo, hi in sarc_members(pack).values(): d = zstandard.ZstdCompressionDict(pack[lo:hi]) self.dicts[d.dict_id()] = d @@ -218,7 +218,27 @@ class Romfs: return raw, None dict_id = zstandard.get_frame_parameters(raw).dict_id d = self.dictionary(dict_id) if dict_id else None - return zstandard.ZstdDecompressor(dict_data=d).decompressobj().decompress(raw), d + return unpack(raw, d), d + + +# How much of a packed file unpack gives the decompressor at a time. +PIECE = 1 << 16 + + +def unpack(raw, dictionary=None): + """The contents of a zstd frame, with the dictionary it was packed with; bytes after the frame are left out. + + The file goes to the decompressor in pieces of PIECE bytes and the pieces' outputs are joined once. Given the + whole file in one call, the library returns one bytes object that it grows by copying as the output comes: made-up + data of the size of the interface texture archive, 69 MB packed and 244 MB unpacked, took 21 s that way on a + fast PC against 0.3 s in pieces. Players' builds waited 44 to 52 s before this tool's first folder, most of + its minute.""" + out, parts = zstandard.ZstdDecompressor(dict_data=dictionary).decompressobj(), [] + for at in range(0, len(raw), PIECE): + parts.append(out.decompress(raw[at:at + PIECE])) + if out.eof: + break + return b''.join(parts) def sarc_members(data):