Files
Triforce-Source/nx2/nup.py
ruckus7382sfgsgdf 6f338c8f56 Triforce v0.5.0
In-game mods (GameBanana browser, search and installs, with the Mask Salesman), code-patch and cheat mods, the devs' test18 to test45 work (full performance overlay, frame pacing on, steady frame rate, save export and import, bug capture, new options rows, Setup's Mods page and Add to Steam), occlusion queries and new defaults, Auto resolution that scales, replay helper threads and robust buffers off by default, the options in sections.
2026-11-17 22:40:50 -06:00

709 lines
33 KiB
Python

"""Read a .nup package (a decrypted Switch 2 dump: titles, each with an exefs/ and a romfs/ tree) and extract it.
python -I nx2/nup.py <file.nup> list the titles and their trees
python -I nx2/nup.py <file.nup> --verify read-only hashes/extents and expected game .zs headers
python -I nx2/nup.py <file.nup> --extract <dir> [--title <n|version>] [--only exefs|romfs]
The layout (worked out from the file; little-endian):
0x000 "NUP\\x01" header, u64 file size at +0x40
0x080 "TOC\\x01", u32 titles, u64 dirs, u64 files, u64 strings offset (from 0x80), u64 strings size
0x0b0 titles, 0x40 each: u64 title id, u32 root dir, u32 exefs dir, u32 romfs dir, u32 version, ...,
u64 offset + u64 size of the title's metadata blob at +0x20
then dirs, 24 bytes each: u32 name offset, u32 name length, u32 first child, u32 children, u64 0
then files, 80 bytes each: u32 name offset, u32 name length, u64 size, u64 data offset, u64 stored size,
32-byte hash, u8 codec (0 raw, 1 zstd), u8 flags, u16 reserved, 12 bytes reserved
then the child list, u32 each: a node index; below the dir count a dir, else (index - dirs) a file
then the name strings (not 0-terminated)
The data of a file is stored as is at its data offset, or, in .nup files from newer dumpers, as a zstd frame whose content is the
file: its size field is then the size of the file, not of the frame (seen on rtld: 10524 bytes in the listing, frame content 10524).
A file that is itself zstd in the game (the romfs .zs files) is not one of those: its frame's content is larger than the file.
"""
import argparse
from contextlib import suppress
import errno
import hashlib
import os
import re
import shutil
import stat
import struct
import sys
from time import monotonic
ZSTD = b"\x28\xb5\x2f\xfd"
WINDOWS_DEVICES = {"CON", "CONIN$", "CONOUT$", "PRN", "AUX", "NUL"} | {p + n for p in ("COM", "LPT") for n in "123456789¹²³"}
def frame_content_size(head):
"""The content size a zstd frame header states, None when it states none"""
if head[:4] != ZSTD or len(head) < 18:
return None
fhd = head[4]
single = (fhd >> 5) & 1
at = 5 + (0 if single else 1) + (0, 1, 2, 4)[fhd & 3]
n = (1 if single else 0, 2, 4, 8)[fhd >> 6]
if not n:
return None
size = int.from_bytes(head[at:at + n], "little")
return size + 256 if n == 2 else size
class Nup:
def __init__(self, path, *, strict=False):
self.path = path
self.f = open(path, "rb")
h = self.f.read(0x80)
if strict and len(h) != 0x80:
self.f.close()
raise ValueError("truncated NUP header")
if h[:4] != b"NUP\x01":
raise SystemExit("%s: not a NUP package (magic %r)" % (path, h[:4]))
self.file_size = struct.unpack_from("<Q", h, 0x40)[0]
self.toc_offset, self.toc_size = struct.unpack_from("<QQ", h, 8)
self.toc_sha256 = h[24:56]
self.blob_offset = struct.unpack_from("<Q", h, 0x38)[0]
# (a package that ends early (a download or copy that did not finish) failed 30 minutes in as "short read in the
# package": said now, with the sizes)
have = os.fstat(self.f.fileno()).st_size
if strict:
try:
self._verify_toc() # check metadata hash/bounds before parsing records
except (ValueError, OSError, struct.error):
self.f.close()
raise
self.f.seek(0x80)
if self.file_size and have < self.file_size:
raise SystemExit("the .nup file is incomplete: it is %d MB, its header says %d MB. Download or copy it again "
"(wait for the download to finish) and check that the drive had room" % (have >> 20, self.file_size >> 20))
t = self.f.read(0x30)
if t[:4] != b"TOC\x01":
raise SystemExit("%s: no TOC at 0x80" % path)
nt, nd, nf, soff, ssz = struct.unpack_from("<IQQQQ", t, 4)
self.nd, self.nf = nd, nf
dirs_at = 0x80 + 0x30 + 0x40 * nt
files_at = dirs_at + 24 * nd
kids_at = files_at + 80 * nf
strings_at = 0x80 + soff
self.f.seek(0)
meta = self.f.read(strings_at + ssz)
self.strings = meta[strings_at:strings_at + ssz]
self.titles = []
for i in range(nt):
o = 0xb0 + 0x40 * i
tid, root, exefs, romfs, ver = struct.unpack_from("<QIIII", meta, o)
self.titles.append(dict(index=i, id=tid, root=root, exefs=exefs, romfs=romfs, version=ver))
self.dirs = [struct.unpack_from("<4I", meta, dirs_at + 24 * i) for i in range(nd)]
self.files = []
self.file_details = [] # hash, codec flags and reserved bytes; keep files tuples compatible
for i in range(nf):
no, nl, size, off, size2 = struct.unpack_from("<IIQQQ", meta, files_at + 80 * i)
self.files.append((no, nl, size, off, size2, meta[files_at + 80 * i + 64]))
at = files_at + 80 * i
self.file_details.append((meta[at + 32:at + 64], meta[at + 65], meta[at + 66:at + 80]))
nk = (strings_at - kids_at) // 4
self.kids = struct.unpack_from("<%dI" % nk, meta, kids_at)
if strict:
try:
self._verify_tree()
except ValueError:
self.f.close()
raise
def _verify_toc(self):
"""Check parsed metadata bounds and the complete TOC hash."""
self.f.seek(0)
h = self.f.read(0x80)
physical = os.fstat(self.f.fileno()).st_size
if (len(h) != 0x80 or self.file_size != physical or self.toc_offset != 0x80 or
self.toc_size < 0x30 or self.toc_offset + self.toc_size > self.blob_offset or
self.blob_offset > physical or self.blob_offset % 16):
raise ValueError("invalid declared file/TOC/blob bounds")
self.f.seek(self.toc_offset)
t = self.f.read(0x30)
if len(t) != 0x30 or t[:4] != b"TOC\x01":
raise ValueError("invalid TOC header")
nt, nd, nf, soff, ssz = struct.unpack_from("<IQQQQ", t, 4)
tables_end = 0x30 + 64 * nt + 24 * nd + 80 * nf
if (not nt or tables_end > soff or (soff - tables_end) % 4 or
soff + ssz > self.toc_size):
raise ValueError("parsed title/directory/file/child/string tables exceed declared TOC")
self.f.seek(self.toc_offset)
left, digest = self.toc_size, hashlib.sha256()
while left:
block = self.f.read(min(left, 8 << 20))
if not block:
raise ValueError("truncated TOC")
left -= len(block)
digest.update(block)
if digest.digest() != self.toc_sha256:
raise ValueError("TOC SHA-256 mismatch")
def _verify_tree(self):
"""Validate ranges/reachability and reject cycles without recursive traversal."""
for no, nl, first, count in self.dirs:
if no + nl > len(self.strings) or first + count > len(self.kids):
raise ValueError("directory name/child range outside table")
self.name(no, nl)
for record in self.files:
if record[0] + record[1] > len(self.strings):
raise ValueError("file name outside string table")
self.name(record[0], record[1])
if any(child >= self.nd + self.nf for child in self.kids):
raise ValueError("child index outside directory/file tables")
reached_dirs, reached_files = set(), set()
for title in self.titles:
if title["root"] >= self.nd or any(title[part] != 0xffffffff and title[part] >= self.nd
for part in ("exefs", "romfs")):
raise ValueError("title %d has an invalid directory index" % title["index"])
seen, active, pending = set(), set(), [(title["root"], False)]
while pending:
directory, leaving = pending.pop()
if leaving:
active.remove(directory)
seen.add(directory)
continue
if directory in active:
raise ValueError("directory cycle in title %d" % title["index"])
if directory in seen:
continue
active.add(directory)
pending.append((directory, True))
_, _, first, count = self.dirs[directory]
for child in self.kids[first:first + count]:
if child < self.nd:
pending.append((child, False))
else:
reached_files.add(child - self.nd)
if any(title[part] != 0xffffffff and title[part] not in seen for part in ("exefs", "romfs")):
raise ValueError("title subtree is not reachable from its root")
reached_dirs.update(seen)
if len(reached_dirs) != self.nd or len(reached_files) != self.nf:
raise ValueError("orphan directory/file records are not reachable from title roots")
def verify_walk(self, directory):
"""Already validated nodes, iteratively (including data/ and deep trees)."""
pending = [(directory, "")]
while pending:
node, prefix = pending.pop()
if node < self.nd:
no, nl, first, count = self.dirs[node]
for child in reversed(self.kids[first:first + count]):
path = prefix + self.name(*self.dirs[child][:2]) + "/" if child < self.nd else prefix
pending.append((child, path))
else:
record = self.files[node - self.nd]
yield prefix + self.name(record[0], record[1]), record
def name(self, off, n):
return self.strings[off:off + n].decode("utf-8")
def walk(self, d, prefix=""):
"""(relative path, file record) for every file under dir d."""
no, nl, first, n = self.dirs[d]
for k in self.kids[first:first + n]:
if k < self.nd:
cno, cnl = self.dirs[k][:2]
yield from self.walk(k, prefix + self.name(cno, cnl) + "/")
else:
r = self.files[k - self.nd]
yield prefix + self.name(r[0], r[1]), r
def inner_is_zstd(self, r):
"""Whether what this frame holds is itself a zstd frame"""
import zstandard
self.f.seek(r[3])
try:
return zstandard.ZstdDecompressor().decompressobj().decompress(self.f.read(min(r[2], 1 << 20)))[:4] == ZSTD
except zstandard.ZstdError: # (made with one of the game's dictionaries: the game's own file)
return False
def is_packed(self, r, name=""):
"""A file the package keeps compressed. A frame that states no size is tried: it is the file when it unpacks to the listed size.
A .zs file is zstd in the game too, and three of them happen to hold exactly as much as they take: that one is packed only if
what it holds is zstd again"""
if r[5] == 1: # The TOC explicitly marks outer compression, even for a non-Zstd .zs payload.
return True
if not name:
name = self.name(r[0], r[1])
self.f.seek(r[3])
head = self.f.read(18)
if head[:4] != ZSTD:
return False
size = frame_content_size(head)
if name.endswith(".zs") and (size is None or size == r[2]):
return size == r[2] and self.inner_is_zstd(r)
if size is not None:
return size == r[2]
import zstandard
self.f.seek(r[3])
d = zstandard.ZstdDecompressor().decompressobj()
got = 0
left = r[2]
try:
while left and got <= r[2] and not d.eof:
b = self.f.read(min(left, 1 << 20))
if not b:
break
left -= len(b)
got += len(d.decompress(b))
except zstandard.ZstdError:
return False
return got == r[2]
def copy_to(self, r, g, name="", *, packed=None):
"""The file, as itself, into the open file g"""
exact = packed is not None
if packed is None:
packed = self.is_packed(r, name)
self.f.seek(r[3])
left = r[2]
if packed:
left = r[4] if r[5] == 1 else left + (1 << 16) # explicit extents must not read the next member
import zstandard
d = zstandard.ZstdDecompressor().decompressobj()
got = 0
while left and not getattr(d, "eof", False):
b = self.f.read(min(left, 8 << 20))
if not b:
break
left -= len(b)
try:
out = d.decompress(b)
except zstandard.ZstdError: # (the end of the frame, and the next file's bytes after it)
break
got += len(out)
if got > r[2]:
raise SystemExit("a compressed file expanded past its declared size of %d bytes" % r[2])
g.write(out)
if exact and (left or d.unused_data):
raise SystemExit("a compressed extent has trailing bytes")
if r[5] == 1 and not d.eof:
raise SystemExit("a compressed file ends before its frame completes")
if got != r[2]:
raise SystemExit("a compressed file came out as %d bytes, %d expected" % (got, r[2]))
return
while left:
b = self.f.read(min(left, 8 << 20))
if not b:
raise SystemExit("the .nup file ends early (short read at %d of %d MB): it is incomplete. Download or copy it again"
% (self.f.tell() >> 20, os.fstat(self.f.fileno()).st_size >> 20))
g.write(b)
left -= len(b)
def read(self, r, n=None):
import io
b = io.BytesIO()
self.copy_to(r, b)
return b.getvalue() if n is None else b.getvalue()[:n]
class VerifySink:
"""A streamed digest and just enough prefix for the inner game-file header."""
def __init__(self):
self.size, self.prefix, self.hash = 0, b"", hashlib.sha256()
def write(self, data):
self.size += len(data)
self.hash.update(data)
self.prefix += data[:max(0, 18 - len(self.prefix))]
def verify(nup, titles=None, only=None):
"""Read-only container checks plus expected .zs headers, not full inner decoding/authenticity."""
import zstandard
bad = checked = 0
cache = {}
physical = os.fstat(nup.f.fileno()).st_size
try:
nup._verify_toc()
nup._verify_tree()
except (ValueError, OSError, struct.error) as error:
print("BAD package: %s" % error)
return dict(checked_records=0, decoded_extents=0, bad=1)
indices = {id(record): i for i, record in enumerate(nup.files)}
for title in nup.titles if titles is None else titles:
parts = [(only, title[only])] if only else [("", title["root"])]
for part, directory in parts:
if directory == 0xffffffff:
continue
for name, record in nup.verify_walk(directory):
checked += 1
expected, flags, reserved = nup.file_details[indices[id(record)]]
size, off, stored, codec = record[2:]
reasons = []
if codec not in (0, 1) or flags or any(reserved):
reasons.append("unsupported codec/flags/reserved bytes")
if (size == 0 and (off or stored or codec)) or (size and (
not stored or off < nup.blob_offset or off % 16 or off > physical or stored > physical - off)):
reasons.append("invalid extent bounds")
if codec == 0 and stored != size:
reasons.append("raw extent length differs from member length")
key = (off, stored, codec, size)
if not reasons:
if key not in cache:
sink, error = VerifySink(), None
try:
# Honor the explicit codec; leave legacy extraction heuristics unchanged.
nup.copy_to(record, sink, name, packed=codec == 1)
except (OSError, SystemExit, zstandard.ZstdError) as failure:
error = str(failure)
cache[key] = (sink.size, sink.hash.digest(), sink.prefix, error)
count, digest, prefix, error = cache[key]
if error:
reasons.append("member decode failed: " + error)
if count != size:
reasons.append("decoded length differs from member length")
if digest != expected:
reasons.append("member SHA-256 mismatch")
if name.endswith(".zs"):
try:
if prefix[:4] != ZSTD:
raise ValueError("missing magic")
zstandard.get_frame_parameters(prefix)
except (ValueError, zstandard.ZstdError):
reasons.append(".zs member lacks the expected game Zstd header")
if reasons:
bad += 1
print("BAD title %d %016x v%d %s/%s: %s" % (
title["index"], title["id"], title["version"], part, name, "; ".join(reasons)))
print("verified %d records, %d unique extents: %d bad; game .zs headers only (not inner decode/authenticity)" % (
checked, len(cache), bad))
return dict(checked_records=checked, decoded_extents=len(cache), bad=bad)
def pick_title(nup, want):
if want is None:
return max(nup.titles, key=lambda t: t["version"]) # (the newest: the update over the base)
for t in nup.titles:
if str(t["index"]) == want or str(t["version"]) == want or "%016x" % t["id"] == want.lower():
return t
raise SystemExit("no title %s" % want)
def long_path(p):
"""(Windows) p as an extended-length path (\\\\?\\...), so a game folder whose deepest file passes 260 characters (MAX_PATH)
still unpacks, links and is read (#1468: a 262-character path in Event\\EventSetting stopped the build with
FileNotFoundError). Python takes these everywhere, without Windows' long-paths setting. Elsewhere, or one already: as is."""
if os.name != "nt" or not p:
return p
p = os.path.abspath(p)
if p.startswith("\\\\?\\"):
return p
if p.startswith("\\\\"): # (a share: \\server\share -> \\?\UNC\server\share)
return "\\\\?\\UNC\\" + p[2:]
return "\\\\?\\" + p
def extraction_path(out, part, rel, temporary_suffix=".part"):
"""A package path stays under its part, without following existing output links."""
parts = rel.split("/")
if (not rel or "\\" in rel or any(not p or p in (".", "..") or p[-1] in " ." or
any(c in '<>:"|?*' or ord(c) < 32 for c in p) or p.split(".")[0].rstrip(" ").upper() in WINDOWS_DEVICES
for p in parts)):
raise SystemExit("unsafe path in the package: %r" % rel)
root = long_path(os.path.realpath(os.path.abspath(out)))
base = os.path.join(root, part)
dst = os.path.join(base, *parts)
if os.path.commonpath((os.path.normcase(base), os.path.normcase(os.path.abspath(dst)))) != os.path.normcase(base):
raise SystemExit("path leaves the extraction folder: %r" % rel)
at = root
paths = [at]
for p in (part, *parts):
at = os.path.join(at, p)
paths.append(at)
temporary = dst + temporary_suffix
paths.append(temporary)
for path in paths:
try:
info = os.lstat(path)
except FileNotFoundError:
continue
if stat.S_ISLNK(info.st_mode) or getattr(info, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0):
raise SystemExit("extraction path is a link or reparse point: %s" % path)
if path == temporary and info.st_nlink > 1:
raise SystemExit("extraction temporary file has other hard links: %s" % path)
return dst
class ProgressSink:
"""Report successful writes without changing the package reader's copy interface."""
def __init__(self, output, report):
self.output = output
self.report = report
def write(self, data):
count = self.output.write(data)
self.report(count)
return count
class HeadSink:
"""The first four bytes written, kept: extract's look at each romfs .zs file as it is unpacked (ZS_CHECKED)."""
def __init__(self, output):
self.output = output
self.head = b""
def write(self, data):
if len(self.head) < 4:
self.head += bytes(data[:4 - len(self.head)])
return self.output.write(data)
# (written into the folder a romfs was extracted to: how many of its .zs files were looked at as they were written, and
# which of them do not start as zstd data, by their path under romfs. nx2/make_game.py reads it in place of opening every
# .zs file again after the unpack, damaged_game_files: 102,185 files, 292 s of an 18 minute build where a virus scanner
# looks at each file that is opened)
ZS_CHECKED = "zs_checked.json"
def file_system(path):
"""The file system of the volume path is on (NTFS, exFAT, FAT32, ReFS): Windows only, None elsewhere"""
if os.name != "nt":
return None
import ctypes
k = ctypes.WinDLL("kernel32")
root, name = ctypes.create_unicode_buffer(261), ctypes.create_unicode_buffer(261)
if not k.GetVolumePathNameW(path, root, 261) or not k.GetVolumeInformationW(root, None, 0, None, None, None, name, 261):
return None
return name.value
def out_of_space(error, dst, part, rel, size):
"""A full drive said with the member being written and what its drive reports, to tell a full disk from a write the drive
refused with room to spare (issue 1388: 196 GB free on NTFS, stopped at 12 GB). None for any other error"""
if not isinstance(error, OSError) or error.errno != errno.ENOSPC:
return None
folder = os.path.dirname(dst)
try:
free = "%.1f GB free there" % (shutil.disk_usage(folder).free / 1e9)
except OSError:
free = "free space unknown"
kind = file_system(folder)
return OSError(errno.ENOSPC, "No space left on device writing %s/%s (%d bytes): %s%s" % (
part, rel, size, free, ", " + kind if kind else ""))
def extract(nup, t, out, only=None, quiet=False, zs=None):
"""zs: a dict that is given "files", the romfs .zs files there are, and "damaged", those of them that do not start as
zstd data (their paths under romfs, "/" between the parts): each one looked at as it is written, and one that was
there already read back"""
total = n = written = 0
if zs is not None:
zs.update(files=0, damaged=[])
interval = 2
next_report = 0 if quiet else monotonic() + interval
def progress(size):
nonlocal written, next_report
written += size
now = monotonic()
if now >= next_report:
print(" %.1f MiB written (%d files completed)" % (written / (1 << 20), n), flush=True)
next_report = now + interval
for part in ("exefs", "romfs"):
if only and part != only:
continue
for rel, r in nup.walk(t[part]):
dst = extraction_path(out, part, rel)
looked = zs is not None and part == "romfs" and rel.endswith(".zs")
if os.path.exists(dst) and os.path.getsize(dst) == r[2]:
if looked:
with open(dst, "rb") as f:
zs["files"] += 1
if not zstd_start(f.read(4)):
zs["damaged"].append(rel)
continue # (re-runs pick up where they stopped)
os.makedirs(os.path.dirname(dst), exist_ok=True)
temporary = dst + ".part"
g = open(temporary, "wb")
try:
with g:
sink = HeadSink(g) if looked else g
nup.copy_to(r, sink if quiet else ProgressSink(sink, progress), rel)
os.replace(temporary, dst)
if looked:
zs["files"] += 1
if not zstd_start(sink.head):
zs["damaged"].append(rel)
except BaseException as error:
full = out_of_space(error, dst, part, rel, r[2]) # (measured before the member's space is freed)
# Free the failed member's space without hiding the extraction error.
with suppress(OSError):
os.unlink(temporary)
if full:
raise full from error
raise
total += r[2]
n += 1
return n, total
def heal(nup, t, out, names):
"""An update's exefs need not hold every module whole: what is not an NSO there is taken from the package's other titles"""
for m in names:
p = extraction_path(out, "exefs", m)
have = b""
if os.path.exists(p):
with open(p, "rb") as f:
have = f.read(16)
if have[:4] == b"NSO0":
continue
for other in reversed(nup.titles):
if other["index"] == t["index"]:
continue
for rel, r in nup.walk(other["exefs"]):
if rel == m and nup.read(r, 4) == b"NSO0":
with open(p + ".part", "wb") as g:
g.write(nup.read(r))
os.replace(p + ".part", p)
print(" %s: taken from title %d (v%d)" % (m, other["index"], other["version"]))
break
else:
continue
break
else:
if have:
print(" %s is not a game module in any title: it starts %s" % (m, have.hex()))
def zstd_start(head):
"""Whether a file starts as zstd data: a frame, or a skippable frame (magic 0x184D2A50 to 0x184D2A5F)"""
return head == ZSTD or (len(head) == 4 and head[1:] == b"\x2a\x4d\x18" and head[0] & 0xf0 == 0x50)
def damaged_game_files(romfs, progress=None):
"""The romfs .zs files that do not start as zstd: in the 23.3 GB dumps three of them begin with zeros (the complete
dump is about 23.8 GB), and the game then leaves out every model that uses one: the Water Temple's walls and floors,
the Great Fairy Fountain's floor (#1002, #718, #1259). progress(done, total), when given, after each file: the files
are listed first, then opened one by one (about 100,000, minutes where a virus scanner looks at each one)"""
root = long_path(romfs) # (walked and read as long paths; the paths returned are under romfs as given)
paths = [] # (the long path read, the path under romfs as given)
for folder, dirs, files in os.walk(root):
dirs.sort()
rel = os.path.relpath(folder, root)
paths += [(os.path.join(folder, name), os.path.normpath(os.path.join(romfs, rel, name)))
for name in sorted(files) if name.endswith(".zs")]
found = []
for i, (path, given) in enumerate(paths):
with open(path, "rb") as f:
if not zstd_start(f.read(4)):
found.append(given)
if progress:
progress(i + 1, len(paths))
return found
def stand_in_for(path, damaged=()):
"""A whole file of the same kind beside a damaged one, to load in its place: the same last part of the name
(_Nrm.bntx.zs: a normal map for a normal map), the longest common start of the name, then the nearest size. Never
one of the damaged files, even one already given a stand-in"""
folder, name = os.path.split(path)
suffix = re.search(r"(_[A-Za-z]+\d*)?(\.[^_]*)$", name).group(0)
size = os.path.getsize(long_path(path))
best = None
for other in os.listdir(long_path(folder)):
if other == name or not other.endswith(suffix):
continue
candidate = os.path.join(folder, other)
if candidate in damaged:
continue
with open(long_path(candidate), "rb") as f:
if not zstd_start(f.read(4)):
continue
score = (len(os.path.commonprefix((name, other))), -abs(os.path.getsize(long_path(candidate)) - size))
if best is None or score > best[0]:
best = (score, candidate)
return best and best[1]
def stand_in_damaged(romfs, progress=None, known=None):
"""Each damaged .zs file in an unpacked romfs replaced by a copy of its stand-in: (damaged, stand-in or None) each.
Only for a romfs this tool unpacked: the player's own folder is never written to. progress: damaged_game_files'.
known: the damaged files as the unpack itself found them (extract's zs, paths under romfs with "/"): they are taken
as they are and no file is opened to look for them"""
done = []
if known is None:
damaged = damaged_game_files(romfs, progress)
else:
damaged = [os.path.normpath(os.path.join(romfs, *rel.split("/"))) for rel in known]
for path in damaged:
other = stand_in_for(path, set(damaged))
if other:
shutil.copyfile(long_path(other), long_path(path + ".part"))
os.replace(long_path(path + ".part"), long_path(path))
done.append((path, other))
return done
def main():
ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
ap.add_argument("nup")
ap.add_argument("--extract", metavar="DIR")
ap.add_argument("--verify", action="store_true", help="read-only hashes/extents and expected game .zs headers; all titles unless --title")
ap.add_argument("--title", help="index, version or title id (extract default: newest; verify default: all)")
ap.add_argument("--only", choices=("exefs", "romfs"))
a = ap.parse_args()
if a.verify and a.extract:
ap.error("--verify does not extract; run extraction separately")
try:
nup = Nup(a.nup, strict=a.verify)
except (OSError, ValueError, struct.error, IndexError, SystemExit) as error:
if not a.verify:
raise
print("BAD package: %s" % error)
return 1
if a.verify:
try:
titles = None if a.title is None else [pick_title(nup, a.title)]
return 1 if verify(nup, titles, a.only)["bad"] else 0
finally:
nup.f.close()
print("%s: %.2f GB, %d titles, %d dirs, %d files" % (os.path.basename(a.nup), nup.file_size / 1e9,
len(nup.titles), nup.nd, nup.nf))
for t in nup.titles:
ex = list(nup.walk(t["exefs"]))
ro = list(nup.walk(t["romfs"]))
print("title %d: %016x v%d exefs %d files (%s) romfs %d files, %.2f GB" % (
t["index"], t["id"], t["version"], len(ex), ", ".join("%s %d" % (p, r[2]) for p, r in ex),
len(ro), sum(r[2] for _, r in ro) / 1e9))
same = sum(1 for _, r in ro if r[2] != r[4])
if same:
print(" (%d romfs files with two different sizes)" % same)
if a.extract:
t = pick_title(nup, a.title)
# Measure the destination volume, including an existing mount or junction.
folder = os.path.realpath(a.extract)
while not os.path.isdir(folder):
parent = os.path.dirname(folder)
if parent == folder:
break
folder = parent
free = shutil.disk_usage(folder).free
print("extracting title %d (v%d) to %s (%.0f GB free)" % (t["index"], t["version"], a.extract, free / 1e9))
zs = {}
n, total = extract(nup, t, a.extract, a.only, zs=zs)
print("done: %d files written, %.2f GB" % (n, total / 1e9))
if a.only != "exefs": # (for nx2/make_game.py: the romfs .zs files as they were seen while they were written)
import json
with open(os.path.join(long_path(os.path.realpath(os.path.abspath(a.extract))), ZS_CHECKED), "w", encoding="utf-8") as f:
json.dump(zs, f)
if a.only != "romfs": # (main is ZBIC-compressed on firmware 22+: made a plain NSO for the tools)
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import unpack
heal(nup, t, a.extract, unpack.NAMES)
for m in unpack.NAMES:
p = extraction_path(a.extract, "exefs", m, temporary_suffix=".plain")
if os.path.exists(p) and unpack.unpack_nso(p):
print(" %s: ZBIC -> plain NSO" % m)
if __name__ == "__main__":
for stream in (sys.stdout, sys.stderr): # (-I ignores PYTHONIOENCODING: a .nup's or a file's name in other letters crashed a print)
try:
stream.reconfigure(encoding="utf-8", errors="replace")
except (AttributeError, ValueError):
pass
sys.exit(main())