From d0e6cd695eb3cf4c303f20e4a5c32a63cd7a4789 Mon Sep 17 00:00:00 2001 From: chrg6 Date: Thu, 12 Nov 2026 22:19:28 -0500 Subject: [PATCH] Ask the disk for room before a download or an unpack starts. A disk that filled in the middle ended a download after four tries with an error that named no cause, and an unpack with half a folder. The rest of a download, the build tools with their unpacked size, an update's archive and a mod's zip are now measured against the free space first, and the refusal says how much is needed and how much is free; a disk that fills during a download is named as full at once. Co-Authored-By: Claude Fable 5.2 --- docs/launcher.md | 7 ++++ launcher/mods.py | 5 ++- launcher/net.py | 55 ++++++++++++++++++++++++++++-- launcher/toolchain.py | 10 ++++++ launcher/updater.py | 7 +++- tests/launcher_test.py | 76 ++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 156 insertions(+), 4 deletions(-) diff --git a/docs/launcher.md b/docs/launcher.md index ee2d3e3e..13ebc314 100644 --- a/docs/launcher.md +++ b/docs/launcher.md @@ -419,6 +419,13 @@ default proxy handling, which reads `HTTPS_PROXY` and `HTTP_PROXY` (and `NO_PROX them, the proxy set in Windows' Internet settings. Only HTTP proxies work; a SOCKS proxy does not. An address that needs a user name and password takes them in the variable, as in `HTTPS_PROXY=http://user:password@host:port`. +Before a download writes its first byte the disk is asked for room (`net.space_problem`): the rest of the file, by +its expected size or else the length the server names, and 32 MB beyond it. The same is asked before the build tools +are fetched (their downloads and their unpacked size, from the lock), before an update's archive is unpacked and +before a mod's zip is. Without room the step stops with how much is needed and how much is free, and a part file +stays for when there is room. A disk that fills in the middle of a download is named as full at once, not tried +again four times. + ## Uninstalling Settings > Uninstall NX2navi, or Settings > Apps > NX2navi (which runs `NX2navi.exe --uninstall`): what the launcher diff --git a/launcher/mods.py b/launcher/mods.py index ca8b0138..8eb8893f 100644 --- a/launcher/mods.py +++ b/launcher/mods.py @@ -32,7 +32,7 @@ import time import zipfile from pathlib import Path -from . import gameini +from . import gameini, net sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tools')) import mod_files # noqa: E402 @@ -409,6 +409,9 @@ def unpack(archive, dest): try: with zipfile.ZipFile(archive) as z: _safe_members(z.namelist()) + full = net.space_problem(dest, sum(i.file_size for i in z.infolist()), 'its files') + if full: # told before a file is written, not as an error halfway through + raise ModError(full) z.extractall(dest) except (zipfile.BadZipFile, NotImplementedError, RuntimeError) as e: raise ModError(f'it is not a zip file the launcher can read ({e})') diff --git a/launcher/net.py b/launcher/net.py index f58c0cca..2aa5f263 100644 --- a/launcher/net.py +++ b/launcher/net.py @@ -17,9 +17,11 @@ Two checks can accept a server's certificate, tried in turn by request(): A certificate both refuse ends the request with CertificateError: what each check said, whether the PC's clock agrees with the server's, and what to do.""" import email.utils +import errno import hashlib import json import os +import shutil import ssl import time import urllib.error @@ -240,10 +242,47 @@ def sha256_file(path): return h.hexdigest() +SPACE_MARGIN = 32 << 20 # bytes a disk keeps free beyond what is about to be written to it (space_problem) + + +class NoSpace(DownloadError): + """The disk has no room for the file. Asking again or another server gets the same answer, so download() does + not retry it, and it keeps the part file: the download goes on from it once there is room.""" + + +def free_bytes(folder): + """The free bytes of the disk that holds folder, asked of the nearest folder above it that exists; None when that + cannot be asked.""" + p = Path(folder).absolute() + for d in (p, *p.parents): + try: + return shutil.disk_usage(d).free + except FileNotFoundError: + continue + except OSError: + return None + return None + + +def space_problem(folder, need, what): + """'' when the disk of folder has room for need more bytes and SPACE_MARGIN, or when its free space cannot be + asked; else the sentence that says there is no room for what, how much it takes and how much is free. Asked + before a download or an unpack starts: a disk that fills in the middle otherwise ends it with an error that + names no cause and leaves half of it behind.""" + free = free_bytes(folder) + if free is None or free >= max(0, need) + SPACE_MARGIN: + return '' + p = Path(folder).absolute() + return (f'there is no room for {what} on {p.anchor or p} ({(max(0, need) + SPACE_MARGIN) / 1e6:,.0f} MB needed, ' + f'{free / 1e6:,.0f} MB free): make room there and try again') + + def download(url, dest, sha256=None, size=None, progress=None, cancel=None, headers=None, retries=4, version='dev'): """Fetch url to dest through dest.part, resuming a part file a cut connection left. progress(done, total) after every chunk; cancel() returning True raises Cancelled. A wrong size or sha256 deletes the file and raises - DownloadError. A dest already holding the expected sha256 is kept without a request.""" + DownloadError. A dest already holding the expected sha256 is kept without a request. NoSpace, before anything + is written, when the disk has no room for the rest of the file (the expected size, else the size the server + names).""" dest = Path(dest) dest.parent.mkdir(parents=True, exist_ok=True) if sha256 and dest.is_file() and sha256_file(dest) == sha256.lower(): @@ -251,6 +290,10 @@ def download(url, dest, sha256=None, size=None, progress=None, cancel=None, head progress(dest.stat().st_size, dest.stat().st_size) return dest part = dest.with_name(dest.name + '.part') + if size: + full = space_problem(dest.parent, size - (part.stat().st_size if part.is_file() else 0), dest.name) + if full: + raise NoSpace(full) last_error = None for attempt in range(retries): have = part.stat().st_size if part.is_file() else 0 @@ -268,6 +311,10 @@ def download(url, dest, sha256=None, size=None, progress=None, cancel=None, head # no length, and the bar ran past its end. coming = int(r.headers.get('Content-Length') or 0) total = size or (coming + have if coming else 0) + if coming and not size: + full = space_problem(dest.parent, coming, dest.name) + if full: + raise NoSpace(full) with open(part, 'ab' if have else 'wb') as f: done = have while True: @@ -284,7 +331,7 @@ def download(url, dest, sha256=None, size=None, progress=None, cancel=None, head if progress: progress(done, total or done) break - except (Cancelled, CertificateError): + except (Cancelled, CertificateError, NoSpace): raise except DownloadError: part.unlink(missing_ok=True) @@ -298,6 +345,10 @@ def download(url, dest, sha256=None, size=None, progress=None, cancel=None, head raise DownloadError(f'{url}: HTTP {e.code}') from e last_error = e except (urllib.error.URLError, OSError, IncompleteRead) as e: + if isinstance(e, OSError) and e.errno == errno.ENOSPC: # the disk filled meanwhile: trying again cannot help + p = dest.parent.absolute() + raise NoSpace(f'{p.anchor or p} is full: {dest.name} could not be written. Make room there and try ' + 'again') from e last_error = e time.sleep(min(2 ** attempt, 16)) else: diff --git a/launcher/toolchain.py b/launcher/toolchain.py index 846d333f..19e3a7d7 100644 --- a/launcher/toolchain.py +++ b/launcher/toolchain.py @@ -251,6 +251,14 @@ def install(layout, lock, progress=None, cancel=None, log=print, version='dev'): and (layout.pkgs / p['filename']).stat().st_size == p['csize'])] total = sum(p['csize'] for p in need) log(f'toolchain: {len(pkgs)} packages, {len(need)} to download ({total / 1e6:.0f} MB)') + # Room for the downloads and for the unpacked tools beside the ones in use (msys64.new), asked before the first + # download: a disk that fills while a package unpacks otherwise ends the step with an error that names no cause. + # A package's installed size is in the lock; of a CLANG64 package only a few files are unpacked (_wanted), for + # which its download size stands in. + unpacked = sum(p['csize'] if p.get('env') else p.get('isize', 0) for p in pkgs) + full = net.space_problem(layout.toolchain, total + unpacked, 'the build tools') + if full: + raise ToolchainError(full) done_before = 0 for p in need: dest = layout.pkgs / p['filename'] @@ -264,6 +272,8 @@ def install(layout, lock, progress=None, cancel=None, log=print, version='dev'): version=version) last_error = None break + except net.NoSpace as e: # another server has the same file for the same full disk + raise ToolchainError(str(e)) from e except net.DownloadError as e: last_error = e log(f'toolchain: {e}') diff --git a/launcher/updater.py b/launcher/updater.py index 3ed49002..74b5e214 100644 --- a/launcher/updater.py +++ b/launcher/updater.py @@ -60,7 +60,8 @@ def runtime_id(layout): def safe_extract(zip_path, dest): - """Unpack a zip into dest, refusing absolute names and names that climb out of it.""" + """Unpack a zip into dest, refusing absolute names and names that climb out of it, and, before anything is + written, a zip whose files do not fit the disk (net.space_problem).""" dest = Path(dest) root = dest.resolve() with zipfile.ZipFile(zip_path) as z: @@ -71,6 +72,10 @@ def safe_extract(zip_path, dest): target = (dest / name).resolve() if target != root and root not in target.parents: raise UpdateError(f'{Path(zip_path).name}: unsafe path {info.filename}') + full = net.space_problem(dest, sum(info.file_size for info in z.infolist()), + f'the files of {Path(zip_path).name}') + if full: + raise UpdateError(full) z.extractall(dest) diff --git a/tests/launcher_test.py b/tests/launcher_test.py index 87e2e699..f9bd2282 100644 --- a/tests/launcher_test.py +++ b/tests/launcher_test.py @@ -693,6 +693,82 @@ class NetTest(Tmp): for d, t in fetch({'Content-Length': '7'}, size=len(data)): self.assertEqual(t, len(data)) + def low_disk(self, free): + """shutil.disk_usage as net sees it, answering with free[0] bytes free.""" + import collections + usage = collections.namedtuple('usage', 'total used free') + return mock.patch.object(net.shutil, 'disk_usage', lambda path: usage(10 ** 12, 0, free[0])) + + def test_the_free_space_is_asked_before_a_download(self): + """A download whose rest does not fit the disk stops before a byte is written, says how much is needed and + how much is free, and keeps its part file for when there is room; a disk that fills in the middle is named + as full at once instead of being tried four times.""" + data = os.urandom(5000) + (self.tmp / 'f.bin').write_bytes(data) + srv = Server(self.tmp) + self.addCleanup(srv.close) + dest = self.tmp / 'out' / 'f.bin' + dest.parent.mkdir() + part = dest.with_name('f.bin.part') + part.write_bytes(data[:1000]) + free = [40 << 20] + with self.low_disk(free): + self.assertEqual(net.space_problem(dest.parent, 1 << 20, 'x'), '') # 1 MB and the margin fit + self.assertEqual(net.space_problem(self.tmp / 'not' / 'made' / 'yet', 1 << 20, 'x'), '') + with self.assertRaises(net.NoSpace) as refused: # the expected size is known: + net.download(srv.url + 'f.bin', dest, size=64 << 20) # no request is made + self.assertEqual(srv.hits, []) + self.assertIn('no room for f.bin on ', str(refused.exception)) + self.assertIn('MB needed, 42 MB free', str(refused.exception)) + self.assertEqual(part.read_bytes(), data[:1000]) + free[0] = 1 << 20 + with self.assertRaises(net.NoSpace): # else the server's length is + net.download(srv.url + 'f.bin', dest) + self.assertEqual(len(srv.hits), 1) + self.assertEqual(part.read_bytes(), data[:1000]) # kept for when there is room + free[0] = 1 << 30 + net.download(srv.url + 'f.bin', dest, sha256=sha(data), size=len(data)) + self.assertEqual(dest.read_bytes(), data) + self.assertEqual(srv.hits[-1][1].get('Range'), 'bytes=1000-') + + class Full(io.BytesIO): # an answer whose bytes cannot be written: the disk filled meanwhile + status, length, headers = 200, None, {} + + def read(self, n=-1): + raise OSError(28, 'No space left on device') + + with mock.patch.object(net, 'request', lambda url, h, timeout, version: Full()), \ + mock.patch.object(net.time, 'sleep') as slept, self.assertRaises(net.NoSpace) as refused: + net.download('https://example.invalid/g.bin', self.tmp / 'out' / 'g.bin') + slept.assert_not_called() + self.assertIn('is full: g.bin could not be written', str(refused.exception)) + with mock.patch.object(net.shutil, 'disk_usage', side_effect=OSError('no such drive')): + self.assertEqual(net.space_problem(dest.parent, 1 << 40, 'x'), '') # cannot be asked: not refused + + def test_the_free_space_is_asked_before_an_unpack(self): + z = make_app_zip(self.tmp, '0.9.0', extra={'big.bin': bytes(3 << 20)}) + free = [2 << 20] + with self.low_disk(free): + with self.assertRaises(updater.UpdateError) as refused: + updater.safe_extract(z, self.tmp / 'x') + self.assertIn('no room for the files of NX2navi-app-0.9.0.zip', str(refused.exception)) + self.assertFalse((self.tmp / 'x').exists()) # nothing was unpacked + from launcher import mods + with self.assertRaises(mods.ModError) as refused: + mods.unpack(z, self.tmp / 'mod') + self.assertIn('no room for its files', str(refused.exception)) + self.assertFalse((self.tmp / 'mod').exists()) + lock = {'packages': [{'name': 'p', 'version': '1', 'filename': 'p.pkg.tar.zst', 'sha256': '0' * 64, + 'csize': 5 << 20, 'isize': 50 << 20}]} + with mock.patch.object(net, 'download') as fetched, self.assertRaises(toolchain.ToolchainError) as refused: + toolchain.install(self.layout, lock, log=lambda line: None) + fetched.assert_not_called() # asked before the first download + self.assertIn('no room for the build tools', str(refused.exception)) + self.assertIn('(91 MB needed, 2 MB free)', str(refused.exception)) # 5 + 50 MiB and the margin + free[0] = 1 << 30 + updater.safe_extract(z, self.tmp / 'x') + self.assertEqual((self.tmp / 'x' / 'big.bin').stat().st_size, 3 << 20) + def test_plain_http_redirects_and_oversize(self): with self.assertRaises(net.DownloadError): # refused before any connection net.request('http://example.invalid/f.bin')