Ask the disk for room before a download or an unpack starts.

A disk that filled in the middle ended a download after four tries with an error that named no cause, and an unpack
with half a folder. The rest of a download, the build tools with their unpacked size, an update's archive and a
mod's zip are now measured against the free space first, and the refusal says how much is needed and how much is
free; a disk that fills during a download is named as full at once.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>

docs/launcher.md

@@ -419,6 +419,13 @@ default proxy handling, which reads `HTTPS_PROXY` and `HTTP_PROXY` (and `NO_PROX
 them, the proxy set in Windows' Internet settings. Only HTTP proxies work; a SOCKS proxy does not. An address that needs
 a user name and password takes them in the variable, as in `HTTPS_PROXY=http://user:password@host:port`.
 
+Before a download writes its first byte the disk is asked for room (`net.space_problem`): the rest of the file, by
+its expected size or else the length the server names, and 32 MB beyond it. The same is asked before the build tools
+are fetched (their downloads and their unpacked size, from the lock), before an update's archive is unpacked and
+before a mod's zip is. Without room the step stops with how much is needed and how much is free, and a part file
+stays for when there is room. A disk that fills in the middle of a download is named as full at once, not tried
+again four times.
+
 ## Uninstalling
 
 Settings > Uninstall NX2navi, or Settings > Apps > NX2navi (which runs `NX2navi.exe --uninstall`): what the launcher

launcher/mods.py

@@ -32,7 +32,7 @@ import time
 import zipfile
 from pathlib import Path
 
-from . import gameini
+from . import gameini, net
 
 sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tools'))
 import mod_files  # noqa: E402
@@ -409,6 +409,9 @@ def unpack(archive, dest):
             try:
                 with zipfile.ZipFile(archive) as z:
                     _safe_members(z.namelist())
+                    full = net.space_problem(dest, sum(i.file_size for i in z.infolist()), 'its files')
+                    if full:   # told before a file is written, not as an error halfway through
+                        raise ModError(full)
                     z.extractall(dest)
             except (zipfile.BadZipFile, NotImplementedError, RuntimeError) as e:
                 raise ModError(f'it is not a zip file the launcher can read ({e})')

launcher/net.py

@@ -17,9 +17,11 @@ Two checks can accept a server's certificate, tried in turn by request():
 A certificate both refuse ends the request with CertificateError: what each check said, whether the PC's clock agrees
 with the server's, and what to do."""
 import email.utils
+import errno
 import hashlib
 import json
 import os
+import shutil
 import ssl
 import time
 import urllib.error
@@ -240,10 +242,47 @@ def sha256_file(path):
     return h.hexdigest()
 
 
+SPACE_MARGIN = 32 << 20   # bytes a disk keeps free beyond what is about to be written to it (space_problem)
+
+
+class NoSpace(DownloadError):
+    """The disk has no room for the file. Asking again or another server gets the same answer, so download() does
+    not retry it, and it keeps the part file: the download goes on from it once there is room."""
+
+
+def free_bytes(folder):
+    """The free bytes of the disk that holds folder, asked of the nearest folder above it that exists; None when that
+    cannot be asked."""
+    p = Path(folder).absolute()
+    for d in (p, *p.parents):
+        try:
+            return shutil.disk_usage(d).free
+        except FileNotFoundError:
+            continue
+        except OSError:
+            return None
+    return None
+
+
+def space_problem(folder, need, what):
+    """'' when the disk of folder has room for need more bytes and SPACE_MARGIN, or when its free space cannot be
+    asked; else the sentence that says there is no room for what, how much it takes and how much is free. Asked
+    before a download or an unpack starts: a disk that fills in the middle otherwise ends it with an error that
+    names no cause and leaves half of it behind."""
+    free = free_bytes(folder)
+    if free is None or free >= max(0, need) + SPACE_MARGIN:
+        return ''
+    p = Path(folder).absolute()
+    return (f'there is no room for {what} on {p.anchor or p} ({(max(0, need) + SPACE_MARGIN) / 1e6:,.0f} MB needed, '
+            f'{free / 1e6:,.0f} MB free): make room there and try again')
+
+
 def download(url, dest, sha256=None, size=None, progress=None, cancel=None, headers=None, retries=4, version='dev'):
     """Fetch url to dest through dest.part, resuming a part file a cut connection left. progress(done, total) after
     every chunk; cancel() returning True raises Cancelled. A wrong size or sha256 deletes the file and raises
-    DownloadError. A dest already holding the expected sha256 is kept without a request."""
+    DownloadError. A dest already holding the expected sha256 is kept without a request. NoSpace, before anything
+    is written, when the disk has no room for the rest of the file (the expected size, else the size the server
+    names)."""
     dest = Path(dest)
     dest.parent.mkdir(parents=True, exist_ok=True)
     if sha256 and dest.is_file() and sha256_file(dest) == sha256.lower():
@@ -251,6 +290,10 @@ def download(url, dest, sha256=None, size=None, progress=None, cancel=None, head
             progress(dest.stat().st_size, dest.stat().st_size)
         return dest
     part = dest.with_name(dest.name + '.part')
+    if size:
+        full = space_problem(dest.parent, size - (part.stat().st_size if part.is_file() else 0), dest.name)
+        if full:
+            raise NoSpace(full)
     last_error = None
     for attempt in range(retries):
         have = part.stat().st_size if part.is_file() else 0
@@ -268,6 +311,10 @@ def download(url, dest, sha256=None, size=None, progress=None, cancel=None, head
                 # no length, and the bar ran past its end.
                 coming = int(r.headers.get('Content-Length') or 0)
                 total = size or (coming + have if coming else 0)
+                if coming and not size:
+                    full = space_problem(dest.parent, coming, dest.name)
+                    if full:
+                        raise NoSpace(full)
                 with open(part, 'ab' if have else 'wb') as f:
                     done = have
                     while True:
@@ -284,7 +331,7 @@ def download(url, dest, sha256=None, size=None, progress=None, cancel=None, head
                         if progress:
                             progress(done, total or done)
             break
-        except (Cancelled, CertificateError):
+        except (Cancelled, CertificateError, NoSpace):
             raise
         except DownloadError:
             part.unlink(missing_ok=True)
@@ -298,6 +345,10 @@ def download(url, dest, sha256=None, size=None, progress=None, cancel=None, head
                 raise DownloadError(f'{url}: HTTP {e.code}') from e
             last_error = e
         except (urllib.error.URLError, OSError, IncompleteRead) as e:
+            if isinstance(e, OSError) and e.errno == errno.ENOSPC:   # the disk filled meanwhile: trying again cannot help
+                p = dest.parent.absolute()
+                raise NoSpace(f'{p.anchor or p} is full: {dest.name} could not be written. Make room there and try '
+                              'again') from e
             last_error = e
         time.sleep(min(2 ** attempt, 16))
     else:

launcher/toolchain.py

@@ -251,6 +251,14 @@ def install(layout, lock, progress=None, cancel=None, log=print, version='dev'):
                                     and (layout.pkgs / p['filename']).stat().st_size == p['csize'])]
     total = sum(p['csize'] for p in need)
     log(f'toolchain: {len(pkgs)} packages, {len(need)} to download ({total / 1e6:.0f} MB)')
+    # Room for the downloads and for the unpacked tools beside the ones in use (msys64.new), asked before the first
+    # download: a disk that fills while a package unpacks otherwise ends the step with an error that names no cause.
+    # A package's installed size is in the lock; of a CLANG64 package only a few files are unpacked (_wanted), for
+    # which its download size stands in.
+    unpacked = sum(p['csize'] if p.get('env') else p.get('isize', 0) for p in pkgs)
+    full = net.space_problem(layout.toolchain, total + unpacked, 'the build tools')
+    if full:
+        raise ToolchainError(full)
     done_before = 0
     for p in need:
         dest = layout.pkgs / p['filename']
@@ -264,6 +272,8 @@ def install(layout, lock, progress=None, cancel=None, log=print, version='dev'):
                              version=version)
                 last_error = None
                 break
+            except net.NoSpace as e:   # another server has the same file for the same full disk
+                raise ToolchainError(str(e)) from e
             except net.DownloadError as e:
                 last_error = e
                 log(f'toolchain: {e}')

launcher/updater.py

@@ -60,7 +60,8 @@ def runtime_id(layout):
 
 
 def safe_extract(zip_path, dest):
-    """Unpack a zip into dest, refusing absolute names and names that climb out of it."""
+    """Unpack a zip into dest, refusing absolute names and names that climb out of it, and, before anything is
+    written, a zip whose files do not fit the disk (net.space_problem)."""
     dest = Path(dest)
     root = dest.resolve()
     with zipfile.ZipFile(zip_path) as z:
@@ -71,6 +72,10 @@ def safe_extract(zip_path, dest):
             target = (dest / name).resolve()
             if target != root and root not in target.parents:
                 raise UpdateError(f'{Path(zip_path).name}: unsafe path {info.filename}')
+        full = net.space_problem(dest, sum(info.file_size for info in z.infolist()),
+                                 f'the files of {Path(zip_path).name}')
+        if full:
+            raise UpdateError(full)
         z.extractall(dest)
 
 

tests/launcher_test.py

@@ -693,6 +693,82 @@ class NetTest(Tmp):
         for d, t in fetch({'Content-Length': '7'}, size=len(data)):
             self.assertEqual(t, len(data))
 
+    def low_disk(self, free):
+        """shutil.disk_usage as net sees it, answering with free[0] bytes free."""
+        import collections
+        usage = collections.namedtuple('usage', 'total used free')
+        return mock.patch.object(net.shutil, 'disk_usage', lambda path: usage(10 ** 12, 0, free[0]))
+
+    def test_the_free_space_is_asked_before_a_download(self):
+        """A download whose rest does not fit the disk stops before a byte is written, says how much is needed and
+        how much is free, and keeps its part file for when there is room; a disk that fills in the middle is named
+        as full at once instead of being tried four times."""
+        data = os.urandom(5000)
+        (self.tmp / 'f.bin').write_bytes(data)
+        srv = Server(self.tmp)
+        self.addCleanup(srv.close)
+        dest = self.tmp / 'out' / 'f.bin'
+        dest.parent.mkdir()
+        part = dest.with_name('f.bin.part')
+        part.write_bytes(data[:1000])
+        free = [40 << 20]
+        with self.low_disk(free):
+            self.assertEqual(net.space_problem(dest.parent, 1 << 20, 'x'), '')            # 1 MB and the margin fit
+            self.assertEqual(net.space_problem(self.tmp / 'not' / 'made' / 'yet', 1 << 20, 'x'), '')
+            with self.assertRaises(net.NoSpace) as refused:                              # the expected size is known:
+                net.download(srv.url + 'f.bin', dest, size=64 << 20)                     # no request is made
+            self.assertEqual(srv.hits, [])
+            self.assertIn('no room for f.bin on ', str(refused.exception))
+            self.assertIn('MB needed, 42 MB free', str(refused.exception))
+            self.assertEqual(part.read_bytes(), data[:1000])
+            free[0] = 1 << 20
+            with self.assertRaises(net.NoSpace):                                         # else the server's length is
+                net.download(srv.url + 'f.bin', dest)
+            self.assertEqual(len(srv.hits), 1)
+            self.assertEqual(part.read_bytes(), data[:1000])                             # kept for when there is room
+            free[0] = 1 << 30
+            net.download(srv.url + 'f.bin', dest, sha256=sha(data), size=len(data))
+            self.assertEqual(dest.read_bytes(), data)
+            self.assertEqual(srv.hits[-1][1].get('Range'), 'bytes=1000-')
+
+        class Full(io.BytesIO):   # an answer whose bytes cannot be written: the disk filled meanwhile
+            status, length, headers = 200, None, {}
+
+            def read(self, n=-1):
+                raise OSError(28, 'No space left on device')
+
+        with mock.patch.object(net, 'request', lambda url, h, timeout, version: Full()), \
+                mock.patch.object(net.time, 'sleep') as slept, self.assertRaises(net.NoSpace) as refused:
+            net.download('https://example.invalid/g.bin', self.tmp / 'out' / 'g.bin')
+        slept.assert_not_called()
+        self.assertIn('is full: g.bin could not be written', str(refused.exception))
+        with mock.patch.object(net.shutil, 'disk_usage', side_effect=OSError('no such drive')):
+            self.assertEqual(net.space_problem(dest.parent, 1 << 40, 'x'), '')            # cannot be asked: not refused
+
+    def test_the_free_space_is_asked_before_an_unpack(self):
+        z = make_app_zip(self.tmp, '0.9.0', extra={'big.bin': bytes(3 << 20)})
+        free = [2 << 20]
+        with self.low_disk(free):
+            with self.assertRaises(updater.UpdateError) as refused:
+                updater.safe_extract(z, self.tmp / 'x')
+            self.assertIn('no room for the files of NX2navi-app-0.9.0.zip', str(refused.exception))
+            self.assertFalse((self.tmp / 'x').exists())                                  # nothing was unpacked
+            from launcher import mods
+            with self.assertRaises(mods.ModError) as refused:
+                mods.unpack(z, self.tmp / 'mod')
+            self.assertIn('no room for its files', str(refused.exception))
+            self.assertFalse((self.tmp / 'mod').exists())
+            lock = {'packages': [{'name': 'p', 'version': '1', 'filename': 'p.pkg.tar.zst', 'sha256': '0' * 64,
+                                  'csize': 5 << 20, 'isize': 50 << 20}]}
+            with mock.patch.object(net, 'download') as fetched, self.assertRaises(toolchain.ToolchainError) as refused:
+                toolchain.install(self.layout, lock, log=lambda line: None)
+            fetched.assert_not_called()                                                  # asked before the first download
+            self.assertIn('no room for the build tools', str(refused.exception))
+            self.assertIn('(91 MB needed, 2 MB free)', str(refused.exception))            # 5 + 50 MiB and the margin
+            free[0] = 1 << 30
+            updater.safe_extract(z, self.tmp / 'x')
+        self.assertEqual((self.tmp / 'x' / 'big.bin').stat().st_size, 3 << 20)
+
     def test_plain_http_redirects_and_oversize(self):
         with self.assertRaises(net.DownloadError):                        # refused before any connection
             net.request('http://example.invalid/f.bin')