Ask the disk for room before a download or an unpack starts.

A disk that filled in the middle ended a download after four tries with an error that named no cause, and an unpack
with half a folder. The rest of a download, the build tools with their unpacked size, an update's archive and a
mod's zip are now measured against the free space first, and the refusal says how much is needed and how much is
free; a disk that fills during a download is named as full at once.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
chrg6
2026-11-12 22:19:28 -05:00
parent fe041188f5
commit d0e6cd695e
6 changed files with 156 additions and 4 deletions

View File

@@ -32,7 +32,7 @@ import time
import zipfile
from pathlib import Path
from . import gameini
from . import gameini, net
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tools'))
import mod_files # noqa: E402
@@ -409,6 +409,9 @@ def unpack(archive, dest):
try:
with zipfile.ZipFile(archive) as z:
_safe_members(z.namelist())
full = net.space_problem(dest, sum(i.file_size for i in z.infolist()), 'its files')
if full: # told before a file is written, not as an error halfway through
raise ModError(full)
z.extractall(dest)
except (zipfile.BadZipFile, NotImplementedError, RuntimeError) as e:
raise ModError(f'it is not a zip file the launcher can read ({e})')

View File

@@ -17,9 +17,11 @@ Two checks can accept a server's certificate, tried in turn by request():
A certificate both refuse ends the request with CertificateError: what each check said, whether the PC's clock agrees
with the server's, and what to do."""
import email.utils
import errno
import hashlib
import json
import os
import shutil
import ssl
import time
import urllib.error
@@ -240,10 +242,47 @@ def sha256_file(path):
return h.hexdigest()
SPACE_MARGIN = 32 << 20 # bytes a disk keeps free beyond what is about to be written to it (space_problem)
class NoSpace(DownloadError):
"""The disk has no room for the file. Asking again or another server gets the same answer, so download() does
not retry it, and it keeps the part file: the download goes on from it once there is room."""
def free_bytes(folder):
"""The free bytes of the disk that holds folder, asked of the nearest folder above it that exists; None when that
cannot be asked."""
p = Path(folder).absolute()
for d in (p, *p.parents):
try:
return shutil.disk_usage(d).free
except FileNotFoundError:
continue
except OSError:
return None
return None
def space_problem(folder, need, what):
"""'' when the disk of folder has room for need more bytes and SPACE_MARGIN, or when its free space cannot be
asked; else the sentence that says there is no room for what, how much it takes and how much is free. Asked
before a download or an unpack starts: a disk that fills in the middle otherwise ends it with an error that
names no cause and leaves half of it behind."""
free = free_bytes(folder)
if free is None or free >= max(0, need) + SPACE_MARGIN:
return ''
p = Path(folder).absolute()
return (f'there is no room for {what} on {p.anchor or p} ({(max(0, need) + SPACE_MARGIN) / 1e6:,.0f} MB needed, '
f'{free / 1e6:,.0f} MB free): make room there and try again')
def download(url, dest, sha256=None, size=None, progress=None, cancel=None, headers=None, retries=4, version='dev'):
"""Fetch url to dest through dest.part, resuming a part file a cut connection left. progress(done, total) after
every chunk; cancel() returning True raises Cancelled. A wrong size or sha256 deletes the file and raises
DownloadError. A dest already holding the expected sha256 is kept without a request."""
DownloadError. A dest already holding the expected sha256 is kept without a request. NoSpace, before anything
is written, when the disk has no room for the rest of the file (the expected size, else the size the server
names)."""
dest = Path(dest)
dest.parent.mkdir(parents=True, exist_ok=True)
if sha256 and dest.is_file() and sha256_file(dest) == sha256.lower():
@@ -251,6 +290,10 @@ def download(url, dest, sha256=None, size=None, progress=None, cancel=None, head
progress(dest.stat().st_size, dest.stat().st_size)
return dest
part = dest.with_name(dest.name + '.part')
if size:
full = space_problem(dest.parent, size - (part.stat().st_size if part.is_file() else 0), dest.name)
if full:
raise NoSpace(full)
last_error = None
for attempt in range(retries):
have = part.stat().st_size if part.is_file() else 0
@@ -268,6 +311,10 @@ def download(url, dest, sha256=None, size=None, progress=None, cancel=None, head
# no length, and the bar ran past its end.
coming = int(r.headers.get('Content-Length') or 0)
total = size or (coming + have if coming else 0)
if coming and not size:
full = space_problem(dest.parent, coming, dest.name)
if full:
raise NoSpace(full)
with open(part, 'ab' if have else 'wb') as f:
done = have
while True:
@@ -284,7 +331,7 @@ def download(url, dest, sha256=None, size=None, progress=None, cancel=None, head
if progress:
progress(done, total or done)
break
except (Cancelled, CertificateError):
except (Cancelled, CertificateError, NoSpace):
raise
except DownloadError:
part.unlink(missing_ok=True)
@@ -298,6 +345,10 @@ def download(url, dest, sha256=None, size=None, progress=None, cancel=None, head
raise DownloadError(f'{url}: HTTP {e.code}') from e
last_error = e
except (urllib.error.URLError, OSError, IncompleteRead) as e:
if isinstance(e, OSError) and e.errno == errno.ENOSPC: # the disk filled meanwhile: trying again cannot help
p = dest.parent.absolute()
raise NoSpace(f'{p.anchor or p} is full: {dest.name} could not be written. Make room there and try '
'again') from e
last_error = e
time.sleep(min(2 ** attempt, 16))
else:

View File

@@ -251,6 +251,14 @@ def install(layout, lock, progress=None, cancel=None, log=print, version='dev'):
and (layout.pkgs / p['filename']).stat().st_size == p['csize'])]
total = sum(p['csize'] for p in need)
log(f'toolchain: {len(pkgs)} packages, {len(need)} to download ({total / 1e6:.0f} MB)')
# Room for the downloads and for the unpacked tools beside the ones in use (msys64.new), asked before the first
# download: a disk that fills while a package unpacks otherwise ends the step with an error that names no cause.
# A package's installed size is in the lock; of a CLANG64 package only a few files are unpacked (_wanted), for
# which its download size stands in.
unpacked = sum(p['csize'] if p.get('env') else p.get('isize', 0) for p in pkgs)
full = net.space_problem(layout.toolchain, total + unpacked, 'the build tools')
if full:
raise ToolchainError(full)
done_before = 0
for p in need:
dest = layout.pkgs / p['filename']
@@ -264,6 +272,8 @@ def install(layout, lock, progress=None, cancel=None, log=print, version='dev'):
version=version)
last_error = None
break
except net.NoSpace as e: # another server has the same file for the same full disk
raise ToolchainError(str(e)) from e
except net.DownloadError as e:
last_error = e
log(f'toolchain: {e}')

View File

@@ -60,7 +60,8 @@ def runtime_id(layout):
def safe_extract(zip_path, dest):
"""Unpack a zip into dest, refusing absolute names and names that climb out of it."""
"""Unpack a zip into dest, refusing absolute names and names that climb out of it, and, before anything is
written, a zip whose files do not fit the disk (net.space_problem)."""
dest = Path(dest)
root = dest.resolve()
with zipfile.ZipFile(zip_path) as z:
@@ -71,6 +72,10 @@ def safe_extract(zip_path, dest):
target = (dest / name).resolve()
if target != root and root not in target.parents:
raise UpdateError(f'{Path(zip_path).name}: unsafe path {info.filename}')
full = net.space_problem(dest, sum(info.file_size for info in z.infolist()),
f'the files of {Path(zip_path).name}')
if full:
raise UpdateError(full)
z.extractall(dest)