mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-19 21:39:58 -04:00
Ask the disk for room before a download or an unpack starts.
A disk that filled in the middle ended a download after four tries with an error that named no cause, and an unpack with half a folder. The rest of a download, the build tools with their unpacked size, an update's archive and a mod's zip are now measured against the free space first, and the refusal says how much is needed and how much is free; a disk that fills during a download is named as full at once. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
@@ -32,7 +32,7 @@ import time
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
from . import gameini
|
||||
from . import gameini, net
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tools'))
|
||||
import mod_files # noqa: E402
|
||||
@@ -409,6 +409,9 @@ def unpack(archive, dest):
|
||||
try:
|
||||
with zipfile.ZipFile(archive) as z:
|
||||
_safe_members(z.namelist())
|
||||
full = net.space_problem(dest, sum(i.file_size for i in z.infolist()), 'its files')
|
||||
if full: # told before a file is written, not as an error halfway through
|
||||
raise ModError(full)
|
||||
z.extractall(dest)
|
||||
except (zipfile.BadZipFile, NotImplementedError, RuntimeError) as e:
|
||||
raise ModError(f'it is not a zip file the launcher can read ({e})')
|
||||
|
||||
@@ -17,9 +17,11 @@ Two checks can accept a server's certificate, tried in turn by request():
|
||||
A certificate both refuse ends the request with CertificateError: what each check said, whether the PC's clock agrees
|
||||
with the server's, and what to do."""
|
||||
import email.utils
|
||||
import errno
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import ssl
|
||||
import time
|
||||
import urllib.error
|
||||
@@ -240,10 +242,47 @@ def sha256_file(path):
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
SPACE_MARGIN = 32 << 20 # bytes a disk keeps free beyond what is about to be written to it (space_problem)
|
||||
|
||||
|
||||
class NoSpace(DownloadError):
|
||||
"""The disk has no room for the file. Asking again or another server gets the same answer, so download() does
|
||||
not retry it, and it keeps the part file: the download goes on from it once there is room."""
|
||||
|
||||
|
||||
def free_bytes(folder):
|
||||
"""The free bytes of the disk that holds folder, asked of the nearest folder above it that exists; None when that
|
||||
cannot be asked."""
|
||||
p = Path(folder).absolute()
|
||||
for d in (p, *p.parents):
|
||||
try:
|
||||
return shutil.disk_usage(d).free
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
except OSError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def space_problem(folder, need, what):
|
||||
"""'' when the disk of folder has room for need more bytes and SPACE_MARGIN, or when its free space cannot be
|
||||
asked; else the sentence that says there is no room for what, how much it takes and how much is free. Asked
|
||||
before a download or an unpack starts: a disk that fills in the middle otherwise ends it with an error that
|
||||
names no cause and leaves half of it behind."""
|
||||
free = free_bytes(folder)
|
||||
if free is None or free >= max(0, need) + SPACE_MARGIN:
|
||||
return ''
|
||||
p = Path(folder).absolute()
|
||||
return (f'there is no room for {what} on {p.anchor or p} ({(max(0, need) + SPACE_MARGIN) / 1e6:,.0f} MB needed, '
|
||||
f'{free / 1e6:,.0f} MB free): make room there and try again')
|
||||
|
||||
|
||||
def download(url, dest, sha256=None, size=None, progress=None, cancel=None, headers=None, retries=4, version='dev'):
|
||||
"""Fetch url to dest through dest.part, resuming a part file a cut connection left. progress(done, total) after
|
||||
every chunk; cancel() returning True raises Cancelled. A wrong size or sha256 deletes the file and raises
|
||||
DownloadError. A dest already holding the expected sha256 is kept without a request."""
|
||||
DownloadError. A dest already holding the expected sha256 is kept without a request. NoSpace, before anything
|
||||
is written, when the disk has no room for the rest of the file (the expected size, else the size the server
|
||||
names)."""
|
||||
dest = Path(dest)
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
if sha256 and dest.is_file() and sha256_file(dest) == sha256.lower():
|
||||
@@ -251,6 +290,10 @@ def download(url, dest, sha256=None, size=None, progress=None, cancel=None, head
|
||||
progress(dest.stat().st_size, dest.stat().st_size)
|
||||
return dest
|
||||
part = dest.with_name(dest.name + '.part')
|
||||
if size:
|
||||
full = space_problem(dest.parent, size - (part.stat().st_size if part.is_file() else 0), dest.name)
|
||||
if full:
|
||||
raise NoSpace(full)
|
||||
last_error = None
|
||||
for attempt in range(retries):
|
||||
have = part.stat().st_size if part.is_file() else 0
|
||||
@@ -268,6 +311,10 @@ def download(url, dest, sha256=None, size=None, progress=None, cancel=None, head
|
||||
# no length, and the bar ran past its end.
|
||||
coming = int(r.headers.get('Content-Length') or 0)
|
||||
total = size or (coming + have if coming else 0)
|
||||
if coming and not size:
|
||||
full = space_problem(dest.parent, coming, dest.name)
|
||||
if full:
|
||||
raise NoSpace(full)
|
||||
with open(part, 'ab' if have else 'wb') as f:
|
||||
done = have
|
||||
while True:
|
||||
@@ -284,7 +331,7 @@ def download(url, dest, sha256=None, size=None, progress=None, cancel=None, head
|
||||
if progress:
|
||||
progress(done, total or done)
|
||||
break
|
||||
except (Cancelled, CertificateError):
|
||||
except (Cancelled, CertificateError, NoSpace):
|
||||
raise
|
||||
except DownloadError:
|
||||
part.unlink(missing_ok=True)
|
||||
@@ -298,6 +345,10 @@ def download(url, dest, sha256=None, size=None, progress=None, cancel=None, head
|
||||
raise DownloadError(f'{url}: HTTP {e.code}') from e
|
||||
last_error = e
|
||||
except (urllib.error.URLError, OSError, IncompleteRead) as e:
|
||||
if isinstance(e, OSError) and e.errno == errno.ENOSPC: # the disk filled meanwhile: trying again cannot help
|
||||
p = dest.parent.absolute()
|
||||
raise NoSpace(f'{p.anchor or p} is full: {dest.name} could not be written. Make room there and try '
|
||||
'again') from e
|
||||
last_error = e
|
||||
time.sleep(min(2 ** attempt, 16))
|
||||
else:
|
||||
|
||||
@@ -251,6 +251,14 @@ def install(layout, lock, progress=None, cancel=None, log=print, version='dev'):
|
||||
and (layout.pkgs / p['filename']).stat().st_size == p['csize'])]
|
||||
total = sum(p['csize'] for p in need)
|
||||
log(f'toolchain: {len(pkgs)} packages, {len(need)} to download ({total / 1e6:.0f} MB)')
|
||||
# Room for the downloads and for the unpacked tools beside the ones in use (msys64.new), asked before the first
|
||||
# download: a disk that fills while a package unpacks otherwise ends the step with an error that names no cause.
|
||||
# A package's installed size is in the lock; of a CLANG64 package only a few files are unpacked (_wanted), for
|
||||
# which its download size stands in.
|
||||
unpacked = sum(p['csize'] if p.get('env') else p.get('isize', 0) for p in pkgs)
|
||||
full = net.space_problem(layout.toolchain, total + unpacked, 'the build tools')
|
||||
if full:
|
||||
raise ToolchainError(full)
|
||||
done_before = 0
|
||||
for p in need:
|
||||
dest = layout.pkgs / p['filename']
|
||||
@@ -264,6 +272,8 @@ def install(layout, lock, progress=None, cancel=None, log=print, version='dev'):
|
||||
version=version)
|
||||
last_error = None
|
||||
break
|
||||
except net.NoSpace as e: # another server has the same file for the same full disk
|
||||
raise ToolchainError(str(e)) from e
|
||||
except net.DownloadError as e:
|
||||
last_error = e
|
||||
log(f'toolchain: {e}')
|
||||
|
||||
@@ -60,7 +60,8 @@ def runtime_id(layout):
|
||||
|
||||
|
||||
def safe_extract(zip_path, dest):
|
||||
"""Unpack a zip into dest, refusing absolute names and names that climb out of it."""
|
||||
"""Unpack a zip into dest, refusing absolute names and names that climb out of it, and, before anything is
|
||||
written, a zip whose files do not fit the disk (net.space_problem)."""
|
||||
dest = Path(dest)
|
||||
root = dest.resolve()
|
||||
with zipfile.ZipFile(zip_path) as z:
|
||||
@@ -71,6 +72,10 @@ def safe_extract(zip_path, dest):
|
||||
target = (dest / name).resolve()
|
||||
if target != root and root not in target.parents:
|
||||
raise UpdateError(f'{Path(zip_path).name}: unsafe path {info.filename}')
|
||||
full = net.space_problem(dest, sum(info.file_size for info in z.infolist()),
|
||||
f'the files of {Path(zip_path).name}')
|
||||
if full:
|
||||
raise UpdateError(full)
|
||||
z.extractall(dest)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user