From e0870b87b48ce1870066064b6d1de3a47715f5e0 Mon Sep 17 00:00:00 2001 From: chrg6 Date: Thu, 12 Nov 2026 22:30:46 -0500 Subject: [PATCH] Resume a part file only for the file it was begun for. A part left by a download of other bytes under the same name was resumed, and the result failed its sha256 check as if the download were damaged. A note beside the part now names the file it belongs to by the expected sha256; a part of another file is deleted with a log line and the download starts over, and the same file is still resumed from another server. Co-Authored-By: Claude Fable 5.2 --- docs/launcher.md | 5 +++++ launcher/jobs.py | 7 ++++--- launcher/net.py | 31 +++++++++++++++++++++++++++-- launcher/toolchain.py | 2 +- launcher/updater.py | 20 +++++++++---------- tests/launcher_test.py | 45 ++++++++++++++++++++++++++++++++++++++++-- 6 files changed, 92 insertions(+), 18 deletions(-) diff --git a/docs/launcher.md b/docs/launcher.md index 9cd48919..f69ca8a0 100644 --- a/docs/launcher.md +++ b/docs/launcher.md @@ -441,6 +441,11 @@ before a mod's zip is. Without room the step stops with how much is needed and h stays for when there is room. A disk that fills in the middle of a download is named as full at once, not tried again four times. +A cut download is resumed from its part file, `.part`, only as a part of the same file: `.part.of` +beside it names the file the part was begun for, by the expected sha256, so the same file is resumed from any +server. A part begun for other bytes under the same name (a release published again) is deleted with a line in +`launcher.log` and the download starts over; a part without the note, from an older launcher, is resumed as before. + ## Uninstalling Settings > Uninstall NX2navi, or Settings > Apps > NX2navi (which runs `NX2navi.exe --uninstall`): what the launcher diff --git a/launcher/jobs.py b/launcher/jobs.py index f8dfef5e..f0f72c4f 100644 --- a/launcher/jobs.py +++ b/launcher/jobs.py @@ -531,10 +531,10 @@ class UpdateDownloadJob(Job): self.begin('download') prog = lambda d, t: self.progress('download', d / max(t, 1), f'{d / 1e6:.1f} of {t / 1e6:.1f} MB') # noqa: E731 folder = updater.stage_app(layout, self.release, progress=prog, cancel=self.cancelled, - version=self.ctx.version) + version=self.ctx.version, log=self.log) if updater.runtime_needed(layout, self.release): updater.stage_runtime(layout, self.release, progress=prog, cancel=self.cancelled, - version=self.ctx.version) + version=self.ctx.version, log=self.log) self.finish('download') self.begin('verify') if not (folder / 'launcher' / 'start.pyw').is_file(): @@ -609,7 +609,8 @@ class StageJob(Job): winsys.background_thread() self.begin('download') prog = lambda d, t: self.progress('download', d / max(t, 1), f'{d / 1e6:.1f} of {t / 1e6:.1f} MB') # noqa: E731 - updater.stage(self.ctx.layout, self.release, progress=prog, cancel=self.cancelled, version=self.ctx.version) + updater.stage(self.ctx.layout, self.release, progress=prog, cancel=self.cancelled, version=self.ctx.version, + log=self.log) self.finish('download') diff --git a/launcher/net.py b/launcher/net.py index 2aa5f263..aade92e6 100644 --- a/launcher/net.py +++ b/launcher/net.py @@ -277,12 +277,22 @@ def space_problem(folder, need, what): f'{free / 1e6:,.0f} MB free): make room there and try again') -def download(url, dest, sha256=None, size=None, progress=None, cancel=None, headers=None, retries=4, version='dev'): +PART_OF = '.of' # .part.of: what the part file beside it is a part of (download) + + +def download(url, dest, sha256=None, size=None, progress=None, cancel=None, headers=None, retries=4, version='dev', + log=None): """Fetch url to dest through dest.part, resuming a part file a cut connection left. progress(done, total) after every chunk; cancel() returning True raises Cancelled. A wrong size or sha256 deletes the file and raises DownloadError. A dest already holding the expected sha256 is kept without a request. NoSpace, before anything is written, when the disk has no room for the rest of the file (the expected size, else the size the server - names).""" + names). + + A part file is resumed only as a part of the same file: dest.part.of beside it names the file it was begun + for, by the expected sha256 (the same file from any server) or, without one, by its address. + A part begun for another file under the same name (a release published again with other bytes) is deleted + with a line for log, where resuming it ended in "sha256 ..., expected ..." as if the download were damaged. A + part without that note (an older launcher's) is resumed as before.""" dest = Path(dest) dest.parent.mkdir(parents=True, exist_ok=True) if sha256 and dest.is_file() and sha256_file(dest) == sha256.lower(): @@ -290,6 +300,22 @@ def download(url, dest, sha256=None, size=None, progress=None, cancel=None, head progress(dest.stat().st_size, dest.stat().st_size) return dest part = dest.with_name(dest.name + '.part') + note = part.with_name(part.name + PART_OF) + wanted = f'sha256 {sha256.lower()}' if sha256 else url + if part.is_file(): + try: + begun_for = note.read_text(encoding='utf-8').strip() + except (OSError, ValueError): + begun_for = '' + if begun_for and begun_for != wanted: + part.unlink(missing_ok=True) + if log: + log(f'{part.name} was begun for another file ({begun_for}), not this one ({wanted}): it is deleted ' + 'and the download starts over') + try: + note.write_text(wanted, encoding='utf-8') + except OSError: # without the note the part is resumed as it always was + pass if size: full = space_problem(dest.parent, size - (part.stat().st_size if part.is_file() else 0), dest.name) if full: @@ -363,4 +389,5 @@ def download(url, dest, sha256=None, size=None, progress=None, cancel=None, head part.unlink(missing_ok=True) raise DownloadError(f'{url}: sha256 {digest}, expected {sha256.lower()}') os.replace(part, dest) + note.unlink(missing_ok=True) return dest diff --git a/launcher/toolchain.py b/launcher/toolchain.py index 19e3a7d7..1b2d97d7 100644 --- a/launcher/toolchain.py +++ b/launcher/toolchain.py @@ -269,7 +269,7 @@ def install(layout, lock, progress=None, cancel=None, log=print, version='dev'): try: net.download(mirror + p['filename'], dest, sha256=p['sha256'], size=p['csize'], cancel=cancel, progress=(lambda d, t, base=done_before: progress and progress('download', base + d, total)), - version=version) + version=version, log=lambda line: log(f'toolchain: {line}')) last_error = None break except net.NoSpace as e: # another server has the same file for the same full disk diff --git a/launcher/updater.py b/launcher/updater.py index 74b5e214..0b19b2d6 100644 --- a/launcher/updater.py +++ b/launcher/updater.py @@ -79,18 +79,18 @@ def safe_extract(zip_path, dest): z.extractall(dest) -def _fetch(release, key, layout, progress, cancel, version): +def _fetch(release, key, layout, progress, cancel, version, log=None): url, headers, size, sha = release.asset(key) entry = release.manifest[key] dest = layout.downloads / entry['name'] net.download(url, dest, sha256=sha, size=size, headers=headers, progress=progress, cancel=cancel, - version=version) + version=version, log=log) return dest -def stage_app(layout, release, progress=None, cancel=None, version='dev'): +def stage_app(layout, release, progress=None, cancel=None, version='dev', log=None): """Download, check and unpack the release's app archive into app/; returns that folder.""" - zip_path = _fetch(release, 'app', layout, progress, cancel, version) + zip_path = _fetch(release, 'app', layout, progress, cancel, version, log) ver = versions.display(release.version) final = layout.apps / ver part = layout.apps / f'{ver}.part' @@ -118,12 +118,12 @@ def runtime_needed(layout, release): return bool(rt) and not layout.dev and rt.get('id') != runtime_id(layout) -def stage_runtime(layout, release, progress=None, cancel=None, version='dev', dest=None): +def stage_runtime(layout, release, progress=None, cancel=None, version='dev', dest=None, log=None): """Unpack a new Python into dest (python.next, for the stub to swap in at the next start); False when the running one is current.""" if not runtime_needed(layout, release): return False - zip_path = _fetch(release, 'runtime', layout, progress, cancel, version) + zip_path = _fetch(release, 'runtime', layout, progress, cancel, version, log) nxt = Path(dest) if dest else layout.root / 'python.next' shutil.rmtree(nxt, ignore_errors=True) part = nxt.with_name(nxt.name + '.part') @@ -147,16 +147,16 @@ def prerelease(release): return True -def stage(layout, release, progress=None, cancel=None, version='dev'): +def stage(layout, release, progress=None, cancel=None, version='dev', log=None): """Download, check and unpack a release into app/, with the Python it brings in python.staged there, without switching to it; staged.json, written last, marks it complete. Earlier staged releases it supersedes are removed. Returns the folder.""" ver = versions.display(release.version) - folder = stage_app(layout, release, progress=progress, cancel=cancel, version=version) + folder = stage_app(layout, release, progress=progress, cancel=cancel, version=version, log=log) runtime = '' if runtime_needed(layout, release): stage_runtime(layout, release, progress=progress, cancel=cancel, version=version, - dest=folder / STAGED_PYTHON) + dest=folder / STAGED_PYTHON, log=log) runtime = release.manifest['runtime']['id'] write_text(folder / STAGED, json.dumps({'version': ver, 'runtime': runtime, 'prerelease': prerelease(release), 'notes': release.notes})) @@ -262,7 +262,7 @@ def prune(layout): shutil.rmtree(d, ignore_errors=True) if layout.downloads.is_dir(): for f in layout.downloads.iterdir(): - if f.suffix in ('.part', '.zip'): + if f.suffix in ('.part', '.zip', net.PART_OF): f.unlink(missing_ok=True) diff --git a/tests/launcher_test.py b/tests/launcher_test.py index 58a0a0fa..d1dbc8e2 100644 --- a/tests/launcher_test.py +++ b/tests/launcher_test.py @@ -693,6 +693,47 @@ class NetTest(Tmp): for d, t in fetch({'Content-Length': '7'}, size=len(data)): self.assertEqual(t, len(data)) + def test_a_part_file_of_another_download_is_not_resumed(self): + """A part file is resumed only for the file it was begun for (f.bin.part.of names it by the expected sha256). + One begun for other bytes under the same name, as when a release is published again, is deleted with a log + line; resumed, it ended in a sha256 error that read as a damaged download. The same file from another + server and a part without a note are resumed.""" + old, new = os.urandom(6000), os.urandom(6000) + (self.tmp / 'f.bin').write_bytes(new) + srv = Server(self.tmp) + self.addCleanup(srv.close) + dest = self.tmp / 'out' / 'f.bin' + dest.parent.mkdir() + part, note = dest.with_name('f.bin.part'), dest.with_name('f.bin.part.of') + part.write_bytes(old[:2000]) # cut while the old release downloaded + note.write_text(f'sha256 {sha(old)}') + lines = [] + net.download(srv.url + 'f.bin', dest, sha256=sha(new), size=len(new), log=lines.append) + self.assertEqual(dest.read_bytes(), new) + self.assertIsNone(srv.hits[-1][1].get('Range')) # from the start + self.assertEqual(len(lines), 1) + self.assertIn('f.bin.part was begun for another file', lines[0]) + self.assertFalse(part.exists() or note.exists()) # nothing stays beside the finished file + dest.unlink() + part.write_bytes(new[:2000]) # cut during this same file, on another server + note.write_text(f'sha256 {sha(new)}') + other = srv.url.replace('127.0.0.1', 'localhost') + net.download(other + 'f.bin', dest, sha256=sha(new), size=len(new), log=lines.append) + self.assertEqual(srv.hits[-1][1].get('Range'), 'bytes=2000-') + self.assertEqual((dest.read_bytes(), len(lines)), (new, 1)) + dest.unlink() + part.write_bytes(new[:3000]) # left by a launcher that wrote no note + net.download(srv.url + 'f.bin', dest, sha256=sha(new), size=len(new), log=lines.append) + self.assertEqual(srv.hits[-1][1].get('Range'), 'bytes=3000-') + self.assertEqual(dest.read_bytes(), new) + with self.assertRaises(net.Cancelled): # a cut download says what its part is of + net.download(srv.url + 'f.bin', self.tmp / 'out' / 'c.bin', sha256=sha(new), cancel=lambda: True) + self.assertEqual((self.tmp / 'out' / 'c.bin.part.of').read_text(), f'sha256 {sha(new)}') + self.layout.downloads.mkdir(parents=True, exist_ok=True) + (self.layout.downloads / 'x.zip.part.of').write_text('sha256 0') + updater.prune(self.layout) # and a note left behind is cleared away + self.assertEqual(list(self.layout.downloads.iterdir()), []) + def low_disk(self, free): """shutil.disk_usage as net sees it, answering with free[0] bytes free.""" import collections @@ -716,14 +757,14 @@ class NetTest(Tmp): self.assertEqual(net.space_problem(dest.parent, 1 << 20, 'x'), '') # 1 MB and the margin fit self.assertEqual(net.space_problem(self.tmp / 'not' / 'made' / 'yet', 1 << 20, 'x'), '') with self.assertRaises(net.NoSpace) as refused: # the expected size is known: - net.download(srv.url + 'f.bin', dest, size=64 << 20) # no request is made + net.download(srv.url + 'f.bin', dest, sha256=sha(data), size=64 << 20) # no request is made self.assertEqual(srv.hits, []) self.assertIn('no room for f.bin on ', str(refused.exception)) self.assertIn('MB needed, 42 MB free', str(refused.exception)) self.assertEqual(part.read_bytes(), data[:1000]) free[0] = 1 << 20 with self.assertRaises(net.NoSpace): # else the server's length is - net.download(srv.url + 'f.bin', dest) + net.download(srv.url + 'f.bin', dest, sha256=sha(data)) self.assertEqual(len(srv.hits), 1) self.assertEqual(part.read_bytes(), data[:1000]) # kept for when there is room free[0] = 1 << 30