Resume a part file only for the file it was begun for.

A part left by a download of other bytes under the same name was resumed, and the result failed its sha256 check
as if the download were damaged. A note beside the part now names the file it belongs to by the expected sha256;
a part of another file is deleted with a log line and the download starts over, and the same file is still resumed
from another server.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
chrg6
2026-11-12 22:30:46 -05:00
parent 9dcd73eebc
commit e0870b87b4
6 changed files with 92 additions and 18 deletions

View File

@@ -441,6 +441,11 @@ before a mod's zip is. Without room the step stops with how much is needed and h
stays for when there is room. A disk that fills in the middle of a download is named as full at once, not tried stays for when there is room. A disk that fills in the middle of a download is named as full at once, not tried
again four times. again four times.
A cut download is resumed from its part file, `<name>.part`, only as a part of the same file: `<name>.part.of`
beside it names the file the part was begun for, by the expected sha256, so the same file is resumed from any
server. A part begun for other bytes under the same name (a release published again) is deleted with a line in
`launcher.log` and the download starts over; a part without the note, from an older launcher, is resumed as before.
## Uninstalling ## Uninstalling
Settings > Uninstall NX2navi, or Settings > Apps > NX2navi (which runs `NX2navi.exe --uninstall`): what the launcher Settings > Uninstall NX2navi, or Settings > Apps > NX2navi (which runs `NX2navi.exe --uninstall`): what the launcher

View File

@@ -531,10 +531,10 @@ class UpdateDownloadJob(Job):
self.begin('download') self.begin('download')
prog = lambda d, t: self.progress('download', d / max(t, 1), f'{d / 1e6:.1f} of {t / 1e6:.1f} MB') # noqa: E731 prog = lambda d, t: self.progress('download', d / max(t, 1), f'{d / 1e6:.1f} of {t / 1e6:.1f} MB') # noqa: E731
folder = updater.stage_app(layout, self.release, progress=prog, cancel=self.cancelled, folder = updater.stage_app(layout, self.release, progress=prog, cancel=self.cancelled,
version=self.ctx.version) version=self.ctx.version, log=self.log)
if updater.runtime_needed(layout, self.release): if updater.runtime_needed(layout, self.release):
updater.stage_runtime(layout, self.release, progress=prog, cancel=self.cancelled, updater.stage_runtime(layout, self.release, progress=prog, cancel=self.cancelled,
version=self.ctx.version) version=self.ctx.version, log=self.log)
self.finish('download') self.finish('download')
self.begin('verify') self.begin('verify')
if not (folder / 'launcher' / 'start.pyw').is_file(): if not (folder / 'launcher' / 'start.pyw').is_file():
@@ -609,7 +609,8 @@ class StageJob(Job):
winsys.background_thread() winsys.background_thread()
self.begin('download') self.begin('download')
prog = lambda d, t: self.progress('download', d / max(t, 1), f'{d / 1e6:.1f} of {t / 1e6:.1f} MB') # noqa: E731 prog = lambda d, t: self.progress('download', d / max(t, 1), f'{d / 1e6:.1f} of {t / 1e6:.1f} MB') # noqa: E731
updater.stage(self.ctx.layout, self.release, progress=prog, cancel=self.cancelled, version=self.ctx.version) updater.stage(self.ctx.layout, self.release, progress=prog, cancel=self.cancelled, version=self.ctx.version,
log=self.log)
self.finish('download') self.finish('download')

View File

@@ -277,12 +277,22 @@ def space_problem(folder, need, what):
f'{free / 1e6:,.0f} MB free): make room there and try again') f'{free / 1e6:,.0f} MB free): make room there and try again')
def download(url, dest, sha256=None, size=None, progress=None, cancel=None, headers=None, retries=4, version='dev'): PART_OF = '.of' # <dest>.part.of: what the part file beside it is a part of (download)
def download(url, dest, sha256=None, size=None, progress=None, cancel=None, headers=None, retries=4, version='dev',
log=None):
"""Fetch url to dest through dest.part, resuming a part file a cut connection left. progress(done, total) after """Fetch url to dest through dest.part, resuming a part file a cut connection left. progress(done, total) after
every chunk; cancel() returning True raises Cancelled. A wrong size or sha256 deletes the file and raises every chunk; cancel() returning True raises Cancelled. A wrong size or sha256 deletes the file and raises
DownloadError. A dest already holding the expected sha256 is kept without a request. NoSpace, before anything DownloadError. A dest already holding the expected sha256 is kept without a request. NoSpace, before anything
is written, when the disk has no room for the rest of the file (the expected size, else the size the server is written, when the disk has no room for the rest of the file (the expected size, else the size the server
names).""" names).
A part file is resumed only as a part of the same file: dest.part.of beside it names the file it was begun
for, by the expected sha256 (the same file from any server) or, without one, by its address.
A part begun for another file under the same name (a release published again with other bytes) is deleted
with a line for log, where resuming it ended in "sha256 ..., expected ..." as if the download were damaged. A
part without that note (an older launcher's) is resumed as before."""
dest = Path(dest) dest = Path(dest)
dest.parent.mkdir(parents=True, exist_ok=True) dest.parent.mkdir(parents=True, exist_ok=True)
if sha256 and dest.is_file() and sha256_file(dest) == sha256.lower(): if sha256 and dest.is_file() and sha256_file(dest) == sha256.lower():
@@ -290,6 +300,22 @@ def download(url, dest, sha256=None, size=None, progress=None, cancel=None, head
progress(dest.stat().st_size, dest.stat().st_size) progress(dest.stat().st_size, dest.stat().st_size)
return dest return dest
part = dest.with_name(dest.name + '.part') part = dest.with_name(dest.name + '.part')
note = part.with_name(part.name + PART_OF)
wanted = f'sha256 {sha256.lower()}' if sha256 else url
if part.is_file():
try:
begun_for = note.read_text(encoding='utf-8').strip()
except (OSError, ValueError):
begun_for = ''
if begun_for and begun_for != wanted:
part.unlink(missing_ok=True)
if log:
log(f'{part.name} was begun for another file ({begun_for}), not this one ({wanted}): it is deleted '
'and the download starts over')
try:
note.write_text(wanted, encoding='utf-8')
except OSError: # without the note the part is resumed as it always was
pass
if size: if size:
full = space_problem(dest.parent, size - (part.stat().st_size if part.is_file() else 0), dest.name) full = space_problem(dest.parent, size - (part.stat().st_size if part.is_file() else 0), dest.name)
if full: if full:
@@ -363,4 +389,5 @@ def download(url, dest, sha256=None, size=None, progress=None, cancel=None, head
part.unlink(missing_ok=True) part.unlink(missing_ok=True)
raise DownloadError(f'{url}: sha256 {digest}, expected {sha256.lower()}') raise DownloadError(f'{url}: sha256 {digest}, expected {sha256.lower()}')
os.replace(part, dest) os.replace(part, dest)
note.unlink(missing_ok=True)
return dest return dest

View File

@@ -269,7 +269,7 @@ def install(layout, lock, progress=None, cancel=None, log=print, version='dev'):
try: try:
net.download(mirror + p['filename'], dest, sha256=p['sha256'], size=p['csize'], cancel=cancel, net.download(mirror + p['filename'], dest, sha256=p['sha256'], size=p['csize'], cancel=cancel,
progress=(lambda d, t, base=done_before: progress and progress('download', base + d, total)), progress=(lambda d, t, base=done_before: progress and progress('download', base + d, total)),
version=version) version=version, log=lambda line: log(f'toolchain: {line}'))
last_error = None last_error = None
break break
except net.NoSpace as e: # another server has the same file for the same full disk except net.NoSpace as e: # another server has the same file for the same full disk

View File

@@ -79,18 +79,18 @@ def safe_extract(zip_path, dest):
z.extractall(dest) z.extractall(dest)
def _fetch(release, key, layout, progress, cancel, version): def _fetch(release, key, layout, progress, cancel, version, log=None):
url, headers, size, sha = release.asset(key) url, headers, size, sha = release.asset(key)
entry = release.manifest[key] entry = release.manifest[key]
dest = layout.downloads / entry['name'] dest = layout.downloads / entry['name']
net.download(url, dest, sha256=sha, size=size, headers=headers, progress=progress, cancel=cancel, net.download(url, dest, sha256=sha, size=size, headers=headers, progress=progress, cancel=cancel,
version=version) version=version, log=log)
return dest return dest
def stage_app(layout, release, progress=None, cancel=None, version='dev'): def stage_app(layout, release, progress=None, cancel=None, version='dev', log=None):
"""Download, check and unpack the release's app archive into app/<version>; returns that folder.""" """Download, check and unpack the release's app archive into app/<version>; returns that folder."""
zip_path = _fetch(release, 'app', layout, progress, cancel, version) zip_path = _fetch(release, 'app', layout, progress, cancel, version, log)
ver = versions.display(release.version) ver = versions.display(release.version)
final = layout.apps / ver final = layout.apps / ver
part = layout.apps / f'{ver}.part' part = layout.apps / f'{ver}.part'
@@ -118,12 +118,12 @@ def runtime_needed(layout, release):
return bool(rt) and not layout.dev and rt.get('id') != runtime_id(layout) return bool(rt) and not layout.dev and rt.get('id') != runtime_id(layout)
def stage_runtime(layout, release, progress=None, cancel=None, version='dev', dest=None): def stage_runtime(layout, release, progress=None, cancel=None, version='dev', dest=None, log=None):
"""Unpack a new Python into dest (python.next, for the stub to swap in at the next start); False when the running """Unpack a new Python into dest (python.next, for the stub to swap in at the next start); False when the running
one is current.""" one is current."""
if not runtime_needed(layout, release): if not runtime_needed(layout, release):
return False return False
zip_path = _fetch(release, 'runtime', layout, progress, cancel, version) zip_path = _fetch(release, 'runtime', layout, progress, cancel, version, log)
nxt = Path(dest) if dest else layout.root / 'python.next' nxt = Path(dest) if dest else layout.root / 'python.next'
shutil.rmtree(nxt, ignore_errors=True) shutil.rmtree(nxt, ignore_errors=True)
part = nxt.with_name(nxt.name + '.part') part = nxt.with_name(nxt.name + '.part')
@@ -147,16 +147,16 @@ def prerelease(release):
return True return True
def stage(layout, release, progress=None, cancel=None, version='dev'): def stage(layout, release, progress=None, cancel=None, version='dev', log=None):
"""Download, check and unpack a release into app/<version>, with the Python it brings in python.staged there, """Download, check and unpack a release into app/<version>, with the Python it brings in python.staged there,
without switching to it; staged.json, written last, marks it complete. Earlier staged releases it supersedes are without switching to it; staged.json, written last, marks it complete. Earlier staged releases it supersedes are
removed. Returns the folder.""" removed. Returns the folder."""
ver = versions.display(release.version) ver = versions.display(release.version)
folder = stage_app(layout, release, progress=progress, cancel=cancel, version=version) folder = stage_app(layout, release, progress=progress, cancel=cancel, version=version, log=log)
runtime = '' runtime = ''
if runtime_needed(layout, release): if runtime_needed(layout, release):
stage_runtime(layout, release, progress=progress, cancel=cancel, version=version, stage_runtime(layout, release, progress=progress, cancel=cancel, version=version,
dest=folder / STAGED_PYTHON) dest=folder / STAGED_PYTHON, log=log)
runtime = release.manifest['runtime']['id'] runtime = release.manifest['runtime']['id']
write_text(folder / STAGED, json.dumps({'version': ver, 'runtime': runtime, 'prerelease': prerelease(release), write_text(folder / STAGED, json.dumps({'version': ver, 'runtime': runtime, 'prerelease': prerelease(release),
'notes': release.notes})) 'notes': release.notes}))
@@ -262,7 +262,7 @@ def prune(layout):
shutil.rmtree(d, ignore_errors=True) shutil.rmtree(d, ignore_errors=True)
if layout.downloads.is_dir(): if layout.downloads.is_dir():
for f in layout.downloads.iterdir(): for f in layout.downloads.iterdir():
if f.suffix in ('.part', '.zip'): if f.suffix in ('.part', '.zip', net.PART_OF):
f.unlink(missing_ok=True) f.unlink(missing_ok=True)

View File

@@ -693,6 +693,47 @@ class NetTest(Tmp):
for d, t in fetch({'Content-Length': '7'}, size=len(data)): for d, t in fetch({'Content-Length': '7'}, size=len(data)):
self.assertEqual(t, len(data)) self.assertEqual(t, len(data))
def test_a_part_file_of_another_download_is_not_resumed(self):
"""A part file is resumed only for the file it was begun for (f.bin.part.of names it by the expected sha256).
One begun for other bytes under the same name, as when a release is published again, is deleted with a log
line; resumed, it ended in a sha256 error that read as a damaged download. The same file from another
server and a part without a note are resumed."""
old, new = os.urandom(6000), os.urandom(6000)
(self.tmp / 'f.bin').write_bytes(new)
srv = Server(self.tmp)
self.addCleanup(srv.close)
dest = self.tmp / 'out' / 'f.bin'
dest.parent.mkdir()
part, note = dest.with_name('f.bin.part'), dest.with_name('f.bin.part.of')
part.write_bytes(old[:2000]) # cut while the old release downloaded
note.write_text(f'sha256 {sha(old)}')
lines = []
net.download(srv.url + 'f.bin', dest, sha256=sha(new), size=len(new), log=lines.append)
self.assertEqual(dest.read_bytes(), new)
self.assertIsNone(srv.hits[-1][1].get('Range')) # from the start
self.assertEqual(len(lines), 1)
self.assertIn('f.bin.part was begun for another file', lines[0])
self.assertFalse(part.exists() or note.exists()) # nothing stays beside the finished file
dest.unlink()
part.write_bytes(new[:2000]) # cut during this same file, on another server
note.write_text(f'sha256 {sha(new)}')
other = srv.url.replace('127.0.0.1', 'localhost')
net.download(other + 'f.bin', dest, sha256=sha(new), size=len(new), log=lines.append)
self.assertEqual(srv.hits[-1][1].get('Range'), 'bytes=2000-')
self.assertEqual((dest.read_bytes(), len(lines)), (new, 1))
dest.unlink()
part.write_bytes(new[:3000]) # left by a launcher that wrote no note
net.download(srv.url + 'f.bin', dest, sha256=sha(new), size=len(new), log=lines.append)
self.assertEqual(srv.hits[-1][1].get('Range'), 'bytes=3000-')
self.assertEqual(dest.read_bytes(), new)
with self.assertRaises(net.Cancelled): # a cut download says what its part is of
net.download(srv.url + 'f.bin', self.tmp / 'out' / 'c.bin', sha256=sha(new), cancel=lambda: True)
self.assertEqual((self.tmp / 'out' / 'c.bin.part.of').read_text(), f'sha256 {sha(new)}')
self.layout.downloads.mkdir(parents=True, exist_ok=True)
(self.layout.downloads / 'x.zip.part.of').write_text('sha256 0')
updater.prune(self.layout) # and a note left behind is cleared away
self.assertEqual(list(self.layout.downloads.iterdir()), [])
def low_disk(self, free): def low_disk(self, free):
"""shutil.disk_usage as net sees it, answering with free[0] bytes free.""" """shutil.disk_usage as net sees it, answering with free[0] bytes free."""
import collections import collections
@@ -716,14 +757,14 @@ class NetTest(Tmp):
self.assertEqual(net.space_problem(dest.parent, 1 << 20, 'x'), '') # 1 MB and the margin fit self.assertEqual(net.space_problem(dest.parent, 1 << 20, 'x'), '') # 1 MB and the margin fit
self.assertEqual(net.space_problem(self.tmp / 'not' / 'made' / 'yet', 1 << 20, 'x'), '') self.assertEqual(net.space_problem(self.tmp / 'not' / 'made' / 'yet', 1 << 20, 'x'), '')
with self.assertRaises(net.NoSpace) as refused: # the expected size is known: with self.assertRaises(net.NoSpace) as refused: # the expected size is known:
net.download(srv.url + 'f.bin', dest, size=64 << 20) # no request is made net.download(srv.url + 'f.bin', dest, sha256=sha(data), size=64 << 20) # no request is made
self.assertEqual(srv.hits, []) self.assertEqual(srv.hits, [])
self.assertIn('no room for f.bin on ', str(refused.exception)) self.assertIn('no room for f.bin on ', str(refused.exception))
self.assertIn('MB needed, 42 MB free', str(refused.exception)) self.assertIn('MB needed, 42 MB free', str(refused.exception))
self.assertEqual(part.read_bytes(), data[:1000]) self.assertEqual(part.read_bytes(), data[:1000])
free[0] = 1 << 20 free[0] = 1 << 20
with self.assertRaises(net.NoSpace): # else the server's length is with self.assertRaises(net.NoSpace): # else the server's length is
net.download(srv.url + 'f.bin', dest) net.download(srv.url + 'f.bin', dest, sha256=sha(data))
self.assertEqual(len(srv.hits), 1) self.assertEqual(len(srv.hits), 1)
self.assertEqual(part.read_bytes(), data[:1000]) # kept for when there is room self.assertEqual(part.read_bytes(), data[:1000]) # kept for when there is room
free[0] = 1 << 30 free[0] = 1 << 30