From da303e830d67871616f174c7cf31af81c616fc06 Mon Sep 17 00:00:00 2001 From: DeepDaddyTTV <32502468+DeepDaddyTTV@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:28:11 -0500 Subject: [PATCH] Retry launcher downloads when the response ends early. --- launcher/net.py | 18 ++++++++++++++---- tests/launcher_test.py | 19 ++++++++++++++++++- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/launcher/net.py b/launcher/net.py index fb1a71ed..064f6a50 100644 --- a/launcher/net.py +++ b/launcher/net.py @@ -26,6 +26,7 @@ import urllib.error import urllib.parse import urllib.request from datetime import timezone +from http.client import IncompleteRead from pathlib import Path from . import NAME @@ -194,6 +195,13 @@ def certificate_problem(url, refused, version='dev'): 'may be replacing the certificate: let NX2navi through it or turn that scanning off, then try again.') +def _check_eof(response): + # HTTPResponse.read(n) accepts an early EOF on a Content-Length body and leaves the missing byte count in length. + remaining = getattr(response, 'length', None) + if remaining: + raise ConnectionError(f'connection closed with {remaining} response bytes still expected') + + def get_bytes(url, headers=None, timeout=30, version='dev', limit=64 << 20, retries=3): """The body of a GET of url, at most limit bytes. A cut connection, a timeout or a server error (5xx, 429) is tried again after 1, then 2 seconds, as download's are: one connection reset during Gather Build Tools (the package @@ -203,19 +211,20 @@ def get_bytes(url, headers=None, timeout=30, version='dev', limit=64 << 20, retr try: with request(url, headers, timeout, version) as r: data = r.read(limit + 1) + if len(data) > limit: + raise DownloadError(f'{url}: more than {limit >> 20} MB') + _check_eof(r) break except urllib.error.HTTPError as e: if e.code != 429 and e.code < 500: raise last_error = e - except (urllib.error.URLError, OSError, TimeoutError) as e: # not DownloadError or CertificateError + except (urllib.error.URLError, OSError, IncompleteRead) as e: # not DownloadError or CertificateError last_error = e if attempt + 1 < retries: time.sleep(min(2 ** attempt, 8)) else: raise last_error - if len(data) > limit: - raise DownloadError(f'{url}: more than {limit >> 20} MB') return data @@ -261,6 +270,7 @@ def download(url, dest, sha256=None, size=None, progress=None, cancel=None, head raise Cancelled() b = r.read(CHUNK) if not b: + _check_eof(r) break f.write(b) done += len(b) @@ -282,7 +292,7 @@ def download(url, dest, sha256=None, size=None, progress=None, cancel=None, head if e.code in (401, 403, 404, 410) or 300 <= e.code < 400: # 3xx: a redirect _Redirect refused raise DownloadError(f'{url}: HTTP {e.code}') from e last_error = e - except (urllib.error.URLError, OSError, TimeoutError) as e: + except (urllib.error.URLError, OSError, IncompleteRead) as e: last_error = e time.sleep(min(2 ** attempt, 16)) else: diff --git a/tests/launcher_test.py b/tests/launcher_test.py index 89af1da8..679588d3 100644 --- a/tests/launcher_test.py +++ b/tests/launcher_test.py @@ -85,7 +85,8 @@ class Server: self.send_response(status) for k, v in headers.items(): self.send_header(k, v) - self.send_header('Content-Length', str(len(body))) + if 'Content-Length' not in headers: + self.send_header('Content-Length', str(len(body))) self.end_headers() self.wfile.write(body) return @@ -521,6 +522,22 @@ class NetTest(Tmp): with self.assertRaises(ConnectionResetError): # every try failed: the last error goes up net.get_bytes('https://example.invalid/down') + def test_early_eof_retries_metadata_and_resumes_file(self): + data = b'{"build":123}' + (self.tmp / 'asset').write_bytes(data) + cut = (200, {'Content-Length': str(len(data))}, data[:7]) + routes = {'/asset': cut} + srv = Server(self.tmp, routes) + self.addCleanup(srv.close) + url = srv.url + 'asset' + with mock.patch.object(net.time, 'sleep', side_effect=lambda _: routes.clear()): + self.assertEqual(net.get_json(url), {'build': 123}) + routes['/asset'] = cut + dest = self.tmp / 'download' + net.download(url, dest, sha256=sha(data), size=len(data)) + self.assertEqual(dest.read_bytes(), data) + self.assertEqual([h.get('Range') for _, h in srv.hits], [None, None, None, 'bytes=7-']) + def test_download_resume_and_checks(self): data = os.urandom(3 * net.CHUNK + 123) (self.tmp / 'f.bin').write_bytes(data)