Retry launcher downloads when the response ends early.

This commit is contained in:
DeepDaddyTTV
2026-11-09 21:28:11 -05:00
committed by chrg6
parent f19ed09f1c
commit da303e830d
2 changed files with 32 additions and 5 deletions

View File

@@ -26,6 +26,7 @@ import urllib.error
import urllib.parse import urllib.parse
import urllib.request import urllib.request
from datetime import timezone from datetime import timezone
from http.client import IncompleteRead
from pathlib import Path from pathlib import Path
from . import NAME from . import NAME
@@ -194,6 +195,13 @@ def certificate_problem(url, refused, version='dev'):
'may be replacing the certificate: let NX2navi through it or turn that scanning off, then try again.') 'may be replacing the certificate: let NX2navi through it or turn that scanning off, then try again.')
def _check_eof(response):
# HTTPResponse.read(n) accepts an early EOF on a Content-Length body and leaves the missing byte count in length.
remaining = getattr(response, 'length', None)
if remaining:
raise ConnectionError(f'connection closed with {remaining} response bytes still expected')
def get_bytes(url, headers=None, timeout=30, version='dev', limit=64 << 20, retries=3): def get_bytes(url, headers=None, timeout=30, version='dev', limit=64 << 20, retries=3):
"""The body of a GET of url, at most limit bytes. A cut connection, a timeout or a server error (5xx, 429) is tried """The body of a GET of url, at most limit bytes. A cut connection, a timeout or a server error (5xx, 429) is tried
again after 1, then 2 seconds, as download's are: one connection reset during Gather Build Tools (the package again after 1, then 2 seconds, as download's are: one connection reset during Gather Build Tools (the package
@@ -203,19 +211,20 @@ def get_bytes(url, headers=None, timeout=30, version='dev', limit=64 << 20, retr
try: try:
with request(url, headers, timeout, version) as r: with request(url, headers, timeout, version) as r:
data = r.read(limit + 1) data = r.read(limit + 1)
if len(data) > limit:
raise DownloadError(f'{url}: more than {limit >> 20} MB')
_check_eof(r)
break break
except urllib.error.HTTPError as e: except urllib.error.HTTPError as e:
if e.code != 429 and e.code < 500: if e.code != 429 and e.code < 500:
raise raise
last_error = e last_error = e
except (urllib.error.URLError, OSError, TimeoutError) as e: # not DownloadError or CertificateError except (urllib.error.URLError, OSError, IncompleteRead) as e: # not DownloadError or CertificateError
last_error = e last_error = e
if attempt + 1 < retries: if attempt + 1 < retries:
time.sleep(min(2 ** attempt, 8)) time.sleep(min(2 ** attempt, 8))
else: else:
raise last_error raise last_error
if len(data) > limit:
raise DownloadError(f'{url}: more than {limit >> 20} MB')
return data return data
@@ -261,6 +270,7 @@ def download(url, dest, sha256=None, size=None, progress=None, cancel=None, head
raise Cancelled() raise Cancelled()
b = r.read(CHUNK) b = r.read(CHUNK)
if not b: if not b:
_check_eof(r)
break break
f.write(b) f.write(b)
done += len(b) done += len(b)
@@ -282,7 +292,7 @@ def download(url, dest, sha256=None, size=None, progress=None, cancel=None, head
if e.code in (401, 403, 404, 410) or 300 <= e.code < 400: # 3xx: a redirect _Redirect refused if e.code in (401, 403, 404, 410) or 300 <= e.code < 400: # 3xx: a redirect _Redirect refused
raise DownloadError(f'{url}: HTTP {e.code}') from e raise DownloadError(f'{url}: HTTP {e.code}') from e
last_error = e last_error = e
except (urllib.error.URLError, OSError, TimeoutError) as e: except (urllib.error.URLError, OSError, IncompleteRead) as e:
last_error = e last_error = e
time.sleep(min(2 ** attempt, 16)) time.sleep(min(2 ** attempt, 16))
else: else:

View File

@@ -85,6 +85,7 @@ class Server:
self.send_response(status) self.send_response(status)
for k, v in headers.items(): for k, v in headers.items():
self.send_header(k, v) self.send_header(k, v)
if 'Content-Length' not in headers:
self.send_header('Content-Length', str(len(body))) self.send_header('Content-Length', str(len(body)))
self.end_headers() self.end_headers()
self.wfile.write(body) self.wfile.write(body)
@@ -521,6 +522,22 @@ class NetTest(Tmp):
with self.assertRaises(ConnectionResetError): # every try failed: the last error goes up with self.assertRaises(ConnectionResetError): # every try failed: the last error goes up
net.get_bytes('https://example.invalid/down') net.get_bytes('https://example.invalid/down')
def test_early_eof_retries_metadata_and_resumes_file(self):
data = b'{"build":123}'
(self.tmp / 'asset').write_bytes(data)
cut = (200, {'Content-Length': str(len(data))}, data[:7])
routes = {'/asset': cut}
srv = Server(self.tmp, routes)
self.addCleanup(srv.close)
url = srv.url + 'asset'
with mock.patch.object(net.time, 'sleep', side_effect=lambda _: routes.clear()):
self.assertEqual(net.get_json(url), {'build': 123})
routes['/asset'] = cut
dest = self.tmp / 'download'
net.download(url, dest, sha256=sha(data), size=len(data))
self.assertEqual(dest.read_bytes(), data)
self.assertEqual([h.get('Range') for _, h in srv.hits], [None, None, None, 'bytes=7-'])
def test_download_resume_and_checks(self): def test_download_resume_and_checks(self):
data = os.urandom(3 * net.CHUNK + 123) data = os.urandom(3 * net.CHUNK + 123)
(self.tmp / 'f.bin').write_bytes(data) (self.tmp / 'f.bin').write_bytes(data)