Retry launcher downloads when the response ends early.

This commit is contained in:
DeepDaddyTTV
2026-11-09 21:28:11 -05:00
committed by chrg6
parent f19ed09f1c
commit da303e830d
2 changed files with 32 additions and 5 deletions

View File

@@ -26,6 +26,7 @@ import urllib.error
import urllib.parse
import urllib.request
from datetime import timezone
from http.client import IncompleteRead
from pathlib import Path
from . import NAME
@@ -194,6 +195,13 @@ def certificate_problem(url, refused, version='dev'):
'may be replacing the certificate: let NX2navi through it or turn that scanning off, then try again.')
def _check_eof(response):
# HTTPResponse.read(n) accepts an early EOF on a Content-Length body and leaves the missing byte count in length.
remaining = getattr(response, 'length', None)
if remaining:
raise ConnectionError(f'connection closed with {remaining} response bytes still expected')
def get_bytes(url, headers=None, timeout=30, version='dev', limit=64 << 20, retries=3):
"""The body of a GET of url, at most limit bytes. A cut connection, a timeout or a server error (5xx, 429) is tried
again after 1, then 2 seconds, as download's are: one connection reset during Gather Build Tools (the package
@@ -203,19 +211,20 @@ def get_bytes(url, headers=None, timeout=30, version='dev', limit=64 << 20, retr
try:
with request(url, headers, timeout, version) as r:
data = r.read(limit + 1)
if len(data) > limit:
raise DownloadError(f'{url}: more than {limit >> 20} MB')
_check_eof(r)
break
except urllib.error.HTTPError as e:
if e.code != 429 and e.code < 500:
raise
last_error = e
except (urllib.error.URLError, OSError, TimeoutError) as e: # not DownloadError or CertificateError
except (urllib.error.URLError, OSError, IncompleteRead) as e: # not DownloadError or CertificateError
last_error = e
if attempt + 1 < retries:
time.sleep(min(2 ** attempt, 8))
else:
raise last_error
if len(data) > limit:
raise DownloadError(f'{url}: more than {limit >> 20} MB')
return data
@@ -261,6 +270,7 @@ def download(url, dest, sha256=None, size=None, progress=None, cancel=None, head
raise Cancelled()
b = r.read(CHUNK)
if not b:
_check_eof(r)
break
f.write(b)
done += len(b)
@@ -282,7 +292,7 @@ def download(url, dest, sha256=None, size=None, progress=None, cancel=None, head
if e.code in (401, 403, 404, 410) or 300 <= e.code < 400: # 3xx: a redirect _Redirect refused
raise DownloadError(f'{url}: HTTP {e.code}') from e
last_error = e
except (urllib.error.URLError, OSError, TimeoutError) as e:
except (urllib.error.URLError, OSError, IncompleteRead) as e:
last_error = e
time.sleep(min(2 ** attempt, 16))
else:

View File

@@ -85,7 +85,8 @@ class Server:
self.send_response(status)
for k, v in headers.items():
self.send_header(k, v)
self.send_header('Content-Length', str(len(body)))
if 'Content-Length' not in headers:
self.send_header('Content-Length', str(len(body)))
self.end_headers()
self.wfile.write(body)
return
@@ -521,6 +522,22 @@ class NetTest(Tmp):
with self.assertRaises(ConnectionResetError): # every try failed: the last error goes up
net.get_bytes('https://example.invalid/down')
def test_early_eof_retries_metadata_and_resumes_file(self):
data = b'{"build":123}'
(self.tmp / 'asset').write_bytes(data)
cut = (200, {'Content-Length': str(len(data))}, data[:7])
routes = {'/asset': cut}
srv = Server(self.tmp, routes)
self.addCleanup(srv.close)
url = srv.url + 'asset'
with mock.patch.object(net.time, 'sleep', side_effect=lambda _: routes.clear()):
self.assertEqual(net.get_json(url), {'build': 123})
routes['/asset'] = cut
dest = self.tmp / 'download'
net.download(url, dest, sha256=sha(data), size=len(data))
self.assertEqual(dest.read_bytes(), data)
self.assertEqual([h.get('Range') for _, h in srv.hits], [None, None, None, 'bytes=7-'])
def test_download_resume_and_checks(self):
data = os.urandom(3 * net.CHUNK + 123)
(self.tmp / 'f.bin').write_bytes(data)