Retry a cut connection when the launcher reads a small file, as downloads already do.

net.get_bytes and get_json, which read the release feed, the package databases and the package index, gave up on the
first network error, so one connection reset (WinError 10054) during Gather Build Tools ended an install (#237). A cut
connection, a timeout or a server error (5xx, 429) is now tried again after 1 and 2 seconds; a refused certificate or
another HTTP error still fails at once.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
chrg6
2026-11-09 10:57:10 -05:00
parent 21eb3de1c2
commit 0f1847d243
2 changed files with 55 additions and 3 deletions

View File

@@ -194,9 +194,26 @@ def certificate_problem(url, refused, version='dev'):
'may be replacing the certificate: let NX2navi through it or turn that scanning off, then try again.')
def get_bytes(url, headers=None, timeout=30, version='dev', limit=64 << 20):
with request(url, headers, timeout, version) as r:
data = r.read(limit + 1)
def get_bytes(url, headers=None, timeout=30, version='dev', limit=64 << 20, retries=3):
"""The body of a GET of url, at most limit bytes. A cut connection, a timeout or a server error (5xx, 429) is tried
again after 1, then 2 seconds, as download's are: one connection reset during Gather Build Tools (the package
index read by toolchain.ensure_zbic) ended the install. A refused certificate or another HTTP error is not."""
last_error = None
for attempt in range(retries):
try:
with request(url, headers, timeout, version) as r:
data = r.read(limit + 1)
break
except urllib.error.HTTPError as e:
if e.code != 429 and e.code < 500:
raise
last_error = e
except (urllib.error.URLError, OSError, TimeoutError) as e: # not DownloadError or CertificateError
last_error = e
if attempt + 1 < retries:
time.sleep(min(2 ** attempt, 8))
else:
raise last_error
if len(data) > limit:
raise DownloadError(f'{url}: more than {limit >> 20} MB')
return data

View File

@@ -485,6 +485,41 @@ class ToolchainTest(Tmp):
# ---------------------------------------------------------------- net
class NetTest(Tmp):
def test_get_bytes_retries_a_cut_connection(self):
"""A connection reset (WinError 10054 during Gather Build Tools) or a server error is tried again; a 404 is not."""
import urllib.error
calls = []
def flaky(url, headers=None, timeout=30, version='dev'):
calls.append(url)
if len(calls) == 1:
raise ConnectionResetError(10054, 'An existing connection was forcibly closed by the remote host')
if len(calls) == 2:
raise urllib.error.HTTPError(url, 503, 'busy', {}, None)
return io.BytesIO(b'{"ok": 1}')
with mock.patch.object(net, 'request', flaky), mock.patch.object(net.time, 'sleep') as slept:
self.assertEqual(net.get_json('https://example.invalid/x.json'), {'ok': 1})
self.assertEqual(len(calls), 3)
self.assertEqual(slept.call_count, 2)
def missing(url, headers=None, timeout=30, version='dev'):
calls.append(url)
raise urllib.error.HTTPError(url, 404, 'not found', {}, None)
calls.clear()
with mock.patch.object(net, 'request', missing), mock.patch.object(net.time, 'sleep'):
with self.assertRaises(urllib.error.HTTPError):
net.get_bytes('https://example.invalid/missing')
self.assertEqual(len(calls), 1)
def down(url, headers=None, timeout=30, version='dev'):
raise ConnectionResetError(10054, 'reset')
with mock.patch.object(net, 'request', down), mock.patch.object(net.time, 'sleep'):
with self.assertRaises(ConnectionResetError): # every try failed: the last error goes up
net.get_bytes('https://example.invalid/down')
def test_download_resume_and_checks(self):
data = os.urandom(3 * net.CHUNK + 123)
(self.tmp / 'f.bin').write_bytes(data)