mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-17 13:39:58 -04:00
Retry a cut connection when the launcher reads a small file, as downloads already do.
net.get_bytes and get_json, which read the release feed, the package databases and the package index, gave up on the first network error, so one connection reset (WinError 10054) during Gather Build Tools ended an install (#237). A cut connection, a timeout or a server error (5xx, 429) is now tried again after 1 and 2 seconds; a refused certificate or another HTTP error still fails at once. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -194,9 +194,26 @@ def certificate_problem(url, refused, version='dev'):
|
||||
'may be replacing the certificate: let NX2navi through it or turn that scanning off, then try again.')
|
||||
|
||||
|
||||
def get_bytes(url, headers=None, timeout=30, version='dev', limit=64 << 20):
|
||||
with request(url, headers, timeout, version) as r:
|
||||
data = r.read(limit + 1)
|
||||
def get_bytes(url, headers=None, timeout=30, version='dev', limit=64 << 20, retries=3):
|
||||
"""The body of a GET of url, at most limit bytes. A cut connection, a timeout or a server error (5xx, 429) is tried
|
||||
again after 1, then 2 seconds, as download's are: one connection reset during Gather Build Tools (the package
|
||||
index read by toolchain.ensure_zbic) ended the install. A refused certificate or another HTTP error is not."""
|
||||
last_error = None
|
||||
for attempt in range(retries):
|
||||
try:
|
||||
with request(url, headers, timeout, version) as r:
|
||||
data = r.read(limit + 1)
|
||||
break
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code != 429 and e.code < 500:
|
||||
raise
|
||||
last_error = e
|
||||
except (urllib.error.URLError, OSError, TimeoutError) as e: # not DownloadError or CertificateError
|
||||
last_error = e
|
||||
if attempt + 1 < retries:
|
||||
time.sleep(min(2 ** attempt, 8))
|
||||
else:
|
||||
raise last_error
|
||||
if len(data) > limit:
|
||||
raise DownloadError(f'{url}: more than {limit >> 20} MB')
|
||||
return data
|
||||
|
||||
@@ -485,6 +485,41 @@ class ToolchainTest(Tmp):
|
||||
# ---------------------------------------------------------------- net
|
||||
|
||||
class NetTest(Tmp):
|
||||
def test_get_bytes_retries_a_cut_connection(self):
|
||||
"""A connection reset (WinError 10054 during Gather Build Tools) or a server error is tried again; a 404 is not."""
|
||||
import urllib.error
|
||||
calls = []
|
||||
|
||||
def flaky(url, headers=None, timeout=30, version='dev'):
|
||||
calls.append(url)
|
||||
if len(calls) == 1:
|
||||
raise ConnectionResetError(10054, 'An existing connection was forcibly closed by the remote host')
|
||||
if len(calls) == 2:
|
||||
raise urllib.error.HTTPError(url, 503, 'busy', {}, None)
|
||||
return io.BytesIO(b'{"ok": 1}')
|
||||
|
||||
with mock.patch.object(net, 'request', flaky), mock.patch.object(net.time, 'sleep') as slept:
|
||||
self.assertEqual(net.get_json('https://example.invalid/x.json'), {'ok': 1})
|
||||
self.assertEqual(len(calls), 3)
|
||||
self.assertEqual(slept.call_count, 2)
|
||||
|
||||
def missing(url, headers=None, timeout=30, version='dev'):
|
||||
calls.append(url)
|
||||
raise urllib.error.HTTPError(url, 404, 'not found', {}, None)
|
||||
|
||||
calls.clear()
|
||||
with mock.patch.object(net, 'request', missing), mock.patch.object(net.time, 'sleep'):
|
||||
with self.assertRaises(urllib.error.HTTPError):
|
||||
net.get_bytes('https://example.invalid/missing')
|
||||
self.assertEqual(len(calls), 1)
|
||||
|
||||
def down(url, headers=None, timeout=30, version='dev'):
|
||||
raise ConnectionResetError(10054, 'reset')
|
||||
|
||||
with mock.patch.object(net, 'request', down), mock.patch.object(net.time, 'sleep'):
|
||||
with self.assertRaises(ConnectionResetError): # every try failed: the last error goes up
|
||||
net.get_bytes('https://example.invalid/down')
|
||||
|
||||
def test_download_resume_and_checks(self):
|
||||
data = os.urandom(3 * net.CHUNK + 123)
|
||||
(self.tmp / 'f.bin').write_bytes(data)
|
||||
|
||||
Reference in New Issue
Block a user