diff --git a/launcher/net.py b/launcher/net.py index 5c228148..fb1a71ed 100644 --- a/launcher/net.py +++ b/launcher/net.py @@ -194,9 +194,26 @@ def certificate_problem(url, refused, version='dev'): 'may be replacing the certificate: let NX2navi through it or turn that scanning off, then try again.') -def get_bytes(url, headers=None, timeout=30, version='dev', limit=64 << 20): - with request(url, headers, timeout, version) as r: - data = r.read(limit + 1) +def get_bytes(url, headers=None, timeout=30, version='dev', limit=64 << 20, retries=3): + """The body of a GET of url, at most limit bytes. A cut connection, a timeout or a server error (5xx, 429) is tried + again after 1, then 2 seconds, as download's are: one connection reset during Gather Build Tools (the package + index read by toolchain.ensure_zbic) ended the install. A refused certificate or another HTTP error is not.""" + last_error = None + for attempt in range(retries): + try: + with request(url, headers, timeout, version) as r: + data = r.read(limit + 1) + break + except urllib.error.HTTPError as e: + if e.code != 429 and e.code < 500: + raise + last_error = e + except (urllib.error.URLError, OSError, TimeoutError) as e: # not DownloadError or CertificateError + last_error = e + if attempt + 1 < retries: + time.sleep(min(2 ** attempt, 8)) + else: + raise last_error if len(data) > limit: raise DownloadError(f'{url}: more than {limit >> 20} MB') return data diff --git a/tests/launcher_test.py b/tests/launcher_test.py index 0e9d06b8..8ad14529 100644 --- a/tests/launcher_test.py +++ b/tests/launcher_test.py @@ -485,6 +485,41 @@ class ToolchainTest(Tmp): # ---------------------------------------------------------------- net class NetTest(Tmp): + def test_get_bytes_retries_a_cut_connection(self): + """A connection reset (WinError 10054 during Gather Build Tools) or a server error is tried again; a 404 is not.""" + import urllib.error + calls = [] + + def flaky(url, headers=None, timeout=30, version='dev'): + calls.append(url) + if len(calls) == 1: + raise ConnectionResetError(10054, 'An existing connection was forcibly closed by the remote host') + if len(calls) == 2: + raise urllib.error.HTTPError(url, 503, 'busy', {}, None) + return io.BytesIO(b'{"ok": 1}') + + with mock.patch.object(net, 'request', flaky), mock.patch.object(net.time, 'sleep') as slept: + self.assertEqual(net.get_json('https://example.invalid/x.json'), {'ok': 1}) + self.assertEqual(len(calls), 3) + self.assertEqual(slept.call_count, 2) + + def missing(url, headers=None, timeout=30, version='dev'): + calls.append(url) + raise urllib.error.HTTPError(url, 404, 'not found', {}, None) + + calls.clear() + with mock.patch.object(net, 'request', missing), mock.patch.object(net.time, 'sleep'): + with self.assertRaises(urllib.error.HTTPError): + net.get_bytes('https://example.invalid/missing') + self.assertEqual(len(calls), 1) + + def down(url, headers=None, timeout=30, version='dev'): + raise ConnectionResetError(10054, 'reset') + + with mock.patch.object(net, 'request', down), mock.patch.object(net.time, 'sleep'): + with self.assertRaises(ConnectionResetError): # every try failed: the last error goes up + net.get_bytes('https://example.invalid/down') + def test_download_resume_and_checks(self): data = os.urandom(3 * net.CHUNK + 123) (self.tmp / 'f.bin').write_bytes(data)