mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-17 11:29:58 -04:00
Retry launcher downloads when the response ends early.
This commit is contained in:
@@ -26,6 +26,7 @@ import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from datetime import timezone
|
||||
from http.client import IncompleteRead
|
||||
from pathlib import Path
|
||||
|
||||
from . import NAME
|
||||
@@ -194,6 +195,13 @@ def certificate_problem(url, refused, version='dev'):
|
||||
'may be replacing the certificate: let NX2navi through it or turn that scanning off, then try again.')
|
||||
|
||||
|
||||
def _check_eof(response):
|
||||
# HTTPResponse.read(n) accepts an early EOF on a Content-Length body and leaves the missing byte count in length.
|
||||
remaining = getattr(response, 'length', None)
|
||||
if remaining:
|
||||
raise ConnectionError(f'connection closed with {remaining} response bytes still expected')
|
||||
|
||||
|
||||
def get_bytes(url, headers=None, timeout=30, version='dev', limit=64 << 20, retries=3):
|
||||
"""The body of a GET of url, at most limit bytes. A cut connection, a timeout or a server error (5xx, 429) is tried
|
||||
again after 1, then 2 seconds, as download's are: one connection reset during Gather Build Tools (the package
|
||||
@@ -203,19 +211,20 @@ def get_bytes(url, headers=None, timeout=30, version='dev', limit=64 << 20, retr
|
||||
try:
|
||||
with request(url, headers, timeout, version) as r:
|
||||
data = r.read(limit + 1)
|
||||
if len(data) > limit:
|
||||
raise DownloadError(f'{url}: more than {limit >> 20} MB')
|
||||
_check_eof(r)
|
||||
break
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code != 429 and e.code < 500:
|
||||
raise
|
||||
last_error = e
|
||||
except (urllib.error.URLError, OSError, TimeoutError) as e: # not DownloadError or CertificateError
|
||||
except (urllib.error.URLError, OSError, IncompleteRead) as e: # not DownloadError or CertificateError
|
||||
last_error = e
|
||||
if attempt + 1 < retries:
|
||||
time.sleep(min(2 ** attempt, 8))
|
||||
else:
|
||||
raise last_error
|
||||
if len(data) > limit:
|
||||
raise DownloadError(f'{url}: more than {limit >> 20} MB')
|
||||
return data
|
||||
|
||||
|
||||
@@ -261,6 +270,7 @@ def download(url, dest, sha256=None, size=None, progress=None, cancel=None, head
|
||||
raise Cancelled()
|
||||
b = r.read(CHUNK)
|
||||
if not b:
|
||||
_check_eof(r)
|
||||
break
|
||||
f.write(b)
|
||||
done += len(b)
|
||||
@@ -282,7 +292,7 @@ def download(url, dest, sha256=None, size=None, progress=None, cancel=None, head
|
||||
if e.code in (401, 403, 404, 410) or 300 <= e.code < 400: # 3xx: a redirect _Redirect refused
|
||||
raise DownloadError(f'{url}: HTTP {e.code}') from e
|
||||
last_error = e
|
||||
except (urllib.error.URLError, OSError, TimeoutError) as e:
|
||||
except (urllib.error.URLError, OSError, IncompleteRead) as e:
|
||||
last_error = e
|
||||
time.sleep(min(2 ** attempt, 16))
|
||||
else:
|
||||
|
||||
@@ -85,7 +85,8 @@ class Server:
|
||||
self.send_response(status)
|
||||
for k, v in headers.items():
|
||||
self.send_header(k, v)
|
||||
self.send_header('Content-Length', str(len(body)))
|
||||
if 'Content-Length' not in headers:
|
||||
self.send_header('Content-Length', str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
return
|
||||
@@ -521,6 +522,22 @@ class NetTest(Tmp):
|
||||
with self.assertRaises(ConnectionResetError): # every try failed: the last error goes up
|
||||
net.get_bytes('https://example.invalid/down')
|
||||
|
||||
def test_early_eof_retries_metadata_and_resumes_file(self):
|
||||
data = b'{"build":123}'
|
||||
(self.tmp / 'asset').write_bytes(data)
|
||||
cut = (200, {'Content-Length': str(len(data))}, data[:7])
|
||||
routes = {'/asset': cut}
|
||||
srv = Server(self.tmp, routes)
|
||||
self.addCleanup(srv.close)
|
||||
url = srv.url + 'asset'
|
||||
with mock.patch.object(net.time, 'sleep', side_effect=lambda _: routes.clear()):
|
||||
self.assertEqual(net.get_json(url), {'build': 123})
|
||||
routes['/asset'] = cut
|
||||
dest = self.tmp / 'download'
|
||||
net.download(url, dest, sha256=sha(data), size=len(data))
|
||||
self.assertEqual(dest.read_bytes(), data)
|
||||
self.assertEqual([h.get('Range') for _, h in srv.hits], [None, None, None, 'bytes=7-'])
|
||||
|
||||
def test_download_resume_and_checks(self):
|
||||
data = os.urandom(3 * net.CHUNK + 123)
|
||||
(self.tmp / 'f.bin').write_bytes(data)
|
||||
|
||||
Reference in New Issue
Block a user