mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-21 10:09:59 -04:00
Retry launcher downloads when the response ends early.
This commit is contained in:
@@ -26,6 +26,7 @@ import urllib.error
|
|||||||
import urllib.parse
|
import urllib.parse
|
||||||
import urllib.request
|
import urllib.request
|
||||||
from datetime import timezone
|
from datetime import timezone
|
||||||
|
from http.client import IncompleteRead
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from . import NAME
|
from . import NAME
|
||||||
@@ -194,6 +195,13 @@ def certificate_problem(url, refused, version='dev'):
|
|||||||
'may be replacing the certificate: let NX2navi through it or turn that scanning off, then try again.')
|
'may be replacing the certificate: let NX2navi through it or turn that scanning off, then try again.')
|
||||||
|
|
||||||
|
|
||||||
|
def _check_eof(response):
|
||||||
|
# HTTPResponse.read(n) accepts an early EOF on a Content-Length body and leaves the missing byte count in length.
|
||||||
|
remaining = getattr(response, 'length', None)
|
||||||
|
if remaining:
|
||||||
|
raise ConnectionError(f'connection closed with {remaining} response bytes still expected')
|
||||||
|
|
||||||
|
|
||||||
def get_bytes(url, headers=None, timeout=30, version='dev', limit=64 << 20, retries=3):
|
def get_bytes(url, headers=None, timeout=30, version='dev', limit=64 << 20, retries=3):
|
||||||
"""The body of a GET of url, at most limit bytes. A cut connection, a timeout or a server error (5xx, 429) is tried
|
"""The body of a GET of url, at most limit bytes. A cut connection, a timeout or a server error (5xx, 429) is tried
|
||||||
again after 1, then 2 seconds, as download's are: one connection reset during Gather Build Tools (the package
|
again after 1, then 2 seconds, as download's are: one connection reset during Gather Build Tools (the package
|
||||||
@@ -203,19 +211,20 @@ def get_bytes(url, headers=None, timeout=30, version='dev', limit=64 << 20, retr
|
|||||||
try:
|
try:
|
||||||
with request(url, headers, timeout, version) as r:
|
with request(url, headers, timeout, version) as r:
|
||||||
data = r.read(limit + 1)
|
data = r.read(limit + 1)
|
||||||
|
if len(data) > limit:
|
||||||
|
raise DownloadError(f'{url}: more than {limit >> 20} MB')
|
||||||
|
_check_eof(r)
|
||||||
break
|
break
|
||||||
except urllib.error.HTTPError as e:
|
except urllib.error.HTTPError as e:
|
||||||
if e.code != 429 and e.code < 500:
|
if e.code != 429 and e.code < 500:
|
||||||
raise
|
raise
|
||||||
last_error = e
|
last_error = e
|
||||||
except (urllib.error.URLError, OSError, TimeoutError) as e: # not DownloadError or CertificateError
|
except (urllib.error.URLError, OSError, IncompleteRead) as e: # not DownloadError or CertificateError
|
||||||
last_error = e
|
last_error = e
|
||||||
if attempt + 1 < retries:
|
if attempt + 1 < retries:
|
||||||
time.sleep(min(2 ** attempt, 8))
|
time.sleep(min(2 ** attempt, 8))
|
||||||
else:
|
else:
|
||||||
raise last_error
|
raise last_error
|
||||||
if len(data) > limit:
|
|
||||||
raise DownloadError(f'{url}: more than {limit >> 20} MB')
|
|
||||||
return data
|
return data
|
||||||
|
|
||||||
|
|
||||||
@@ -261,6 +270,7 @@ def download(url, dest, sha256=None, size=None, progress=None, cancel=None, head
|
|||||||
raise Cancelled()
|
raise Cancelled()
|
||||||
b = r.read(CHUNK)
|
b = r.read(CHUNK)
|
||||||
if not b:
|
if not b:
|
||||||
|
_check_eof(r)
|
||||||
break
|
break
|
||||||
f.write(b)
|
f.write(b)
|
||||||
done += len(b)
|
done += len(b)
|
||||||
@@ -282,7 +292,7 @@ def download(url, dest, sha256=None, size=None, progress=None, cancel=None, head
|
|||||||
if e.code in (401, 403, 404, 410) or 300 <= e.code < 400: # 3xx: a redirect _Redirect refused
|
if e.code in (401, 403, 404, 410) or 300 <= e.code < 400: # 3xx: a redirect _Redirect refused
|
||||||
raise DownloadError(f'{url}: HTTP {e.code}') from e
|
raise DownloadError(f'{url}: HTTP {e.code}') from e
|
||||||
last_error = e
|
last_error = e
|
||||||
except (urllib.error.URLError, OSError, TimeoutError) as e:
|
except (urllib.error.URLError, OSError, IncompleteRead) as e:
|
||||||
last_error = e
|
last_error = e
|
||||||
time.sleep(min(2 ** attempt, 16))
|
time.sleep(min(2 ** attempt, 16))
|
||||||
else:
|
else:
|
||||||
|
|||||||
@@ -85,6 +85,7 @@ class Server:
|
|||||||
self.send_response(status)
|
self.send_response(status)
|
||||||
for k, v in headers.items():
|
for k, v in headers.items():
|
||||||
self.send_header(k, v)
|
self.send_header(k, v)
|
||||||
|
if 'Content-Length' not in headers:
|
||||||
self.send_header('Content-Length', str(len(body)))
|
self.send_header('Content-Length', str(len(body)))
|
||||||
self.end_headers()
|
self.end_headers()
|
||||||
self.wfile.write(body)
|
self.wfile.write(body)
|
||||||
@@ -521,6 +522,22 @@ class NetTest(Tmp):
|
|||||||
with self.assertRaises(ConnectionResetError): # every try failed: the last error goes up
|
with self.assertRaises(ConnectionResetError): # every try failed: the last error goes up
|
||||||
net.get_bytes('https://example.invalid/down')
|
net.get_bytes('https://example.invalid/down')
|
||||||
|
|
||||||
|
def test_early_eof_retries_metadata_and_resumes_file(self):
|
||||||
|
data = b'{"build":123}'
|
||||||
|
(self.tmp / 'asset').write_bytes(data)
|
||||||
|
cut = (200, {'Content-Length': str(len(data))}, data[:7])
|
||||||
|
routes = {'/asset': cut}
|
||||||
|
srv = Server(self.tmp, routes)
|
||||||
|
self.addCleanup(srv.close)
|
||||||
|
url = srv.url + 'asset'
|
||||||
|
with mock.patch.object(net.time, 'sleep', side_effect=lambda _: routes.clear()):
|
||||||
|
self.assertEqual(net.get_json(url), {'build': 123})
|
||||||
|
routes['/asset'] = cut
|
||||||
|
dest = self.tmp / 'download'
|
||||||
|
net.download(url, dest, sha256=sha(data), size=len(data))
|
||||||
|
self.assertEqual(dest.read_bytes(), data)
|
||||||
|
self.assertEqual([h.get('Range') for _, h in srv.hits], [None, None, None, 'bytes=7-'])
|
||||||
|
|
||||||
def test_download_resume_and_checks(self):
|
def test_download_resume_and_checks(self):
|
||||||
data = os.urandom(3 * net.CHUNK + 123)
|
data = os.urandom(3 * net.CHUNK + 123)
|
||||||
(self.tmp / 'f.bin').write_bytes(data)
|
(self.tmp / 'f.bin').write_bytes(data)
|
||||||
|
|||||||
Reference in New Issue
Block a user