Also retry downloads used during bootstrap

setup/__init__.py

@@ -70,14 +70,15 @@ def curl_supports_etags():
     return '--etag-compare' in subprocess.check_output(['curl', '--help', 'all']).decode('utf-8')
 
 
-def download_securely(url):
+def _download_securely(url):
     # We use curl here as on some OSes (OS X) when bootstrapping calibre,
     # python will be unable to validate certificates until after cacerts is
     # installed
     if is_ci and iswindows:
         # curl is failing for wikipedia urls on CI (used for browser_data)
         from urllib.request import urlopen
-        return urlopen(url).read()
+        with urlopen(url) as f:
+            return f.read()
     if not curl_supports_etags():
         return subprocess.check_output(['curl', '-fsSL', url])
     url_hash = hashlib.sha1(url.encode('utf-8')).hexdigest()
@@ -91,6 +92,17 @@ def download_securely(url):
         return f.read()
 
 
+def download_securely(url, retry_count: int = 5 if is_ci else 3, sleep_time: float = 1):
+    for i in range(retry_count):
+        try:
+            return _download_securely(url)
+        except Exception as err:
+            if i >= retry_count - 1:
+                raise
+            print(f'Download of {url} failed with error {err}, retrying...', file=sys.stderr)
+            time.sleep(sleep_time)
+
+
 def build_cache_dir():
     global _cache_dir_built
     ans = os.path.join(os.path.dirname(SRC), '.build-cache')

setup/unix-ci.py

@@ -27,7 +27,8 @@ def download_with_retry(url, count=5):
         count -= 1
         try:
             print('Downloading', url, flush=True)
-            return urlopen(url).read()
+            with urlopen(url) as f:
+                return f.read()
         except Exception:
             if count <= 0:
                 raise

setup/win-ci.py

@@ -21,7 +21,8 @@ def download_with_retry(url, count=5):
         count -= 1
         try:
             printf('Downloading', url)
-            return urlopen(url).read()
+            with urlopen(url) as f:
+                return f.read()
         except Exception:
             if count <= 0:
                 raise