Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-07 10:14:46 -04:00
Code Issues Packages Projects Releases Wiki Activity

Cache downloads by etag during repeated building

Browse Source
This commit is contained in:
Kovid Goyal 2021-06-26 07:20:10 +05:30
parent 0885b45097
commit 3247692512
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
1 changed files with 27 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
setup/__init__.py
View File

@ -6,8 +6,18 @@ __license__ = 'GPL v3'
__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>' __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en' __docformat__ = 'restructuredtext en'
import sys, re, os, platform, subprocess, time, errno, tempfile, shutil import errno
import os
import platform
import re
import shutil
import subprocess
import sys
import tempfile
import time
import hashlib
from contextlib import contextmanager from contextlib import contextmanager
from functools import lru_cache
is64bit = platform.architecture()[0] == '64bit' is64bit = platform.architecture()[0] == '64bit'
iswindows = re.search('win(32|64)', sys.platform) iswindows = re.search('win(32|64)', sys.platform)
@ -53,11 +63,26 @@ def dump_json(obj, path, indent=4):
f.write(data) f.write(data)
@lru_cache
def curl_supports_etags():
return '--etag-compare' in subprocess.check_output(['curl', '--help', 'all']).decode('utf-8')
def download_securely(url): def download_securely(url):
# We use curl here as on some OSes (OS X) when bootstrapping calibre, # We use curl here as on some OSes (OS X) when bootstrapping calibre,
# python will be unable to validate certificates until after cacerts is # python will be unable to validate certificates until after cacerts is
# installed # installed
return subprocess.check_output(['curl', '-fsSL', url]) if not curl_supports_etags():
return subprocess.check_output(['curl', '-fsSL', url])
url_hash = hashlib.sha1(url.encode('utf-8')).hexdigest()
cache_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), '.cache', 'download', url_hash)
os.makedirs(cache_dir, exist_ok=True)
subprocess.check_call(
['curl', '-fsSL', '--etag-compare', 'etag.txt', '--etag-save', 'etag.txt', '-o', 'data.bin', url],
cwd=cache_dir
)
with open(os.path.join(cache_dir, 'data.bin'), 'rb') as f:
return f.read()
def build_cache_dir(): def build_cache_dir():