diff --git a/setup/commands.py b/setup/commands.py index 266b4a1dcc..6b3ec6e4da 100644 --- a/setup/commands.py +++ b/setup/commands.py @@ -6,7 +6,7 @@ __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' __all__ = [ - 'pot', 'translations', 'get_translations', 'iso639', 'iso3166', + 'pot', 'translations', 'get_translations', 'iso_data', 'iso639', 'iso3166', 'build', 'mathjax', 'man_pages', 'gui', 'git_version', @@ -33,6 +33,7 @@ build_dep = BuildDep() export_packages = ExportPackages() from setup.translations import ISO639, ISO3166, POT, GetTranslations, Translations +from setup.iso_codes import iso_data pot = POT() translations = Translations() diff --git a/setup/install.py b/setup/install.py index a9c3a40430..2074abb337 100644 --- a/setup/install.py +++ b/setup/install.py @@ -355,17 +355,32 @@ class Bootstrap(Command): description = 'Bootstrap a fresh checkout of calibre from git to a state where it can be installed. Requires various development tools/libraries/headers' TRANSLATIONS_REPO = 'kovidgoyal/calibre-translations' - sub_commands = 'build iso639 iso3166 translations gui resources cacerts recent_uas'.split() + sub_commands = 'build resources iso639 iso3166 translations gui cacerts recent_uas'.split() def add_options(self, parser): parser.add_option('--ephemeral', default=False, action='store_true', help='Do not download all history for the translations. Speeds up first time download but subsequent downloads will be slower.') + parser.add_option('--path-to-translations', help='Path to sources of translations') def pre_sub_commands(self, opts): tdir = self.j(self.d(self.SRC), 'translations') clone_cmd = [ 'git', 'clone', f'https://github.com/{self.TRANSLATIONS_REPO}.git', 'translations'] - if opts.ephemeral: + if opts.path_to_translations: + if os.path.exists(tdir): + shutil.rmtree(tdir) + shutil.copytree(opts.path_to_translations, tdir) + # Change permissions for the top-level folder + os.chmod(tdir, 0o755) + for root, dirs, files in os.walk(tdir): + # set perms on sub-directories + for momo in dirs: + os.chmod(os.path.join(root, momo), 0o755) + # set perms on files + for momo in files: + os.chmod(os.path.join(root, momo), 0o644) + + elif opts.ephemeral: if os.path.exists(tdir): shutil.rmtree(tdir) diff --git a/setup/iso_codes.py b/setup/iso_codes.py index 66d196357f..b50e8497db 100644 --- a/setup/iso_codes.py +++ b/setup/iso_codes.py @@ -8,29 +8,45 @@ import time import zipfile from io import BytesIO -from setup import download_securely +from setup import download_securely, Command -class ISOData: - URL = 'https://salsa.debian.org/iso-codes-team/iso-codes/-/archive/main/iso-codes-main.zip' +class ISOData(Command): + description = 'Get iso-codes data' + URL = f'https://salsa.debian.org/iso-codes-team/iso-codes/-/archive/main/iso-codes-main.zip' + + def add_options(self, parser): + parser.add_option('--path-to-isocodes', help='Path to iso-codes.zip') + + def run(self, opts): + if self._zip_data is None: + if opts.path_to_isocodes: + with open(opts.path_to_isocodes, 'rb') as f: + self._zip_data = BytesIO(f.read()) + # get top level directory + top = {item.split('/')[0] for item in zipfile.ZipFile(self.zip_data).namelist()} + assert len(top) == 1 + self.top_level = top.pop() + else: + self._zip_data = BytesIO(download_securely(self.URL)) def __init__(self): + super().__init__() self._zip_data = None + self.top_level = 'iso-codes-main' @property def zip_data(self): - if self._zip_data is None: - self._zip_data = BytesIO(download_securely(self.URL)) return self._zip_data def db_data(self, name: str) -> bytes: with zipfile.ZipFile(self.zip_data) as zf: - with zf.open(f'iso-codes-main/data/{name}') as f: + with zf.open(f'{self.top_level}/data/{name}') as f: return f.read() def extract_po_files(self, name: str, output_dir: str) -> None: name = name.split('.', 1)[0] - pat = f'iso-codes-main/{name}/*.po' + pat = f'{self.top_level}/{name}/*.po' with zipfile.ZipFile(self.zip_data) as zf: for name in fnmatch.filter(zf.namelist(), pat): dest = os.path.join(output_dir, name.split('/')[-1]) @@ -40,4 +56,5 @@ class ISOData: date_time = time.mktime(zi.date_time + (0, 0, -1)) os.utime(dest, (date_time, date_time)) + iso_data = ISOData() diff --git a/setup/resources.py b/setup/resources.py index a88b511b24..9f9a198aaa 100644 --- a/setup/resources.py +++ b/setup/resources.py @@ -141,22 +141,29 @@ class CACerts(Command): # {{{ description = 'Get updated mozilla CA certificate bundle' CA_PATH = os.path.join(Command.RESOURCES, 'mozilla-ca-certs.pem') + def add_options(self, parser): + parser.add_option('--path-to-cacerts', help='Path to mozilla-ca-certs.pem') + def run(self, opts): - try: - with open(self.CA_PATH, 'rb') as f: - raw = f.read() - except OSError as err: - if err.errno != errno.ENOENT: - raise - raw = b'' - nraw = download_securely('https://curl.haxx.se/ca/cacert.pem') - if not nraw: - raise RuntimeError('Failed to download CA cert bundle') - if nraw != raw: - self.info('Updating Mozilla CA certificates') - with open(self.CA_PATH, 'wb') as f: - f.write(nraw) - self.verify_ca_certs() + if opts.path_to_cacerts: + shutil.copyfile(opts.path_to_cacerts, self.CA_PATH) + os.chmod(self.CA_PATH, 0o644) + else: + try: + with open(self.CA_PATH, 'rb') as f: + raw = f.read() + except OSError as err: + if err.errno != errno.ENOENT: + raise + raw = b'' + nraw = download_securely('https://curl.haxx.se/ca/cacert.pem') + if not nraw: + raise RuntimeError('Failed to download CA cert bundle') + if nraw != raw: + self.info('Updating Mozilla CA certificates') + with open(self.CA_PATH, 'wb') as f: + f.write(nraw) + self.verify_ca_certs() def verify_ca_certs(self): from calibre.utils.https import get_https_resource_securely @@ -169,11 +176,18 @@ class RecentUAs(Command): # {{{ description = 'Get updated list of common browser user agents' UA_PATH = os.path.join(Command.RESOURCES, 'user-agent-data.json') + def add_options(self, parser): + parser.add_option('--path-to-user-agent-data', help='Path to user-agent-data.json') + def run(self, opts): from setup.browser_data import get_data - data = get_data() - with open(self.UA_PATH, 'w', encoding='utf-8') as f: - json.dump(data, f, indent=2, ensure_ascii=False, sort_keys=True) + if opts.path_to_user_agent_data: + shutil.copyfile(opts.path_to_user_agent_data, self.UA_PATH) + os.chmod(self.UA_PATH, 0o644) + else: + data = get_data() + with open(self.UA_PATH, 'w', encoding='utf-8') as f: + json.dump(data, f, indent=2, ensure_ascii=False, sort_keys=True) # }}} @@ -199,7 +213,7 @@ class RapydScript(Command): # {{{ class Resources(Command): # {{{ description = 'Compile various needed calibre resources' - sub_commands = ['kakasi', 'liberation_fonts', 'mathjax', 'rapydscript', 'hyphenation'] + sub_commands = ['kakasi', 'liberation_fonts', 'mathjax', 'rapydscript', 'hyphenation', 'iso_data'] def run(self, opts): from calibre.utils.serialize import msgpack_dumps