From fc7d4830a921d5aeba1479f7e915f6f329f2f349 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 30 Jan 2016 12:40:50 +0530 Subject: [PATCH] Switch to using a private openssl lib on OS X --- .gitignore | 1 + setup/build_environment.py | 3 +++ setup/commands.py | 5 +++-- setup/extensions.py | 3 --- setup/install.py | 2 +- setup/installer/osx/app/main.py | 15 ++++++++++----- setup/publish.py | 1 + setup/resources.py | 30 ++++++++++++++++++++++++++++-- src/calibre/gui2/__init__.py | 5 +++-- src/calibre/test_build.py | 16 ++++++++++------ 10 files changed, 60 insertions(+), 21 deletions(-) diff --git a/.gitignore b/.gitignore index bb107c4a89..b1c640ea45 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ resources/template-functions.json resources/editor-functions.json resources/user-manual-translation-stats.json resources/content-server/main.js +resources/mozilla-ca-certs.pem icons/icns/*.iconset setup/installer/windows/calibre/build.log tags diff --git a/setup/build_environment.py b/setup/build_environment.py index eaab44b9b3..c3b77c45e3 100644 --- a/setup/build_environment.py +++ b/setup/build_environment.py @@ -196,6 +196,9 @@ elif isosx: ft_inc_dirs = [sw + '/include/freetype2'] icu_inc_dirs = [sw + '/include'] icu_lib_dirs = [sw + '/lib'] + SSL = os.environ.get('OPENSSL_DIR', os.path.join(sw, 'private', 'ssl')) + openssl_inc_dirs = [os.path.join(SSL, 'include')] + openssl_lib_dirs = [os.path.join(SSL, 'lib')] else: QT_DLLS += ['Qt5DBus', 'Qt5XcbQpa'] # PYQT_MODULES += ('QtDBus',) diff --git a/setup/commands.py b/setup/commands.py index e19896dada..c97ed77556 100644 --- a/setup/commands.py +++ b/setup/commands.py @@ -11,7 +11,7 @@ __all__ = [ 'build', 'mathjax', 'gui', 'develop', 'install', - 'kakasi', 'coffee', 'resources', + 'kakasi', 'coffee', 'cacerts', 'resources', 'check', 'sdist', 'bootstrap', 'manual', 'tag_release', @@ -50,10 +50,11 @@ gui = GUI() from setup.check import Check check = Check() -from setup.resources import Resources, Kakasi, Coffee +from setup.resources import Resources, Kakasi, Coffee, CACerts resources = Resources() kakasi = Kakasi() coffee = Coffee() +cacerts = CACerts() from setup.publish import Manual, TagRelease, Stage1, Stage2, \ Stage3, Stage4, Stage5, Publish, PublishBetas diff --git a/setup/extensions.py b/setup/extensions.py index 5e7f511672..33bb378c1a 100644 --- a/setup/extensions.py +++ b/setup/extensions.py @@ -112,9 +112,6 @@ extensions = [ Extension('certgen', ['calibre/utils/certgen.c'], libraries=['libeay32'] if iswindows else ['crypto'], - # Apple has deprecated openssl in OSX, so we need this, until we - # build our own private copy of openssl - cflags=['-Wno-deprecated-declarations'] if isosx else [], inc_dirs=openssl_inc_dirs, lib_dirs=openssl_lib_dirs, ), diff --git a/setup/install.py b/setup/install.py index 68b41320e5..351d791485 100644 --- a/setup/install.py +++ b/setup/install.py @@ -327,7 +327,7 @@ class Bootstrap(Command): description = 'Bootstrap a fresh checkout of calibre from git to a state where it can be installed. Requires various development tools/libraries/headers' TRANSLATIONS_REPO = 'https://github.com/kovidgoyal/calibre-translations.git' - sub_commands = 'build iso639 iso3166 translations gui resources'.split() + sub_commands = 'cacerts build iso639 iso3166 translations gui resources'.split() def pre_sub_commands(self, opts): tdir = self.j(self.d(self.SRC), 'translations') diff --git a/setup/installer/osx/app/main.py b/setup/installer/osx/app/main.py index c38d07fd95..5b00c38326 100644 --- a/setup/installer/osx/app/main.py +++ b/setup/installer/osx/app/main.py @@ -30,7 +30,8 @@ ENV = dict( MAGICK_CODER_FILTER_PATH=MAGICK_HOME+'/modules-Q16/filters', QT_PLUGIN_PATH='@executable_path/../MacOS/qt-plugins', PYTHONIOENCODING='UTF-8', - ) + SSL_CERT_FILE='@executable_path/../Resources/resources/mozilla-ca-certs.pem', +) info = warn = None @@ -261,7 +262,7 @@ class Py2App(object): @flush def get_local_dependencies(self, path_to_lib): for x, is_id in self.get_dependencies(path_to_lib): - for y in (SW+'/lib/', SW+'/qt/lib/', SW+'/python/Python.framework/',): + for y in (SW+'/lib/', SW+'/qt/lib/', SW+'/python/Python.framework/', SW+'/private/ssl/lib/'): if x.startswith(y): if y == SW+'/python/Python.framework/': y = SW+'/python/' @@ -468,11 +469,15 @@ class Py2App(object): @flush def add_misc_libraries(self): - for x in ('usb-1.0.0', 'mtp.9', 'ltdl.7', - 'chm.0', 'sqlite3.0', 'icudata.53', 'icui18n.53', 'icuio.53', 'icuuc.53'): + for x in ( + 'usb-1.0.0', 'mtp.9', 'ltdl.7', 'chm.0', 'sqlite3.0', + 'icudata.53', 'icui18n.53', 'icuio.53', 'icuuc.53', + 'crypto.1.0.0', 'ssl.1.0.0' + ): info('\nAdding', x) x = 'lib%s.dylib'%x - shutil.copy2(join(SW, 'lib', x), self.frameworks_dir) + src = join(SW, 'private', 'ssl', 'lib', x) if ('ssl' in x or 'crypto' in x) else join(SW, 'lib', x) + shutil.copy2(src, self.frameworks_dir) dest = join(self.frameworks_dir, x) self.set_id(dest, self.FID+'/'+x) self.fix_dependencies_in_lib(dest) diff --git a/setup/publish.py b/setup/publish.py index 4eda2dbcb7..6ca2cf663e 100644 --- a/setup/publish.py +++ b/setup/publish.py @@ -18,6 +18,7 @@ class Stage1(Command): sub_commands = [ 'check', + 'cacerts', 'pot', 'build', 'resources', diff --git a/setup/resources.py b/setup/resources.py index 149e3408b3..a01354289a 100644 --- a/setup/resources.py +++ b/setup/resources.py @@ -6,7 +6,7 @@ __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, cPickle, re, shutil, marshal, zipfile, glob, time, sys, hashlib, json +import os, cPickle, re, shutil, marshal, zipfile, glob, time, sys, hashlib, json, urllib, errno from zlib import compress from itertools import chain @@ -222,6 +222,33 @@ class Kakasi(Command): # {{{ shutil.rmtree(kakasi) # }}} +class CACerts(Command): # {{{ + + description = 'Get updated mozilla CA certificate bundle' + CA_PATH = os.path.join(Command.RESOURCES, 'mozilla-ca-certs.pem') + + def run(self, opts): + try: + with open(self.CA_PATH, 'rb') as f: + raw = f.read() + except EnvironmentError as err: + if err.errno != errno.ENOENT: + raise + raw = b'' + nraw = urllib.urlopen('https://curl.haxx.se/ca/cacert.pem').read() + if not nraw: + raise RuntimeError('Failed to download CA cert bundle') + if nraw != raw: + self.info('Updating Mozilla CA certificates') + with open(self.CA_PATH, 'wb') as f: + f.write(nraw) + self.verify_ca_certs() + + def verify_ca_certs(self): + from calibre.utils.https import get_https_resource_securely + get_https_resource_securely('https://calibre-ebook.com', cacerts=self.b(self.CA_PATH)) +# }}} + class Resources(Command): # {{{ description = 'Compile various needed calibre resources' @@ -349,4 +376,3 @@ class Resources(Command): # {{{ if os.path.exists(x): os.remove(x) # }}} - diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py index 98034c91f4..522bf87d09 100644 --- a/src/calibre/gui2/__init__.py +++ b/src/calibre/gui2/__init__.py @@ -1040,8 +1040,9 @@ def sanitize_env_vars(): elif iswindows: env_vars = {k:None for k in 'MAGICK_HOME MAGICK_CONFIGURE_PATH MAGICK_CODER_MODULE_PATH MAGICK_FILTER_MODULE_PATH QT_PLUGIN_PATH'.split()} elif isosx: - env_vars = {k:None for k in - 'FONTCONFIG_FILE FONTCONFIG_PATH MAGICK_CONFIGURE_PATH MAGICK_CODER_MODULE_PATH MAGICK_FILTER_MODULE_PATH QT_PLUGIN_PATH'.split()} + env_vars = {k:None for k in ( + 'FONTCONFIG_FILE FONTCONFIG_PATH MAGICK_CONFIGURE_PATH MAGICK_CODER_MODULE_PATH' + ' MAGICK_FILTER_MODULE_PATH QT_PLUGIN_PATH SSL_CERT_FILE').split()} else: env_vars = {} diff --git a/src/calibre/test_build.py b/src/calibre/test_build.py index 7871be7685..7963f9a28f 100644 --- a/src/calibre/test_build.py +++ b/src/calibre/test_build.py @@ -173,11 +173,6 @@ def test_unrar(): test_basic() fprint('Unrar OK!') -def test_ssl(): - import ssl - ssl - fprint('SSL OK!') - def test_icu(): fprint('Testing ICU') from calibre.utils.icu_test import test_build @@ -269,6 +264,15 @@ def test_image_compression(): test() fprint('Image compression OK!') +def test_openssl(): + import ssl + ssl.PROTOCOL_TLSv1_2 + if isosx: + cafile = ssl.get_default_verify_paths().cafile + if not cafile or not cafile.endswith('/mozilla-ca-certs.pem') or not os.access(cafile, os.R_OK): + raise ValueError('Mozilla CA certs not loaded') + fprint('SSL OK!') + def test(): if iswindows: test_dlls() @@ -279,7 +283,7 @@ def test(): test_dukpy() test_spell() test_lxml() - test_ssl() + test_openssl() test_sqlite() test_apsw() test_imaging()