Switch to using a private openssl lib on OS X

.gitignore

@@ -21,6 +21,7 @@ resources/template-functions.json
 resources/editor-functions.json
 resources/user-manual-translation-stats.json
 resources/content-server/main.js
+resources/mozilla-ca-certs.pem
 icons/icns/*.iconset
 setup/installer/windows/calibre/build.log
 tags

setup/build_environment.py

@@ -196,6 +196,9 @@ elif isosx:
     ft_inc_dirs = [sw + '/include/freetype2']
     icu_inc_dirs = [sw + '/include']
     icu_lib_dirs = [sw + '/lib']
+    SSL = os.environ.get('OPENSSL_DIR', os.path.join(sw, 'private', 'ssl'))
+    openssl_inc_dirs = [os.path.join(SSL, 'include')]
+    openssl_lib_dirs = [os.path.join(SSL, 'lib')]
 else:
     QT_DLLS += ['Qt5DBus', 'Qt5XcbQpa']
     # PYQT_MODULES += ('QtDBus',)

setup/commands.py

@@ -11,7 +11,7 @@ __all__ = [
         'build', 'mathjax',
         'gui',
         'develop', 'install',
-        'kakasi', 'coffee', 'resources',
+        'kakasi', 'coffee', 'cacerts', 'resources',
         'check',
         'sdist', 'bootstrap',
         'manual', 'tag_release',
@@ -50,10 +50,11 @@ gui = GUI()
 from setup.check import Check
 check = Check()
 
-from setup.resources import Resources, Kakasi, Coffee
+from setup.resources import Resources, Kakasi, Coffee, CACerts
 resources = Resources()
 kakasi = Kakasi()
 coffee = Coffee()
+cacerts = CACerts()
 
 from setup.publish import Manual, TagRelease, Stage1, Stage2, \
         Stage3, Stage4, Stage5, Publish, PublishBetas

setup/extensions.py

@@ -112,9 +112,6 @@ extensions = [
     Extension('certgen',
         ['calibre/utils/certgen.c'],
         libraries=['libeay32'] if iswindows else ['crypto'],
-        # Apple has deprecated openssl in OSX, so we need this, until we
-        # build our own private copy of openssl
-        cflags=['-Wno-deprecated-declarations'] if isosx else [],
         inc_dirs=openssl_inc_dirs, lib_dirs=openssl_lib_dirs,
         ),
 

setup/install.py

@@ -327,7 +327,7 @@ class Bootstrap(Command):
 
     description = 'Bootstrap a fresh checkout of calibre from git to a state where it can be installed. Requires various development tools/libraries/headers'
     TRANSLATIONS_REPO = 'https://github.com/kovidgoyal/calibre-translations.git'
-    sub_commands = 'build iso639 iso3166 translations gui resources'.split()
+    sub_commands = 'cacerts build iso639 iso3166 translations gui resources'.split()
 
     def pre_sub_commands(self, opts):
         tdir = self.j(self.d(self.SRC), 'translations')

setup/installer/osx/app/main.py

@@ -30,7 +30,8 @@ ENV = dict(
         MAGICK_CODER_FILTER_PATH=MAGICK_HOME+'/modules-Q16/filters',
         QT_PLUGIN_PATH='@executable_path/../MacOS/qt-plugins',
         PYTHONIOENCODING='UTF-8',
-        )
+        SSL_CERT_FILE='@executable_path/../Resources/resources/mozilla-ca-certs.pem',
+)
 
 
 info = warn = None
@@ -261,7 +262,7 @@ class Py2App(object):
     @flush
     def get_local_dependencies(self, path_to_lib):
         for x, is_id in self.get_dependencies(path_to_lib):
-            for y in (SW+'/lib/', SW+'/qt/lib/', SW+'/python/Python.framework/',):
+            for y in (SW+'/lib/', SW+'/qt/lib/', SW+'/python/Python.framework/', SW+'/private/ssl/lib/'):
                 if x.startswith(y):
                     if y == SW+'/python/Python.framework/':
                         y = SW+'/python/'
@@ -468,11 +469,15 @@ class Py2App(object):
 
     @flush
     def add_misc_libraries(self):
-        for x in ('usb-1.0.0', 'mtp.9', 'ltdl.7',
+        for x in (
-                  'chm.0', 'sqlite3.0', 'icudata.53', 'icui18n.53', 'icuio.53', 'icuuc.53'):
+                'usb-1.0.0', 'mtp.9', 'ltdl.7', 'chm.0', 'sqlite3.0',
+                'icudata.53', 'icui18n.53', 'icuio.53', 'icuuc.53',
+                'crypto.1.0.0', 'ssl.1.0.0'
+        ):
             info('\nAdding', x)
             x = 'lib%s.dylib'%x
-            shutil.copy2(join(SW, 'lib', x), self.frameworks_dir)
+            src = join(SW, 'private', 'ssl', 'lib', x) if ('ssl' in x or 'crypto' in x) else join(SW, 'lib', x)
+            shutil.copy2(src, self.frameworks_dir)
             dest = join(self.frameworks_dir, x)
             self.set_id(dest, self.FID+'/'+x)
             self.fix_dependencies_in_lib(dest)

setup/publish.py

@@ -18,6 +18,7 @@ class Stage1(Command):
 
     sub_commands = [
             'check',
+            'cacerts',
             'pot',
             'build',
             'resources',

setup/resources.py

@@ -6,7 +6,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os, cPickle, re, shutil, marshal, zipfile, glob, time, sys, hashlib, json
+import os, cPickle, re, shutil, marshal, zipfile, glob, time, sys, hashlib, json, urllib, errno
 from zlib import compress
 from itertools import chain
 
@@ -222,6 +222,33 @@ class Kakasi(Command):  # {{{
             shutil.rmtree(kakasi)
 # }}}
 
+class CACerts(Command):  # {{{
+
+    description = 'Get updated mozilla CA certificate bundle'
+    CA_PATH = os.path.join(Command.RESOURCES, 'mozilla-ca-certs.pem')
+
+    def run(self, opts):
+        try:
+            with open(self.CA_PATH, 'rb') as f:
+                raw = f.read()
+        except EnvironmentError as err:
+            if err.errno != errno.ENOENT:
+                raise
+            raw = b''
+        nraw = urllib.urlopen('https://curl.haxx.se/ca/cacert.pem').read()
+        if not nraw:
+            raise RuntimeError('Failed to download CA cert bundle')
+        if nraw != raw:
+            self.info('Updating Mozilla CA certificates')
+            with open(self.CA_PATH, 'wb') as f:
+                f.write(nraw)
+            self.verify_ca_certs()
+
+    def verify_ca_certs(self):
+        from calibre.utils.https import get_https_resource_securely
+        get_https_resource_securely('https://calibre-ebook.com', cacerts=self.b(self.CA_PATH))
+# }}}
+
 class Resources(Command):  # {{{
 
     description = 'Compile various needed calibre resources'
@@ -349,4 +376,3 @@ class Resources(Command):  # {{{
             if os.path.exists(x):
                 os.remove(x)
 # }}}
-

src/calibre/gui2/__init__.py

@@ -1040,8 +1040,9 @@ def sanitize_env_vars():
     elif iswindows:
         env_vars = {k:None for k in 'MAGICK_HOME MAGICK_CONFIGURE_PATH MAGICK_CODER_MODULE_PATH MAGICK_FILTER_MODULE_PATH QT_PLUGIN_PATH'.split()}
     elif isosx:
-        env_vars = {k:None for k in
+        env_vars = {k:None for k in (
-                    'FONTCONFIG_FILE FONTCONFIG_PATH MAGICK_CONFIGURE_PATH MAGICK_CODER_MODULE_PATH MAGICK_FILTER_MODULE_PATH QT_PLUGIN_PATH'.split()}
+                    'FONTCONFIG_FILE FONTCONFIG_PATH MAGICK_CONFIGURE_PATH MAGICK_CODER_MODULE_PATH'
+                    ' MAGICK_FILTER_MODULE_PATH QT_PLUGIN_PATH SSL_CERT_FILE').split()}
     else:
         env_vars = {}
 

src/calibre/test_build.py

@@ -173,11 +173,6 @@ def test_unrar():
     test_basic()
     fprint('Unrar OK!')
 
-def test_ssl():
-    import ssl
-    ssl
-    fprint('SSL OK!')
-
 def test_icu():
     fprint('Testing ICU')
     from calibre.utils.icu_test import test_build
@@ -269,6 +264,15 @@ def test_image_compression():
     test()
     fprint('Image compression OK!')
 
+def test_openssl():
+    import ssl
+    ssl.PROTOCOL_TLSv1_2
+    if isosx:
+        cafile = ssl.get_default_verify_paths().cafile
+        if not cafile or not cafile.endswith('/mozilla-ca-certs.pem') or not os.access(cafile, os.R_OK):
+            raise ValueError('Mozilla CA certs not loaded')
+    fprint('SSL OK!')
+
 def test():
     if iswindows:
         test_dlls()
@@ -279,7 +283,7 @@ def test():
     test_dukpy()
     test_spell()
     test_lxml()
-    test_ssl()
+    test_openssl()
     test_sqlite()
     test_apsw()
     test_imaging()