More delay load optimizations to reduce worker startup time

src/calibre/__init__.py

@@ -3,9 +3,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import sys, os, re, logging, time, random, __builtin__, warnings
+import sys, os, re, time, random, __builtin__, warnings
-from urllib import getproxies
-from urllib2 import unquote as urllib2_unquote
 __builtin__.__dict__['dynamic_property'] = lambda(func): func(None)
 from htmlentitydefs import name2codepoint
 from math import floor
@@ -15,13 +13,12 @@ warnings.simplefilter('ignore', DeprecationWarning)
 
 
 from calibre.constants import (iswindows, isosx, islinux, isfreebsd, isfrozen,
-                              terminal_controller, preferred_encoding,
+                              preferred_encoding, __appname__, __version__, __author__,
-                              __appname__, __version__, __author__,
                               win32event, win32api, winerror, fcntl,
                               filesystem_encoding, plugins, config_dir)
-from calibre.startup import winutil, winutilerror, guess_type
+from calibre.startup import winutil, winutilerror
 
-if islinux and not getattr(sys, 'frozen', False):
+if False and islinux and not getattr(sys, 'frozen', False):
     # Imported before PyQt4 to workaround PyQt4 util-linux conflict discovered on gentoo
     # See http://bugs.gentoo.org/show_bug.cgi?id=317557
     # Importing uuid is slow so get rid of this at some point, maybe in a few
@@ -33,8 +30,33 @@ if islinux and not getattr(sys, 'frozen', False):
 if False:
     # Prevent pyflakes from complaining
     winutil, winutilerror, __appname__, islinux, __version__
-    fcntl, win32event, isfrozen, __author__, terminal_controller
+    fcntl, win32event, isfrozen, __author__
-    winerror, win32api, isfreebsd, guess_type
+    winerror, win32api, isfreebsd
+
+_mt_inited = False
+def _init_mimetypes():
+    global _mt_inited
+    import mimetypes
+    mimetypes.init([P('mime.types')])
+    _mt_inited = True
+
+def guess_type(*args, **kwargs):
+    import mimetypes
+    if not _mt_inited:
+        _init_mimetypes()
+    return mimetypes.guess_type(*args, **kwargs)
+
+def guess_all_extensions(*args, **kwargs):
+    import mimetypes
+    if not _mt_inited:
+        _init_mimetypes()
+    return mimetypes.guess_all_extensions(*args, **kwargs)
+
+def get_types_map():
+    import mimetypes
+    if not _mt_inited:
+        _init_mimetypes()
+    return mimetypes.types_map
 
 def to_unicode(raw, encoding='utf-8', errors='strict'):
     if isinstance(raw, unicode):
@@ -182,6 +204,7 @@ class CommandLineError(Exception):
     pass
 
 def setup_cli_handlers(logger, level):
+    import logging
     if os.environ.get('CALIBRE_WORKER', None) is not None and logger.handlers:
         return
     logger.setLevel(level)
@@ -243,6 +266,7 @@ def extract(path, dir):
     extractor(path, dir)
 
 def get_proxies(debug=True):
+    from urllib import getproxies
     proxies = getproxies()
     for key, proxy in list(proxies.items()):
         if not proxy or '..' in proxy:
@@ -552,6 +576,8 @@ def get_download_filename(url, cookie_file=None):
     Get a local filename for a URL using the content disposition header
     '''
     from contextlib import closing
+    from urllib2 import unquote as urllib2_unquote
+
     filename = ''
 
     br = browser()

src/calibre/constants.py

@@ -1,23 +1,27 @@
+from future_builtins import map
+
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
-__appname__   = 'calibre'
+__appname__   = u'calibre'
-__version__   = '0.7.56'
+numeric_version = (0, 7, 56)
-__author__    = "Kovid Goyal <kovid@kovidgoyal.net>"
+__version__   = u'.'.join(map(unicode, numeric_version))
-
+__author__    = u"Kovid Goyal <kovid@kovidgoyal.net>"
-import re, importlib
-_ver = __version__.split('.')
-_ver = [int(re.search(r'(\d+)', x).group(1)) for x in _ver]
-numeric_version = tuple(_ver)
 
 '''
 Various run time constants.
 '''
 
-import sys, locale, codecs, os
+import sys, locale, codecs, os, importlib
-from calibre.utils.terminfo import TerminalController
+
+_tc = None
+def terminal_controller():
+    global _tc
+    if _tc is None:
+        from calibre.utils.terminfo import TerminalController
+        _tc = TerminalController(sys.stdout)
+    return _tc
 
-terminal_controller = TerminalController(sys.stdout)
 
 iswindows = 'win32' in sys.platform.lower() or 'win64' in sys.platform.lower()
 isosx     = 'darwin' in sys.platform.lower()

src/calibre/debug.py

@@ -106,7 +106,7 @@ def migrate(old, new):
     from calibre.library.database import LibraryDatabase
     from calibre.library.database2 import LibraryDatabase2
     from calibre.utils.terminfo import ProgressBar
-    from calibre import terminal_controller
+    from calibre.constants import terminal_controller
     class Dummy(ProgressBar):
         def setLabelText(self, x): pass
         def setAutoReset(self, y): pass
@@ -119,7 +119,7 @@ def migrate(old, new):
 
     db = LibraryDatabase(old)
     db2 = LibraryDatabase2(new)
-    db2.migrate_old(db, Dummy(terminal_controller, 'Migrating database...'))
+    db2.migrate_old(db, Dummy(terminal_controller(), 'Migrating database...'))
     prefs['library_path'] = os.path.abspath(new)
     print 'Database migrated to', os.path.abspath(new)
 

src/calibre/ebooks/chm/reader.py

@@ -5,8 +5,8 @@ __copyright__  = '2008, Kovid Goyal <kovid at kovidgoyal.net>,' \
                  ' and Alex Bramley <a.bramley at gmail.com>.'
 
 import os, re
-from mimetypes import guess_type as guess_mimetype
 
+from calibre import guess_type as guess_mimetype
 from calibre.ebooks.BeautifulSoup import BeautifulSoup, NavigableString
 from calibre.constants import iswindows, filesystem_encoding
 from calibre.utils.chm.chm import CHMFile

src/calibre/ebooks/conversion/plumber.py

@@ -14,7 +14,8 @@ from calibre.ebooks.conversion.preprocess import HTMLPreProcessor
 from calibre.ptempfile import PersistentTemporaryDirectory
 from calibre.utils.date import parse_date
 from calibre.utils.zipfile import ZipFile
-from calibre import extract, walk, isbytestring, filesystem_encoding
+from calibre import (extract, walk, isbytestring, filesystem_encoding,
+        get_types_map)
 from calibre.constants import __version__
 
 DEBUG_README=u'''
@@ -877,6 +878,7 @@ OptionRecommendation(name='sr3_replace',
         self.flush()
         import cssutils, logging
         cssutils.log.setLevel(logging.WARN)
+        get_types_map() # Ensure the mimetypes module is intialized
 
         if self.opts.debug_pipeline is not None:
             self.opts.verbose = max(self.opts.verbose, 4)

src/calibre/ebooks/fb2/fb2ml.py

@@ -10,7 +10,6 @@ Transform OEB content into FB2 markup
 
 from base64 import b64encode
 from datetime import datetime
-from mimetypes import types_map
 import re
 import uuid
 
@@ -259,7 +258,7 @@ class FB2MLizer(object):
                 continue
             if item.media_type in OEB_RASTER_IMAGES:
                 try:
-                    if not item.media_type == types_map['.jpeg'] or not item.media_type == types_map['.jpg']:
+                    if item.media_type != 'image/jpeg':
                         im = Image()
                         im.load(item.data)
                         im.set_compression_quality(70)

src/calibre/ebooks/metadata/__init__.py

@@ -6,11 +6,11 @@ __docformat__ = 'restructuredtext en'
 """
 Provides abstraction for metadata reading.writing from a variety of ebook formats.
 """
-import os, mimetypes, sys, re
+import os, sys, re
 from urllib import unquote, quote
 from urlparse import urlparse
 
-from calibre import relpath
+from calibre import relpath, guess_type
 
 from calibre.utils.config import tweaks
 
@@ -118,7 +118,7 @@ class Resource(object):
         self.path = None
         self.fragment = ''
         try:
-            self.mime_type = mimetypes.guess_type(href_or_path)[0]
+            self.mime_type = guess_type(href_or_path)[0]
         except:
             self.mime_type = None
         if self.mime_type is None:

src/calibre/ebooks/metadata/fb2.py

@@ -5,11 +5,12 @@ __copyright__ = '2008, Anatoly Shipitsin <norguhtar at gmail.com>'
 
 '''Read meta information from fb2 files'''
 
-import mimetypes, os
+import os
 from base64 import b64decode
 from lxml import etree
 from calibre.ebooks.metadata import MetaInformation
 from calibre.ebooks.chardet import xml_to_unicode
+from calibre import guess_all_extensions
 
 XLINK_NS     = 'http://www.w3.org/1999/xlink'
 def XLINK(name):
@@ -71,7 +72,7 @@ def get_metadata(stream):
             binary = XPath('//fb2:binary[@id="%s"]'%id)(root)
             if binary:
                 mt = binary[0].get('content-type', 'image/jpeg')
-                exts = mimetypes.guess_all_extensions(mt)
+                exts = guess_all_extensions(mt)
                 if not exts:
                     exts = ['.jpg']
                 cdata = (exts[0][1:], b64decode(tostring(binary[0])))

src/calibre/ebooks/metadata/opf2.py

@@ -7,7 +7,7 @@ __docformat__ = 'restructuredtext en'
 lxml based OPF parser.
 '''
 
-import re, sys, unittest, functools, os, mimetypes, uuid, glob, cStringIO, json
+import re, sys, unittest, functools, os, uuid, glob, cStringIO, json
 from urllib import unquote
 from urlparse import urlparse
 
@@ -20,7 +20,7 @@ from calibre.ebooks.metadata import string_to_authors, MetaInformation, check_is
 from calibre.ebooks.metadata.book.base import Metadata
 from calibre.utils.date import parse_date, isoformat
 from calibre.utils.localization import get_lang
-from calibre import prints
+from calibre import prints, guess_type
 from calibre.utils.cleantext import clean_ascii_chars
 
 class Resource(object): # {{{
@@ -42,7 +42,7 @@ class Resource(object): # {{{
         self.path = None
         self.fragment = ''
         try:
-            self.mime_type = mimetypes.guess_type(href_or_path)[0]
+            self.mime_type = guess_type(href_or_path)[0]
         except:
             self.mime_type = None
         if self.mime_type is None:
@@ -1000,7 +1000,7 @@ class OPF(object): # {{{
                 for t in ('cover', 'other.ms-coverimage-standard', 'other.ms-coverimage'):
                     for item in self.guide:
                         if item.type.lower() == t:
-                            self.create_manifest_item(item.href(), mimetypes.guess_type(path)[0])
+                            self.create_manifest_item(item.href(), guess_type(path)[0])
 
         return property(fget=fget, fset=fset)
 

src/calibre/ebooks/oeb/base.py

@@ -8,7 +8,6 @@ __copyright__ = '2008, Marshall T. Vandegrift <llasram@gmail.com>'
 __docformat__ = 'restructuredtext en'
 
 import os, re, uuid, logging
-from mimetypes import types_map
 from collections import defaultdict
 from itertools import count
 from urlparse import urldefrag, urlparse, urlunparse, urljoin
@@ -20,7 +19,7 @@ from calibre.translations.dynamic import translate
 from calibre.ebooks.chardet import xml_to_unicode
 from calibre.ebooks.oeb.entitydefs import ENTITYDEFS
 from calibre.ebooks.conversion.preprocess import CSSPreProcessor
-from calibre import isbytestring, as_unicode
+from calibre import isbytestring, as_unicode, get_types_map
 
 RECOVER_PARSER = etree.XMLParser(recover=True, no_network=True)
 
@@ -247,7 +246,7 @@ def rewrite_links(root, link_repl_func, resolve_base_href=False):
                 el.attrib['style'] = repl
 
 
-
+types_map = get_types_map()
 EPUB_MIME      = types_map['.epub']
 XHTML_MIME     = types_map['.xhtml']
 CSS_MIME       = types_map['.css']

src/calibre/ebooks/oeb/reader.py

@@ -10,7 +10,6 @@ import sys, os, uuid, copy, re, cStringIO
 from itertools import izip
 from urlparse import urldefrag, urlparse
 from urllib import unquote as urlunquote
-from mimetypes import guess_type
 from collections import defaultdict
 
 from lxml import etree
@@ -29,6 +28,7 @@ from calibre.ebooks.oeb.entitydefs import ENTITYDEFS
 from calibre.utils.localization import get_lang
 from calibre.ptempfile import TemporaryDirectory
 from calibre.constants import __appname__, __version__
+from calibre import guess_type
 
 __all__ = ['OEBReader']
 

src/calibre/ebooks/snb/snbfile.py

@@ -5,7 +5,8 @@ __copyright__ = '2010, Li Fanxi <lifanxi@freemindworld.com>'
 __docformat__ = 'restructuredtext en'
 
 import sys, struct, zlib, bz2, os
-from mimetypes import types_map
+
+from calibre import guess_type
 
 class FileStream:
     def IsBinary(self):
@@ -180,7 +181,7 @@ class SNBFile:
                 file = open(os.path.join(path, fname), 'wb')
                 file.write(f.fileBody)
                 file.close()
-                fileNames.append((fname, types_map[ext]))
+                fileNames.append((fname, guess_type('a'+ext)[0]))
         return fileNames
 
     def Output(self, outputFile):

src/calibre/library/cli.py

@@ -10,8 +10,7 @@ Command line interface to the calibre database.
 import sys, os, cStringIO, re
 from textwrap import TextWrapper
 
-from calibre import terminal_controller, preferred_encoding, prints, \
+from calibre import preferred_encoding, prints, isbytestring
-    isbytestring
 from calibre.utils.config import OptionParser, prefs, tweaks
 from calibre.ebooks.metadata.meta import get_metadata
 from calibre.library.database2 import LibraryDatabase2
@@ -53,6 +52,8 @@ def get_db(dbpath, options):
 
 def do_list(db, fields, afields, sort_by, ascending, search_text, line_width, separator,
             prefix, subtitle='Books in the calibre database'):
+    from calibre.constants import terminal_controller as tc
+    terminal_controller = tc()
     if sort_by:
         db.sort(sort_by, ascending)
     if search_text:
@@ -1087,6 +1088,9 @@ def command_list_categories(args, dbpath):
     fields = ['category', 'tag_name', 'count', 'rating']
 
     def do_list():
+        from calibre.constants import terminal_controller as tc
+        terminal_controller = tc()
+
         separator = ' '
         widths = list(map(lambda x : 0, fields))
         for i in data:

src/calibre/startup.py

@@ -163,10 +163,6 @@ if not _run_once:
     __builtin__.__dict__['icu_upper'] = icu_upper
     __builtin__.__dict__['icu_title'] = title_case
 
-    import mimetypes
-    mimetypes.init([P('mime.types')])
-    guess_type = mimetypes.guess_type
-
 def test_lopen():
     from calibre.ptempfile import TemporaryDirectory
     from calibre import CurrentDir

src/calibre/utils/config.py

@@ -6,15 +6,15 @@ __docformat__ = 'restructuredtext en'
 '''
 Manage application-wide preferences.
 '''
-import os, re, cPickle, textwrap, traceback, plistlib, json, base64, datetime
+import os, re, cPickle, traceback, base64, datetime
 from copy import deepcopy
 from functools import partial
 from optparse import OptionParser as _OptionParser
 from optparse import IndentedHelpFormatter
 from collections import defaultdict
 
-from calibre.constants import terminal_controller, config_dir, CONFIG_DIR_MODE, \
+from calibre.constants import (config_dir, CONFIG_DIR_MODE, __appname__,
-                              __appname__, __version__, __author__
+        __version__, __author__, terminal_controller)
 from calibre.utils.lock import LockError, ExclusiveFile
 
 plugin_dir = os.path.join(config_dir, 'plugins')
@@ -29,23 +29,28 @@ def check_config_write_access():
 class CustomHelpFormatter(IndentedHelpFormatter):
 
     def format_usage(self, usage):
-        return _("%sUsage%s: %s\n") % (terminal_controller.BLUE, terminal_controller.NORMAL, usage)
+        tc = terminal_controller()
+        return _("%sUsage%s: %s\n") % (tc.BLUE, tc.NORMAL, usage)
 
     def format_heading(self, heading):
-        return "%*s%s%s%s:\n" % (self.current_indent, terminal_controller.BLUE,
+        tc = terminal_controller()
-                                 "", heading, terminal_controller.NORMAL)
+        return "%*s%s%s%s:\n" % (self.current_indent, tc.BLUE,
+                                 "", heading, tc.NORMAL)
 
     def format_option(self, option):
+        import textwrap
+        tc = terminal_controller()
+
         result = []
         opts = self.option_strings[option]
         opt_width = self.help_position - self.current_indent - 2
         if len(opts) > opt_width:
             opts = "%*s%s\n" % (self.current_indent, "",
-                                    terminal_controller.GREEN+opts+terminal_controller.NORMAL)
+                                    tc.GREEN+opts+tc.NORMAL)
             indent_first = self.help_position
         else:                       # start help on same line as opts
-            opts = "%*s%-*s  " % (self.current_indent, "", opt_width + len(terminal_controller.GREEN + terminal_controller.NORMAL),
+            opts = "%*s%-*s  " % (self.current_indent, "", opt_width +
-                                  terminal_controller.GREEN + opts + terminal_controller.NORMAL)
+                    len(tc.GREEN + tc.NORMAL), tc.GREEN + opts + tc.NORMAL)
             indent_first = 0
         result.append(opts)
         if option.help:
@@ -71,9 +76,12 @@ class OptionParser(_OptionParser):
                  gui_mode=False,
                  conflict_handler='resolve',
                  **kwds):
+        import textwrap
+        tc = terminal_controller()
+
         usage = textwrap.dedent(usage)
         if epilog is None:
-            epilog = _('Created by ')+terminal_controller.RED+__author__+terminal_controller.NORMAL
+            epilog = _('Created by ')+tc.RED+__author__+tc.NORMAL
         usage += '\n\n'+_('''Whenever you pass arguments to %prog that have spaces in them, '''
                  '''enclose the arguments in quotation marks.''')
         _OptionParser.__init__(self, usage=usage, version=version, epilog=epilog,
@@ -579,9 +587,11 @@ class XMLConfig(dict):
         self.refresh()
 
     def raw_to_object(self, raw):
+        import plistlib
         return plistlib.readPlistFromString(raw)
 
     def to_raw(self):
+        import plistlib
         return plistlib.writePlistToString(self)
 
     def refresh(self):
@@ -601,6 +611,7 @@ class XMLConfig(dict):
         self.update(d)
 
     def __getitem__(self, key):
+        import plistlib
         try:
             ans = dict.__getitem__(self, key)
             if isinstance(ans, plistlib.Data):
@@ -610,6 +621,7 @@ class XMLConfig(dict):
             return self.defaults.get(key, None)
 
     def get(self, key, default=None):
+        import plistlib
         try:
             ans = dict.__getitem__(self, key)
             if isinstance(ans, plistlib.Data):
@@ -619,6 +631,7 @@ class XMLConfig(dict):
             return self.defaults.get(key, default)
 
     def __setitem__(self, key, val):
+        import plistlib
         if isinstance(val, (bytes, str)):
             val = plistlib.Data(val)
         dict.__setitem__(self, key, val)
@@ -667,9 +680,11 @@ class JSONConfig(XMLConfig):
     EXTENSION = '.json'
 
     def raw_to_object(self, raw):
+        import json
         return json.loads(raw.decode('utf-8'), object_hook=from_json)
 
     def to_raw(self):
+        import json
         return json.dumps(self, indent=2, default=to_json)
 
     def __getitem__(self, key):