Decrease startup time for calibre worker processes by 30%

This commit is contained in:
Kovid Goyal 2013-05-02 18:34:02 +05:30
parent 927c5ccc54
commit 881b30ad62
10 changed files with 62 additions and 42 deletions

View File

@ -1761,7 +1761,7 @@ if __name__ == '__main__':
for x in ('lxml', 'calibre.ebooks.BeautifulSoup', 'uuid',
'calibre.utils.terminal', 'calibre.utils.magick', 'PIL', 'Image',
'sqlite3', 'mechanize', 'httplib', 'xml'):
'sqlite3', 'mechanize', 'httplib', 'xml', 'inspect'):
if x in sys.modules:
ret = 1
print (x, 'has been loaded by a plugin')

View File

@ -2622,7 +2622,7 @@ class ITUNES(DriverBase):
# for deletion from booklist[0] during add_books_to_metadata
for book in self.cached_books:
if (self.cached_books[book]['uuid'] == metadata.uuid or
(self.cached_books[book]['title'] == metadata.title and \
(self.cached_books[book]['title'] == metadata.title and
self.cached_books[book]['author'] == metadata.author)):
self.update_list.append(self.cached_books[book])
if DEBUG:

View File

@ -7,7 +7,7 @@ Created on 29 Jun 2012
@author: charles
'''
import socket, select, json, inspect, os, traceback, time, sys, random
import socket, select, json, os, traceback, time, sys, random
import posixpath
import hashlib, threading
import Queue
@ -345,6 +345,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
def _debug(self, *args):
# manual synchronization so we don't lose the calling method name
import inspect
with self.sync_lock:
if not DEBUG:
return

View File

@ -14,7 +14,6 @@ from calibre.constants import preferred_encoding
from calibre import isbytestring, force_unicode
from calibre.utils.config import device_prefs, tweaks
from calibre.utils.icu import sort_key
from calibre.utils.formatter import EvalFormatter
class Book(Metadata):
def __init__(self, prefix, lpath, size=None, other=None):
@ -109,6 +108,7 @@ class CollectionsBookList(BookList):
return None
def compute_category_name(self, field_key, field_value, field_meta):
from calibre.utils.formatter import EvalFormatter
renames = tweaks['sony_collection_renaming_rules']
field_name = renames.get(field_key, None)
if field_name is None:

View File

@ -10,7 +10,6 @@ import os
from contextlib import closing
from calibre.customize import FileTypePlugin
from calibre.utils.zipfile import ZipFile, stringFileHeader
def is_comic(list_of_names):
extensions = set([x.rpartition('.')[-1].lower() for x in list_of_names
@ -19,6 +18,7 @@ def is_comic(list_of_names):
return len(extensions - comic_extensions) == 0
def archive_type(stream):
from calibre.utils.zipfile import stringFileHeader
try:
pos = stream.tell()
except:
@ -47,6 +47,7 @@ class ArchiveExtract(FileTypePlugin):
on_import = True
def run(self, archive):
from calibre.utils.zipfile import ZipFile
is_rar = archive.lower().endswith('.rar')
if is_rar:
from calibre.utils.unrar import extract_member, names

View File

@ -15,7 +15,6 @@ from calibre.ebooks.metadata.book import (SC_COPYABLE_FIELDS,
from calibre.library.field_metadata import FieldMetadata
from calibre.utils.date import isoformat, format_date, parse_only_date
from calibre.utils.icu import sort_key
from calibre.utils.formatter import TemplateFormatter
# Special sets used to optimize the performance of getting and setting
# attributes on Metadata objects
@ -44,38 +43,6 @@ NULL_VALUES = {
field_metadata = FieldMetadata()
class SafeFormat(TemplateFormatter):
def get_value(self, orig_key, args, kwargs):
if not orig_key:
return ''
key = orig_key = orig_key.lower()
if key != 'title_sort' and key not in TOP_LEVEL_IDENTIFIERS and \
key not in ALL_METADATA_FIELDS:
key = field_metadata.search_term_to_field_key(key)
if key is None or (self.book and
key not in self.book.all_field_keys()):
if hasattr(self.book, orig_key):
key = orig_key
else:
raise ValueError(_('Value: unknown field ') + orig_key)
try:
b = self.book.get_user_metadata(key, False)
except:
b = None
if b and ((b['datatype'] == 'int' and self.book.get(key, 0) == 0) or
(b['datatype'] == 'float' and self.book.get(key, 0.0) == 0.0)):
v = ''
else:
v = self.book.format_field(key, series_with_index=False)[1]
if v is None:
return ''
if v == '':
return ''
return v
# DEPRECATED. This is not thread safe. Do not use.
composite_formatter = SafeFormat()
class Metadata(object):
@ -116,6 +83,7 @@ class Metadata(object):
# List of strings or []
self.author = list(authors) if authors else []# Needed for backward compatibility
self.authors = list(authors) if authors else []
from calibre.ebooks.metadata.book.formatter import SafeFormat
self.formatter = SafeFormat()
self.template_cache = template_cache
@ -454,6 +422,7 @@ class Metadata(object):
'''
if not ops:
return
from calibre.ebooks.metadata.book.formatter import SafeFormat
formatter = SafeFormat()
for op in ops:
try:

View File

@ -0,0 +1,48 @@
#!/usr/bin/env python
# vim:fileencoding=utf-8
from __future__ import (unicode_literals, division, absolute_import,
print_function)
__license__ = 'GPL v3'
__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
from calibre.ebooks.metadata.book import TOP_LEVEL_IDENTIFIERS, ALL_METADATA_FIELDS
from calibre.utils.formatter import TemplateFormatter
class SafeFormat(TemplateFormatter):
def __init__(self):
TemplateFormatter.__init__(self)
from calibre.ebooks.metadata.book.base import field_metadata
self.field_metadata = field_metadata
def get_value(self, orig_key, args, kwargs):
if not orig_key:
return ''
key = orig_key = orig_key.lower()
if key != 'title_sort' and key not in TOP_LEVEL_IDENTIFIERS and \
key not in ALL_METADATA_FIELDS:
key = self.field_metadata.search_term_to_field_key(key)
if key is None or (self.book and
key not in self.book.all_field_keys()):
if hasattr(self.book, orig_key):
key = orig_key
else:
raise ValueError(_('Value: unknown field ') + orig_key)
try:
b = self.book.get_user_metadata(key, False)
except:
b = None
if b and ((b['datatype'] == 'int' and self.book.get(key, 0) == 0) or
(b['datatype'] == 'float' and self.book.get(key, 0.0) == 0.0)):
v = ''
else:
v = self.book.format_field(key, series_with_index=False)[1]
if v is None:
return ''
if v == '':
return ''
return v

View File

@ -12,7 +12,7 @@ from PyQt4.Qt import Qt, QDialog, QGridLayout, QVBoxLayout, QFont, QLabel, \
from calibre.gui2.dialogs.metadata_bulk_ui import Ui_MetadataBulkDialog
from calibre.gui2.dialogs.tag_editor import TagEditor
from calibre.ebooks.metadata import string_to_authors, authors_to_string, title_sort
from calibre.ebooks.metadata.book.base import SafeFormat
from calibre.ebooks.metadata.book.formatter import SafeFormat
from calibre.gui2.custom_column_widgets import populate_metadata_page
from calibre.gui2 import error_dialog, ResizableDialog, UNDEFINED_QDATETIME, \
gprefs, question_dialog

View File

@ -11,7 +11,8 @@ from PyQt4.Qt import (Qt, QDialog, QDialogButtonBox, QSyntaxHighlighter, QFont,
from calibre.gui2 import error_dialog
from calibre.gui2.dialogs.template_dialog_ui import Ui_TemplateDialog
from calibre.utils.formatter_functions import formatter_functions
from calibre.ebooks.metadata.book.base import SafeFormat, Metadata
from calibre.ebooks.metadata.book.base import Metadata
from calibre.ebooks.metadata.book.formatter import SafeFormat
from calibre.library.coloring import (displayable_columns)
@ -178,7 +179,7 @@ class TemplateHighlighter(QSyntaxHighlighter):
list = reversed(self.paren_positions[0:found_pp])
for pp in list:
if pp.paren == chr:
stack += 1;
stack += 1
elif stack:
stack -= 1
else:

View File

@ -14,7 +14,7 @@ from PyQt4.Qt import (QAbstractTableModel, Qt, pyqtSignal, QIcon, QImage,
from calibre.gui2 import NONE, UNDEFINED_QDATETIME, error_dialog
from calibre.utils.search_query_parser import ParseException
from calibre.ebooks.metadata import fmt_sidx, authors_to_string, string_to_authors
from calibre.ebooks.metadata.book.base import SafeFormat
from calibre.ebooks.metadata.book.formatter import SafeFormat
from calibre.ptempfile import PersistentTemporaryFile
from calibre.utils.config import tweaks, device_prefs, prefs
from calibre.utils.date import dt_factory, qt_to_dt, as_local_time