From b3cbbd3ea8e05e1cd75b12e70d28ca1decc505d4 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Thu, 26 Aug 2010 14:32:57 +0100
Subject: [PATCH 001/207] Initial attempt, including full cached metadata, json
serialization, and custom fields in save paths. Custom fields in collections
probably work, but they haven't been tested.
---
src/calibre/devices/apple/driver.py | 7 +-
src/calibre/devices/interface.py | 4 +-
src/calibre/devices/kobo/books.py | 21 +-
src/calibre/devices/usbms/books.py | 43 +--
src/calibre/devices/usbms/driver.py | 21 +-
src/calibre/ebooks/metadata/__init__.py | 216 +------------
src/calibre/ebooks/metadata/book/__init__.py | 41 ++-
src/calibre/ebooks/metadata/book/base.py | 296 +++++++++++++++---
.../ebooks/metadata/book/json_codec.py | 125 ++++++++
src/calibre/ebooks/metadata/isbndb.py | 6 +-
src/calibre/ebooks/metadata/opf2.py | 9 +-
.../gui2/dialogs/config/save_template.py | 29 +-
src/calibre/gui2/library/models.py | 3 +-
src/calibre/library/database2.py | 31 +-
src/calibre/library/save_to_disk.py | 19 +-
15 files changed, 514 insertions(+), 357 deletions(-)
create mode 100644 src/calibre/ebooks/metadata/book/json_codec.py
diff --git a/src/calibre/devices/apple/driver.py b/src/calibre/devices/apple/driver.py
index 916c88f203..75517e9df7 100644
--- a/src/calibre/devices/apple/driver.py
+++ b/src/calibre/devices/apple/driver.py
@@ -13,7 +13,8 @@ from calibre.devices.errors import UserFeedback
from calibre.devices.usbms.deviceconfig import DeviceConfig
from calibre.devices.interface import DevicePlugin
from calibre.ebooks.BeautifulSoup import BeautifulSoup
-from calibre.ebooks.metadata import MetaInformation, authors_to_string
+from calibre.ebooks.metadata import authors_to_string
+from calibre.ebooks.metadata.book.base import Metadata
from calibre.ebooks.metadata.epub import set_metadata
from calibre.library.server.utils import strftime
from calibre.utils.config import config_dir
@@ -2998,14 +2999,14 @@ class BookList(list):
'''
return {}
-class Book(MetaInformation):
+class Book(Metadata):
'''
A simple class describing a book in the iTunes Books Library.
- See ebooks.metadata.__init__ for all fields
'''
def __init__(self,title,author):
- MetaInformation.__init__(self, title, authors=[author])
+ Metadata.__init__(self, title, authors=[author])
@dynamic_property
def title_sorter(self):
diff --git a/src/calibre/devices/interface.py b/src/calibre/devices/interface.py
index 1384fa03d9..7783173c9c 100644
--- a/src/calibre/devices/interface.py
+++ b/src/calibre/devices/interface.py
@@ -316,7 +316,7 @@ class DevicePlugin(Plugin):
being uploaded to the device.
:param names: A list of file names that the books should have
once uploaded to the device. len(names) == len(files)
- :param metadata: If not None, it is a list of :class:`MetaInformation` objects.
+ :param metadata: If not None, it is a list of :class:`Metadata` objects.
The idea is to use the metadata to determine where on the device to
put the book. len(metadata) == len(files). Apart from the regular
cover (path to cover), there may also be a thumbnail attribute, which should
@@ -335,7 +335,7 @@ class DevicePlugin(Plugin):
the device.
:param locations: Result of a call to L{upload_books}
- :param metadata: List of :class:`MetaInformation` objects, same as for
+ :param metadata: List of :class:`Metadata` objects, same as for
:meth:`upload_books`.
:param booklists: A tuple containing the result of calls to
(:meth:`books(oncard=None)`,
diff --git a/src/calibre/devices/kobo/books.py b/src/calibre/devices/kobo/books.py
index a5b2e98d2f..11ab42c29f 100644
--- a/src/calibre/devices/kobo/books.py
+++ b/src/calibre/devices/kobo/books.py
@@ -7,11 +7,11 @@ import os
import re
import time
-from calibre.ebooks.metadata import MetaInformation
+from calibre.ebooks.metadata.book.base import Metadata
from calibre.constants import filesystem_encoding, preferred_encoding
from calibre import isbytestring
-class Book(MetaInformation):
+class Book(Metadata):
BOOK_ATTRS = ['lpath', 'size', 'mime', 'device_collections', '_new_book']
@@ -23,9 +23,9 @@ class Book(MetaInformation):
'uuid',
]
- def __init__(self, prefix, lpath, title, authors, mime, date, ContentType, thumbnail_name, other=None):
-
- MetaInformation.__init__(self, '')
+ def __init__(self, prefix, lpath, title, authors, mime, date, ContentType,
+ thumbnail_name, other=None):
+ Metadata.__init__(self, '')
self.device_collections = []
self._new_book = False
@@ -34,7 +34,7 @@ class Book(MetaInformation):
self.path = self.path.replace('/', '\\')
self.lpath = lpath.replace('\\', '/')
else:
- self.lpath = lpath
+ self.lpath = lpath
self.title = title
if not authors:
@@ -52,10 +52,9 @@ class Book(MetaInformation):
else:
self.datetime = time.gmtime(os.path.getctime(self.path))
except:
- self.datetime = time.gmtime()
-
- if thumbnail_name is not None:
- self.thumbnail = ImageWrapper(thumbnail_name)
+ self.datetime = time.gmtime()
+ if thumbnail_name is not None:
+ self.thumbnail = ImageWrapper(thumbnail_name)
self.tags = []
if other:
self.smart_update(other)
@@ -90,7 +89,7 @@ class Book(MetaInformation):
in C{other} takes precedence, unless the information in C{other} is NULL.
'''
- MetaInformation.smart_update(self, other)
+ Metadata.smart_update(self, other)
for attr in self.BOOK_ATTRS:
if hasattr(other, attr):
diff --git a/src/calibre/devices/usbms/books.py b/src/calibre/devices/usbms/books.py
index 959f26199c..e3c405ee4e 100644
--- a/src/calibre/devices/usbms/books.py
+++ b/src/calibre/devices/usbms/books.py
@@ -6,29 +6,18 @@ __docformat__ = 'restructuredtext en'
import os, re, time, sys
-from calibre.ebooks.metadata import MetaInformation
+from calibre.ebooks.metadata.book.base import Metadata
from calibre.devices.mime import mime_type_ext
from calibre.devices.interface import BookList as _BookList
from calibre.constants import filesystem_encoding, preferred_encoding
from calibre import isbytestring
from calibre.utils.config import prefs
-class Book(MetaInformation):
-
- BOOK_ATTRS = ['lpath', 'size', 'mime', 'device_collections', '_new_book']
-
- JSON_ATTRS = [
- 'lpath', 'title', 'authors', 'mime', 'size', 'tags', 'author_sort',
- 'title_sort', 'comments', 'category', 'publisher', 'series',
- 'series_index', 'rating', 'isbn', 'language', 'application_id',
- 'book_producer', 'lccn', 'lcc', 'ddc', 'rights', 'publication_type',
- 'uuid',
- ]
-
+class Book(Metadata):
def __init__(self, prefix, lpath, size=None, other=None):
from calibre.ebooks.metadata.meta import path_to_ext
- MetaInformation.__init__(self, '')
+ Metadata.__init__(self, '')
self._new_book = False
self.device_collections = []
@@ -72,32 +61,6 @@ class Book(MetaInformation):
def thumbnail(self):
return None
- def smart_update(self, other, replace_metadata=False):
- '''
- Merge the information in C{other} into self. In case of conflicts, the information
- in C{other} takes precedence, unless the information in C{other} is NULL.
- '''
-
- MetaInformation.smart_update(self, other, replace_metadata)
-
- for attr in self.BOOK_ATTRS:
- if hasattr(other, attr):
- val = getattr(other, attr, None)
- setattr(self, attr, val)
-
- def to_json(self):
- json = {}
- for attr in self.JSON_ATTRS:
- val = getattr(self, attr)
- if isbytestring(val):
- enc = filesystem_encoding if attr == 'lpath' else preferred_encoding
- val = val.decode(enc, 'replace')
- elif isinstance(val, (list, tuple)):
- val = [x.decode(preferred_encoding, 'replace') if
- isbytestring(x) else x for x in val]
- json[attr] = val
- return json
-
class BookList(_BookList):
def __init__(self, oncard, prefix, settings):
diff --git a/src/calibre/devices/usbms/driver.py b/src/calibre/devices/usbms/driver.py
index 0d28f06f49..a0d1d9dbf8 100644
--- a/src/calibre/devices/usbms/driver.py
+++ b/src/calibre/devices/usbms/driver.py
@@ -13,7 +13,6 @@ for a particular device.
import os
import re
import time
-import json
from itertools import cycle
from calibre import prints, isbytestring
@@ -21,6 +20,7 @@ from calibre.constants import filesystem_encoding, DEBUG
from calibre.devices.usbms.cli import CLI
from calibre.devices.usbms.device import Device
from calibre.devices.usbms.books import BookList, Book
+from calibre.ebooks.metadata.book.json_codec import JsonCodec
BASE_TIME = None
def debug_print(*args):
@@ -288,6 +288,7 @@ class USBMS(CLI, Device):
# at the end just before the return
def sync_booklists(self, booklists, end_session=True):
debug_print('USBMS: starting sync_booklists')
+ json_codec = JsonCodec()
if not os.path.exists(self.normalize_path(self._main_prefix)):
os.makedirs(self.normalize_path(self._main_prefix))
@@ -296,10 +297,8 @@ class USBMS(CLI, Device):
if prefix is not None and isinstance(booklists[listid], self.booklist_class):
if not os.path.exists(prefix):
os.makedirs(self.normalize_path(prefix))
- js = [item.to_json() for item in booklists[listid] if
- hasattr(item, 'to_json')]
with open(self.normalize_path(os.path.join(prefix, self.METADATA_CACHE)), 'wb') as f:
- f.write(json.dumps(js, indent=2, encoding='utf-8'))
+ json_codec.encode_to_file(f, booklists[listid])
write_prefix(self._main_prefix, 0)
write_prefix(self._card_a_prefix, 1)
write_prefix(self._card_b_prefix, 2)
@@ -345,19 +344,13 @@ class USBMS(CLI, Device):
@classmethod
def parse_metadata_cache(cls, bl, prefix, name):
- # bl = cls.booklist_class()
- js = []
+ json_codec = JsonCodec()
need_sync = False
cache_file = cls.normalize_path(os.path.join(prefix, name))
if os.access(cache_file, os.R_OK):
try:
with open(cache_file, 'rb') as f:
- js = json.load(f, encoding='utf-8')
- for item in js:
- book = cls.book_class(prefix, item.get('lpath', None))
- for key in item.keys():
- setattr(book, key, item[key])
- bl.append(book)
+ json_codec.decode_from_file(f, bl, cls.book_class, prefix)
except:
import traceback
traceback.print_exc()
@@ -392,7 +385,7 @@ class USBMS(CLI, Device):
@classmethod
def book_from_path(cls, prefix, lpath):
- from calibre.ebooks.metadata import MetaInformation
+ from calibre.ebooks.metadata.book.base import Metadata
if cls.settings().read_metadata or cls.MUST_READ_METADATA:
mi = cls.metadata_from_path(cls.normalize_path(os.path.join(prefix, lpath)))
@@ -401,7 +394,7 @@ class USBMS(CLI, Device):
mi = metadata_from_filename(cls.normalize_path(os.path.basename(lpath)),
cls.build_template_regexp())
if mi is None:
- mi = MetaInformation(os.path.splitext(os.path.basename(lpath))[0],
+ mi = Metadata(os.path.splitext(os.path.basename(lpath))[0],
[_('Unknown')])
size = os.stat(cls.normalize_path(os.path.join(prefix, lpath))).st_size
book = cls.book_class(prefix, lpath, other=mi, size=size)
diff --git a/src/calibre/ebooks/metadata/__init__.py b/src/calibre/ebooks/metadata/__init__.py
index d4a21e2c8c..fb894d3bbd 100644
--- a/src/calibre/ebooks/metadata/__init__.py
+++ b/src/calibre/ebooks/metadata/__init__.py
@@ -221,214 +221,18 @@ class ResourceCollection(object):
-class MetaInformation(object):
- '''Convenient encapsulation of book metadata'''
-
- @staticmethod
- def copy(mi):
- ans = MetaInformation(mi.title, mi.authors)
- for attr in ('author_sort', 'title_sort', 'comments', 'category',
- 'publisher', 'series', 'series_index', 'rating',
- 'isbn', 'tags', 'cover_data', 'application_id', 'guide',
- 'manifest', 'spine', 'toc', 'cover', 'language',
- 'book_producer', 'timestamp', 'lccn', 'lcc', 'ddc',
- 'author_sort_map',
- 'pubdate', 'rights', 'publication_type', 'uuid'):
- if hasattr(mi, attr):
- setattr(ans, attr, getattr(mi, attr))
-
- def __init__(self, title, authors=(_('Unknown'),)):
- '''
+def MetaInformation(title, authors=(_('Unknown'),)):
+ ''' Convenient encapsulation of book metadata, needed for compatibility
@param title: title or ``_('Unknown')`` or a MetaInformation object
@param authors: List of strings or []
- '''
- mi = None
- if hasattr(title, 'title') and hasattr(title, 'authors'):
- mi = title
- title = mi.title
- authors = mi.authors
- self.title = title
- self.author = list(authors) if authors else []# Needed for backward compatibility
- #: List of strings or []
- self.authors = list(authors) if authors else []
- self.tags = getattr(mi, 'tags', [])
- #: mi.cover_data = (ext, data)
- self.cover_data = getattr(mi, 'cover_data', (None, None))
- self.author_sort_map = getattr(mi, 'author_sort_map', {})
-
- for x in ('author_sort', 'title_sort', 'comments', 'category', 'publisher',
- 'series', 'series_index', 'rating', 'isbn', 'language',
- 'application_id', 'manifest', 'toc', 'spine', 'guide', 'cover',
- 'book_producer', 'timestamp', 'lccn', 'lcc', 'ddc', 'pubdate',
- 'rights', 'publication_type', 'uuid',
- ):
- setattr(self, x, getattr(mi, x, None))
-
- def print_all_attributes(self):
- for x in ('title','author', 'author_sort', 'title_sort', 'comments', 'category', 'publisher',
- 'series', 'series_index', 'tags', 'rating', 'isbn', 'language',
- 'application_id', 'manifest', 'toc', 'spine', 'guide', 'cover',
- 'book_producer', 'timestamp', 'lccn', 'lcc', 'ddc', 'pubdate',
- 'rights', 'publication_type', 'uuid', 'author_sort_map'
- ):
- prints(x, getattr(self, x, 'None'))
-
- def smart_update(self, mi, replace_metadata=False):
- '''
- Merge the information in C{mi} into self. In case of conflicts, the
- information in C{mi} takes precedence, unless the information in mi is
- NULL. If replace_metadata is True, then the information in mi always
- takes precedence.
- '''
- if mi.title and mi.title != _('Unknown'):
- self.title = mi.title
-
- if mi.authors and mi.authors[0] != _('Unknown'):
- self.authors = mi.authors
-
- for attr in ('author_sort', 'title_sort', 'category',
- 'publisher', 'series', 'series_index', 'rating',
- 'isbn', 'application_id', 'manifest', 'spine', 'toc',
- 'cover', 'guide', 'book_producer',
- 'timestamp', 'lccn', 'lcc', 'ddc', 'pubdate', 'rights',
- 'publication_type', 'uuid'):
- if replace_metadata:
- setattr(self, attr, getattr(mi, attr, 1.0 if \
- attr == 'series_index' else None))
- elif hasattr(mi, attr):
- val = getattr(mi, attr)
- if val is not None:
- setattr(self, attr, val)
-
- if replace_metadata:
- self.tags = mi.tags
- elif mi.tags:
- self.tags += mi.tags
- self.tags = list(set(self.tags))
-
- if mi.author_sort_map:
- self.author_sort_map.update(mi.author_sort_map)
-
- if getattr(mi, 'cover_data', False):
- other_cover = mi.cover_data[-1]
- self_cover = self.cover_data[-1] if self.cover_data else ''
- if not self_cover: self_cover = ''
- if not other_cover: other_cover = ''
- if len(other_cover) > len(self_cover):
- self.cover_data = mi.cover_data
-
- if replace_metadata:
- self.comments = getattr(mi, 'comments', '')
- else:
- my_comments = getattr(self, 'comments', '')
- other_comments = getattr(mi, 'comments', '')
- if not my_comments:
- my_comments = ''
- if not other_comments:
- other_comments = ''
- if len(other_comments.strip()) > len(my_comments.strip()):
- self.comments = other_comments
-
- other_lang = getattr(mi, 'language', None)
- if other_lang and other_lang.lower() != 'und':
- self.language = other_lang
-
-
- def format_series_index(self):
- try:
- x = float(self.series_index)
- except ValueError:
- x = 1
- return fmt_sidx(x)
-
- def authors_from_string(self, raw):
- self.authors = string_to_authors(raw)
-
- def format_authors(self):
- return authors_to_string(self.authors)
-
- def format_tags(self):
- return u', '.join([unicode(t) for t in self.tags])
-
- def format_rating(self):
- return unicode(self.rating)
-
- def __unicode__(self):
- ans = []
- def fmt(x, y):
- ans.append(u'%-20s: %s'%(unicode(x), unicode(y)))
-
- fmt('Title', self.title)
- if self.title_sort:
- fmt('Title sort', self.title_sort)
- if self.authors:
- fmt('Author(s)', authors_to_string(self.authors) + \
- ((' [' + self.author_sort + ']') if self.author_sort else ''))
- if self.publisher:
- fmt('Publisher', self.publisher)
- if getattr(self, 'book_producer', False):
- fmt('Book Producer', self.book_producer)
- if self.category:
- fmt('Category', self.category)
- if self.comments:
- fmt('Comments', self.comments)
- if self.isbn:
- fmt('ISBN', self.isbn)
- if self.tags:
- fmt('Tags', u', '.join([unicode(t) for t in self.tags]))
- if self.series:
- fmt('Series', self.series + ' #%s'%self.format_series_index())
- if self.language:
- fmt('Language', self.language)
- if self.rating is not None:
- fmt('Rating', self.rating)
- if self.timestamp is not None:
- fmt('Timestamp', isoformat(self.timestamp))
- if self.pubdate is not None:
- fmt('Published', isoformat(self.pubdate))
- if self.rights is not None:
- fmt('Rights', unicode(self.rights))
- if self.lccn:
- fmt('LCCN', unicode(self.lccn))
- if self.lcc:
- fmt('LCC', unicode(self.lcc))
- if self.ddc:
- fmt('DDC', unicode(self.ddc))
-
- return u'\n'.join(ans)
-
- def to_html(self):
- ans = [(_('Title'), unicode(self.title))]
- ans += [(_('Author(s)'), (authors_to_string(self.authors) if self.authors else _('Unknown')))]
- ans += [(_('Publisher'), unicode(self.publisher))]
- ans += [(_('Producer'), unicode(self.book_producer))]
- ans += [(_('Comments'), unicode(self.comments))]
- ans += [('ISBN', unicode(self.isbn))]
- if self.lccn:
- ans += [('LCCN', unicode(self.lccn))]
- if self.lcc:
- ans += [('LCC', unicode(self.lcc))]
- if self.ddc:
- ans += [('DDC', unicode(self.ddc))]
- ans += [(_('Tags'), u', '.join([unicode(t) for t in self.tags]))]
- if self.series:
- ans += [(_('Series'), unicode(self.series)+ ' #%s'%self.format_series_index())]
- ans += [(_('Language'), unicode(self.language))]
- if self.timestamp is not None:
- ans += [(_('Timestamp'), unicode(self.timestamp.isoformat(' ')))]
- if self.pubdate is not None:
- ans += [(_('Published'), unicode(self.pubdate.isoformat(' ')))]
- if self.rights is not None:
- ans += [(_('Rights'), unicode(self.rights))]
- for i, x in enumerate(ans):
- ans[i] = u'
%s
%s
'%x
- return u'
%s
'%u'\n'.join(ans)
-
- def __str__(self):
- return self.__unicode__().encode('utf-8')
-
- def __nonzero__(self):
- return bool(self.title or self.author or self.comments or self.tags)
+ '''
+ from calibre.ebooks.metadata.book.base import Metadata
+ mi = None
+ if hasattr(title, 'title') and hasattr(title, 'authors'):
+ mi = title
+ title = mi.title
+ authors = mi.authors
+ return Metadata(title, authors, mi)
def check_isbn10(isbn):
try:
diff --git a/src/calibre/ebooks/metadata/book/__init__.py b/src/calibre/ebooks/metadata/book/__init__.py
index c3b95f1188..9de7ca1c6b 100644
--- a/src/calibre/ebooks/metadata/book/__init__.py
+++ b/src/calibre/ebooks/metadata/book/__init__.py
@@ -24,6 +24,8 @@ SOCIAL_METADATA_FIELDS = frozenset([
# For example: {'isbn':'123456789', 'doi':'xxxx', ... }
'classifiers',
'isbn', # Pseudo field for convenience, should get/set isbn classifier
+ # TODO: not sure what this is, but it is used by OPF
+ 'category',
])
@@ -69,7 +71,8 @@ BOOK_STRUCTURE_FIELDS = frozenset([
])
USER_METADATA_FIELDS = frozenset([
- # A dict of a form to be specified
+ # A dict of dicts similar to field_metadata. Each field description dict
+ # also contains a value field with the key #value#.
'user_metadata',
])
@@ -86,16 +89,42 @@ DEVICE_METADATA_FIELDS = frozenset([
CALIBRE_METADATA_FIELDS = frozenset([
# An application id
# Semantics to be defined. Is it a db key? a db name + key? A uuid?
+ # (It is currently set to the db_id.)
'application_id',
+ # the calibre primary key of the item. May want to remove this once Sony's no longer use it
+ 'db_id',
]
)
+ALL_METADATA_FIELDS = SOCIAL_METADATA_FIELDS.union(
+ PUBLICATION_METADATA_FIELDS).union(
+ BOOK_STRUCTURE_FIELDS).union(
+ USER_METADATA_FIELDS).union(
+ DEVICE_METADATA_FIELDS).union(
+ CALIBRE_METADATA_FIELDS)
-SERIALIZABLE_FIELDS = SOCIAL_METADATA_FIELDS.union(
- USER_METADATA_FIELDS).union(
- PUBLICATION_METADATA_FIELDS).union(
- CALIBRE_METADATA_FIELDS).union(
- frozenset(['lpath'])) # I don't think we need device_collections
+# All fields except custom fields
+STANDARD_METADATA_FIELDS = SOCIAL_METADATA_FIELDS.union(
+ PUBLICATION_METADATA_FIELDS).union(
+ BOOK_STRUCTURE_FIELDS).union(
+ DEVICE_METADATA_FIELDS).union(
+ CALIBRE_METADATA_FIELDS)
+
+# Metadata fields that smart update should copy without special handling
+COPYABLE_METADATA_FIELDS = SOCIAL_METADATA_FIELDS.union(
+ PUBLICATION_METADATA_FIELDS).union(
+ BOOK_STRUCTURE_FIELDS).union(
+ DEVICE_METADATA_FIELDS).union(
+ CALIBRE_METADATA_FIELDS) - \
+ frozenset(['title', 'authors', 'comments', 'cover_data'])
+
+SERIALIZABLE_FIELDS = SOCIAL_METADATA_FIELDS.union(
+ USER_METADATA_FIELDS).union(
+ PUBLICATION_METADATA_FIELDS).union(
+ CALIBRE_METADATA_FIELDS).union(
+ DEVICE_METADATA_FIELDS) - \
+ frozenset(['device_collections'])
+ # I don't think we need device_collections
# Serialization of covers/thumbnails will have to be handled carefully, maybe
# as an option to the serializer class
diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index 3fed47091f..697de8d890 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -6,8 +6,13 @@ __copyright__ = '2010, Kovid Goyal '
__docformat__ = 'restructuredtext en'
import copy
+import traceback
+
+from calibre import prints
+from calibre.ebooks.metadata.book import COPYABLE_METADATA_FIELDS
+from calibre.ebooks.metadata.book import STANDARD_METADATA_FIELDS
+from calibre.utils.date import isoformat
-from calibre.ebooks.metadata.book import RESERVED_METADATA_FIELDS
NULL_VALUES = {
'user_metadata': {},
@@ -24,98 +29,313 @@ NULL_VALUES = {
class Metadata(object):
'''
- This class must expose a superset of the API of MetaInformation in terms
- of attribute access and methods. Only the __init__ method is different.
- MetaInformation will simply become a function that creates and fills in
- the attributes of this class.
+ A class representing all the metadata for a book.
Please keep the method based API of this class to a minimum. Every method
becomes a reserved field name.
'''
- def __init__(self):
+ def __init__(self, title, authors=(_('Unknown'),), other=None):
+ '''
+ @param title: title or ``_('Unknown')``
+ @param authors: List of strings or []
+ @param other: None or a metadata object
+ '''
object.__setattr__(self, '_data', copy.deepcopy(NULL_VALUES))
+ if other is not None:
+ self.smart_update(other)
+ else:
+ if title:
+ self.title = title
+ if authors:
+ #: List of strings or []
+ self.author = list(authors) if authors else []# Needed for backward compatibility
+ self.authors = list(authors) if authors else []
def __getattribute__(self, field):
_data = object.__getattribute__(self, '_data')
- if field in RESERVED_METADATA_FIELDS:
+ if field in STANDARD_METADATA_FIELDS:
return _data.get(field, None)
try:
return object.__getattribute__(self, field)
except AttributeError:
pass
if field in _data['user_metadata'].iterkeys():
- # TODO: getting user metadata values
- pass
+ return _data['user_metadata'][field]['#value#']
raise AttributeError(
'Metadata object has no attribute named: '+ repr(field))
-
def __setattr__(self, field, val):
_data = object.__getattribute__(self, '_data')
- if field in RESERVED_METADATA_FIELDS:
- if field != 'user_metadata':
- if not val:
- val = NULL_VALUES[field]
- _data[field] = val
- else:
- raise AttributeError('You cannot set user_metadata directly.')
+ if field in STANDARD_METADATA_FIELDS:
+ if not val:
+ val = NULL_VALUES.get(field, None)
+ _data[field] = val
elif field in _data['user_metadata'].iterkeys():
- # TODO: Setting custom column values
- pass
+ _data['user_metadata'][field]['#value#'] = val
else:
# You are allowed to stick arbitrary attributes onto this object as
# long as they dont conflict with global or user metadata names
# Don't abuse this privilege
self.__dict__[field] = val
+ def get(self, field):
+ return self.__getattribute__(field)
+
+ def set(self, field, val):
+ self.__setattr__(field, val)
+
@property
- def user_metadata_names(self):
+ def user_metadata_keys(self):
'The set of user metadata names this object knows about'
_data = object.__getattribute__(self, '_data')
return frozenset(_data['user_metadata'].iterkeys())
- # Old MetaInformation API {{{
- def copy(self):
- pass
+ @property
+ def all_user_metadata(self):
+ '''
+ return a dict containing all the custom field metadata associated with
+ the book. Return a deep copy, just in case the user wants to change
+ values in the dict (json does).
+ '''
+ _data = object.__getattribute__(self, '_data')
+ _data = _data['user_metadata']
+ res = {}
+ for k in _data:
+ res[k] = copy.deepcopy(_data[k])
+ return res
+
+ def get_user_metadata(self, field):
+ '''
+ return field metadata from the object if it is there. Otherwise return
+ None. field is the key name, not the label. Return a shallow copy,
+ just in case the user wants to change values in the dict (json does).
+ '''
+ _data = object.__getattribute__(self, '_data')
+ _data = _data['user_metadata']
+ if field in _data:
+ return copy.deepcopy(_data[field])
+ return None
+
+ def set_all_user_metadata(self, metadata):
+ '''
+ store custom field metadata into the object. Field is the key name
+ not the label
+ '''
+ if metadata is None:
+ traceback.print_stack()
+ else:
+ for key in metadata:
+ self.set_user_metadata(key, metadata[key])
+
+ def set_user_metadata(self, field, metadata):
+ '''
+ store custom field metadata for one column into the object. Field is
+ the key name not the label
+ '''
+ if field is not None:
+ if metadata is None:
+ traceback.print_stack()
+ metadata = copy.deepcopy(metadata)
+ if '#value#' not in metadata:
+ metadata['#value#'] = None
+ _data = object.__getattribute__(self, '_data')
+ _data['user_metadata'][field] = metadata
+
+ @property
+ def all_attributes(self):
+ result = {}
+ _data = object.__getattribute__(self, '_data')
+ for attr in STANDARD_METADATA_FIELDS:
+ v = _data.get(attr, None)
+ if v is not None:
+ result[attr] = v
+ for attr in self.user_metadata_keys:
+ if self.get(attr) is not None:
+ result[attr] = self.get(attr)
+ return result
+
+ # Old Metadata API {{{
+ @staticmethod
+ def copy(mi):
+ ans = Metadata(mi.title, mi.authors)
+ for attr in STANDARD_METADATA_FIELDS:
+ if hasattr(mi, attr):
+ setattr(ans, attr, copy.deepcopy(getattr(mi, attr)))
+ for x in mi.user_metadata_keys:
+ meta = mi.get_user_metadata(x)
+ if meta is not None:
+ ans.set_user_metadata(x, meta) # get... did the deep copy
def print_all_attributes(self):
- pass
+ for x in STANDARD_METADATA_FIELDS:
+ prints('%s:'%x, getattr(self, x, 'None'))
+ for x in self.user_metadata_keys:
+ meta = self.get_user_metadata(x)
+ if meta is not None:
+ prints(x, meta)
+ prints('--------------')
def smart_update(self, other, replace_metadata=False):
- pass
+ '''
+ Merge the information in C{other} into self. In case of conflicts, the information
+ in C{other} takes precedence, unless the information in other is NULL.
+ '''
+ if other.title and other.title != _('Unknown'):
+ self.title = other.title
+
+ if other.authors and other.authors[0] != _('Unknown'):
+ self.authors = other.authors
+
+ for attr in COPYABLE_METADATA_FIELDS:
+ if replace_metadata:
+ setattr(self, attr, getattr(other, attr, 1.0 if \
+ attr == 'series_index' else None))
+ elif hasattr(other, attr):
+ val = getattr(other, attr)
+ if val is not None:
+ setattr(self, attr, copy.deepcopy(val))
+
+ if replace_metadata:
+ self.tags = other.tags
+ elif other.tags:
+ self.tags += other.tags
+ self.tags = list(set(self.tags))
+
+ if getattr(other, 'author_sort_map', None):
+ self.author_sort_map.update(other.author_sort_map)
+
+ if getattr(other, 'cover_data', False):
+ other_cover = other.cover_data[-1]
+ self_cover = self.cover_data[-1] if self.cover_data else ''
+ if not self_cover: self_cover = ''
+ if not other_cover: other_cover = ''
+ if len(other_cover) > len(self_cover):
+ self.cover_data = other.cover_data
+
+ if getattr(other, 'user_metadata_keys', None):
+ for x in other.user_metadata_keys:
+ meta = other.get_user_metadata(x)
+ if meta is not None or replace_metadata:
+ self.set_user_metadata(x, meta) # get... did the deepcopy
+
+ if replace_metadata:
+ self.comments = getattr(other, 'comments', '')
+ else:
+ my_comments = getattr(self, 'comments', '')
+ other_comments = getattr(other, 'comments', '')
+ if not my_comments:
+ my_comments = ''
+ if not other_comments:
+ other_comments = ''
+ if len(other_comments.strip()) > len(my_comments.strip()):
+ self.comments = other_comments
+
+ other_lang = getattr(other, 'language', None)
+ if other_lang and other_lang.lower() != 'und':
+ self.language = other_lang
+
def format_series_index(self):
- pass
+ from calibre.ebooks.metadata import fmt_sidx
+ try:
+ x = float(self.series_index)
+ except ValueError:
+ x = 1
+ return fmt_sidx(x)
def authors_from_string(self, raw):
- pass
+ from calibre.ebooks.metadata import string_to_authors
+ self.authors = string_to_authors(raw)
def format_authors(self):
- pass
+ from calibre.ebooks.metadata import authors_to_string
+ return authors_to_string(self.authors)
def format_tags(self):
- pass
+ return u', '.join([unicode(t) for t in self.tags])
def format_rating(self):
return unicode(self.rating)
def __unicode__(self):
- pass
+ from calibre.ebooks.metadata import authors_to_string
+ ans = []
+ def fmt(x, y):
+ ans.append(u'%-20s: %s'%(unicode(x), unicode(y)))
+
+ fmt('Title', self.title)
+ if self.title_sort:
+ fmt('Title sort', self.title_sort)
+ if self.authors:
+ fmt('Author(s)', authors_to_string(self.authors) + \
+ ((' [' + self.author_sort + ']') if self.author_sort else ''))
+ if self.publisher:
+ fmt('Publisher', self.publisher)
+ if getattr(self, 'book_producer', False):
+ fmt('Book Producer', self.book_producer)
+ if self.category:
+ fmt('Category', self.category)
+ if self.comments:
+ fmt('Comments', self.comments)
+ if self.isbn:
+ fmt('ISBN', self.isbn)
+ if self.tags:
+ fmt('Tags', u', '.join([unicode(t) for t in self.tags]))
+ if self.series:
+ fmt('Series', self.series + ' #%s'%self.format_series_index())
+ if self.language:
+ fmt('Language', self.language)
+ if self.rating is not None:
+ fmt('Rating', self.rating)
+ if self.timestamp is not None:
+ fmt('Timestamp', isoformat(self.timestamp))
+ if self.pubdate is not None:
+ fmt('Published', isoformat(self.pubdate))
+ if self.rights is not None:
+ fmt('Rights', unicode(self.rights))
+ if self.lccn:
+ fmt('LCCN', unicode(self.lccn))
+ if self.lcc:
+ fmt('LCC', unicode(self.lcc))
+ if self.ddc:
+ fmt('DDC', unicode(self.ddc))
+ # CUSTFIELD: What to do about custom fields?
+ return u'\n'.join(ans)
def to_html(self):
- pass
+ from calibre.ebooks.metadata import authors_to_string
+ ans = [(_('Title'), unicode(self.title))]
+ ans += [(_('Author(s)'), (authors_to_string(self.authors) if self.authors else _('Unknown')))]
+ ans += [(_('Publisher'), unicode(self.publisher))]
+ ans += [(_('Producer'), unicode(self.book_producer))]
+ ans += [(_('Comments'), unicode(self.comments))]
+ ans += [('ISBN', unicode(self.isbn))]
+ if self.lccn:
+ ans += [('LCCN', unicode(self.lccn))]
+ if self.lcc:
+ ans += [('LCC', unicode(self.lcc))]
+ if self.ddc:
+ ans += [('DDC', unicode(self.ddc))]
+ ans += [(_('Tags'), u', '.join([unicode(t) for t in self.tags]))]
+ if self.series:
+ ans += [(_('Series'), unicode(self.series)+ ' #%s'%self.format_series_index())]
+ ans += [(_('Language'), unicode(self.language))]
+ if self.timestamp is not None:
+ ans += [(_('Timestamp'), unicode(self.timestamp.isoformat(' ')))]
+ if self.pubdate is not None:
+ ans += [(_('Published'), unicode(self.pubdate.isoformat(' ')))]
+ if self.rights is not None:
+ ans += [(_('Rights'), unicode(self.rights))]
+ for i, x in enumerate(ans):
+ ans[i] = u'
%s
%s
'%x
+ # CUSTFIELD: What to do about custom fields
+ return u'
%s
'%u'\n'.join(ans)
def __str__(self):
return self.__unicode__().encode('utf-8')
def __nonzero__(self):
- return True
+ return bool(self.title or self.author or self.comments or self.tags)
# }}}
-
-# We don't need reserved field names for this object any more. Lets just use a
-# protocol like the last char of a user field label should be _ when using this
-# object
-# So mi.tags returns the builtin tags and mi.tags_ returns the user tags
-
diff --git a/src/calibre/ebooks/metadata/book/json_codec.py b/src/calibre/ebooks/metadata/book/json_codec.py
new file mode 100644
index 0000000000..5e13650f0e
--- /dev/null
+++ b/src/calibre/ebooks/metadata/book/json_codec.py
@@ -0,0 +1,125 @@
+'''
+Created on 4 Jun 2010
+
+@author: charles
+'''
+
+from base64 import b64encode, b64decode
+import json
+import traceback
+from PIL import Image
+
+from . import SERIALIZABLE_FIELDS
+from calibre.constants import filesystem_encoding, preferred_encoding
+from calibre.library.field_metadata import FieldMetadata
+from calibre.utils.date import parse_date, isoformat, UNDEFINED_DATE
+
+# Translate datetimes to and from strings. The string form is the datetime in
+# UTC. The returned date is also UTC
+def string_to_datetime(src):
+ if src == "None":
+ return None
+# dt = strptime(src, '%d %m %Y %H:%M:%S', assume_utc=True, as_utc=True)
+# if dt == UNDEFINED_DATE:
+# return None
+ return parse_date(src)
+
+def datetime_to_string(dateval):
+ if dateval is None or dateval == UNDEFINED_DATE:
+ return "None"
+# tt = date_to_utc(dateval).timetuple()
+# res = "%02d %02d %04d %02d:%02d:%02d"%(tt.tm_mday, tt.tm_mon, tt.tm_year,
+# tt.tm_hour, tt.tm_min, tt.tm_sec)
+ return isoformat(dateval)
+
+def encode_thumbnail(thumbnail):
+ '''
+ Encode the image part of a thumbnail, then return the 3 part tuple
+ '''
+ if thumbnail is None:
+ return None
+ return (thumbnail[0], thumbnail[1], b64encode(str(thumbnail[2])))
+
+def decode_thumbnail(tup):
+ '''
+ Decode an encoded thumbnail into its 3 component parts
+ '''
+ if tup is None:
+ return None
+ return (tup[0], tup[1], b64decode(tup[2]))
+
+class JsonCodec(object):
+
+ def __init__(self):
+ self.field_metadata = FieldMetadata()
+
+ def encode_to_file(self, file, booklist):
+ json.dump(self.encode_booklist_metadata(booklist), file, indent=2, encoding='utf-8')
+
+ def encode_booklist_metadata(self, booklist):
+ result = []
+ for book in booklist:
+ result.append(self.encode_book_metadata(book))
+ return result
+
+ def encode_book_metadata(self, book):
+ result = {}
+ for key in SERIALIZABLE_FIELDS:
+ result[key] = self.encode_metadata_attr(book, key)
+ return result
+
+ def encode_metadata_attr(self, book, key):
+ if key == 'user_metadata':
+ meta = book.all_user_metadata
+ for k in meta:
+ if meta[k]['datatype'] == 'datetime':
+ meta[k]['#value#'] = datetime_to_string(meta[k]['#value#'])
+ return meta
+ if key in self.field_metadata:
+ datatype = self.field_metadata[key]['datatype']
+ else:
+ datatype = None
+ value = book.get(key)
+ if key == 'thumbnail':
+ return encode_thumbnail(value)
+ elif isinstance(value, str): # str includes bytes
+ enc = filesystem_encoding if key == 'lpath' else preferred_encoding
+ return value.decode(enc, 'replace')
+ elif isinstance(value, (list, tuple)):
+ return [x.decode(preferred_encoding, 'replace') if
+ isinstance(x, str) else x for x in value]
+ elif datatype == 'datetime':
+ return datetime_to_string(value)
+ else:
+ return value
+
+ def decode_from_file(self, file, booklist, book_class, prefix):
+ js = []
+ try:
+ js = json.load(file, encoding='utf-8')
+ for item in js:
+ book = book_class(prefix, item.get('lpath', None))
+ for key in item.keys():
+ meta = self.decode_metadata(key, item[key])
+ if key == 'user_metadata':
+ book.set_all_user_metadata(meta)
+ else:
+ setattr(book, key, meta)
+ booklist.append(book)
+ except:
+ print 'exception during JSON decoding'
+ traceback.print_exc()
+ booklist = []
+
+ def decode_metadata(self, key, value):
+ if key == 'user_metadata':
+ for k in value:
+ if value[k]['datatype'] == 'datetime':
+ value[k]['#value#'] = string_to_datetime(value[k]['#value#'])
+ return value
+ elif key in self.field_metadata:
+ if self.field_metadata[key]['datatype'] == 'datetime':
+ return string_to_datetime(value)
+ if key == 'thumbnail':
+ return decode_thumbnail(value)
+ return value
diff --git a/src/calibre/ebooks/metadata/isbndb.py b/src/calibre/ebooks/metadata/isbndb.py
index 356cc3f1b1..b5fc5830c8 100644
--- a/src/calibre/ebooks/metadata/isbndb.py
+++ b/src/calibre/ebooks/metadata/isbndb.py
@@ -8,7 +8,7 @@ import sys, re
from urllib import quote
from calibre.utils.config import OptionParser
-from calibre.ebooks.metadata import MetaInformation
+from calibre.ebooks.metadata.book.base import Metadata
from calibre.ebooks.BeautifulSoup import BeautifulStoneSoup
from calibre import browser
@@ -42,10 +42,10 @@ def fetch_metadata(url, max=100, timeout=5.):
return books
-class ISBNDBMetadata(MetaInformation):
+class ISBNDBMetadata(Metadata):
def __init__(self, book):
- MetaInformation.__init__(self, None, [])
+ Metadata.__init__(self, None, [])
self.isbn = book.get('isbn13', book.get('isbn'))
self.title = book.find('titlelong').string
diff --git a/src/calibre/ebooks/metadata/opf2.py b/src/calibre/ebooks/metadata/opf2.py
index f93b614ef2..0ab6d3bbc0 100644
--- a/src/calibre/ebooks/metadata/opf2.py
+++ b/src/calibre/ebooks/metadata/opf2.py
@@ -16,7 +16,8 @@ from lxml import etree
from calibre.ebooks.chardet import xml_to_unicode
from calibre.constants import __appname__, __version__, filesystem_encoding
from calibre.ebooks.metadata.toc import TOC
-from calibre.ebooks.metadata import MetaInformation, string_to_authors
+from calibre.ebooks.metadata import string_to_authors
+from calibre.ebooks.metadata.book.base import Metadata
from calibre.utils.date import parse_date, isoformat
from calibre.utils.localization import get_lang
@@ -926,16 +927,16 @@ class OPF(object):
setattr(self, attr, val)
-class OPFCreator(MetaInformation):
+class OPFCreator(Metadata):
- def __init__(self, base_path, *args, **kwargs):
+ def __init__(self, base_path, other):
'''
Initialize.
@param base_path: An absolute path to the directory in which this OPF file
will eventually be. This is used by the L{create_manifest} method
to convert paths to files into relative paths.
'''
- MetaInformation.__init__(self, *args, **kwargs)
+ Metadata.__init__(self, title='', other=other)
self.base_path = os.path.abspath(base_path)
if self.application_id is None:
self.application_id = str(uuid.uuid4())
diff --git a/src/calibre/gui2/dialogs/config/save_template.py b/src/calibre/gui2/dialogs/config/save_template.py
index 71eb15f4aa..2157d5b3bf 100644
--- a/src/calibre/gui2/dialogs/config/save_template.py
+++ b/src/calibre/gui2/dialogs/config/save_template.py
@@ -34,25 +34,24 @@ class SaveTemplate(QWidget, Ui_Form):
self.option_name = name
def validate(self):
- tmpl = preprocess_template(self.opt_template.text())
- fa = {}
- for x in FORMAT_ARG_DESCS.keys():
- fa[x]='random long string'
- try:
- tmpl.format(**fa)
- except Exception, err:
- error_dialog(self, _('Invalid template'),
- '
'+_('The template %s is invalid:')%tmpl + \
- ' '+str(err), show=True)
- return False
+ # TODO: I haven't figured out how to get the custom columns into here,
+ # so for the moment make all templates valid.
return True
+# tmpl = preprocess_template(self.opt_template.text())
+# fa = {}
+# for x in FORMAT_ARG_DESCS.keys():
+# fa[x]='random long string'
+# try:
+# tmpl.format(**fa)
+# except Exception, err:
+# error_dialog(self, _('Invalid template'),
+# '
'+_('The template %s is invalid:')%tmpl + \
+# ' '+str(err), show=True)
+# return False
+# return True
def save_settings(self, config, name):
val = unicode(self.opt_template.text())
config.set(name, val)
self.opt_template.save_history(self.option_name+'_template_history')
-
-
-
-
diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index 89008735fe..fdf21ecc23 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -372,7 +372,6 @@ class BooksModel(QAbstractTableModel): # {{{
return ans
def get_metadata(self, rows, rows_are_ids=False, full_metadata=False):
- # Should this add the custom columns? It doesn't at the moment
metadata, _full_metadata = [], []
if not rows_are_ids:
rows = [self.db.id(row.row()) for row in rows]
@@ -1053,7 +1052,7 @@ class DeviceBooksModel(BooksModel): # {{{
if hasattr(cdata, 'image_path'):
img.load(cdata.image_path)
else:
- img.loadFromData(cdata)
+ img.loadFromData(cdata[2])
if img.isNull():
img = self.default_image
data['cover'] = img
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index ef74188bdf..9f21fe0eda 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -509,15 +509,15 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
'''
Convenience method to return metadata as a L{MetaInformation} object.
'''
- aum = self.authors(idx, index_is_id=index_is_id)
- if aum: aum = [a.strip().replace('|', ',') for a in aum.split(',')]
+ aut_list = self.authors_with_sort_strings(idx, index_is_id=index_is_id)
+ aum = []
+ aus = {}
+ for (author, author_sort) in aut_list:
+ aum.append(author)
+ aus[author] = author_sort
mi = MetaInformation(self.title(idx, index_is_id=index_is_id), aum)
mi.author_sort = self.author_sort(idx, index_is_id=index_is_id)
- if mi.authors:
- mi.author_sort_map = {}
- for name, sort in zip(mi.authors, self.authors_sort_strings(idx,
- index_is_id)):
- mi.author_sort_map[name] = sort
+ mi.author_sort_map = aus
mi.comments = self.comments(idx, index_is_id=index_is_id)
mi.publisher = self.publisher(idx, index_is_id=index_is_id)
mi.timestamp = self.timestamp(idx, index_is_id=index_is_id)
@@ -534,6 +534,10 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
mi.isbn = self.isbn(idx, index_is_id=index_is_id)
id = idx if index_is_id else self.id(idx)
mi.application_id = id
+ for key,meta in self.field_metadata.iteritems():
+ if meta['is_custom']:
+ mi.set_user_metadata(key, meta)
+ mi.set(key, self.get_custom(idx, label=meta['label'], index_is_id=index_is_id))
if get_cover:
mi.cover = self.cover(id, index_is_id=True, as_path=True)
return mi
@@ -1049,6 +1053,19 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
result.append(sort)
return result
+ # Given a book, return the map of author sort strings for the book's authors
+ def authors_with_sort_strings(self, id, index_is_id=False):
+ id = id if index_is_id else self.id(id)
+ aut_strings = self.conn.get('''
+ SELECT authors.name, authors.sort
+ FROM authors, books_authors_link as bl
+ WHERE bl.book=? and authors.id=bl.author
+ ORDER BY bl.id''', (id,))
+ result = []
+ for (author, sort,) in aut_strings:
+ result.append((author.replace('|', ','), sort))
+ return result
+
# Given a book, return the author_sort string for authors of the book
def author_sort_from_book(self, id, index_is_id=False):
auts = self.authors_sort_strings(id, index_is_id)
diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py
index 15020855f7..258ea7ba9e 100644
--- a/src/calibre/library/save_to_disk.py
+++ b/src/calibre/library/save_to_disk.py
@@ -105,6 +105,8 @@ def safe_format(x, format_args):
pass
except AttributeError: # Thrown if user used a non existing attribute
pass
+ except KeyError: # Thrown if user used custom field w/value None
+ pass
return ''
def get_components(template, mi, id, timefmt='%b %Y', length=250,
@@ -113,13 +115,12 @@ def get_components(template, mi, id, timefmt='%b %Y', length=250,
library_order = tweaks['save_template_title_series_sorting'] == 'library_order'
tsfmt = title_sort if library_order else lambda x: x
format_args = dict(**FORMAT_ARGS)
+ format_args.update(mi.all_attributes)
if mi.title:
format_args['title'] = tsfmt(mi.title)
if mi.authors:
format_args['authors'] = mi.format_authors()
format_args['author'] = format_args['authors']
- if mi.author_sort:
- format_args['author_sort'] = mi.author_sort
if mi.tags:
format_args['tags'] = mi.format_tags()
if format_args['tags'].startswith('/'):
@@ -132,15 +133,21 @@ def get_components(template, mi, id, timefmt='%b %Y', length=250,
template = re.sub(r'\{series_index[^}]*?\}', '', template)
if mi.rating is not None:
format_args['rating'] = mi.format_rating()
- if mi.isbn:
- format_args['isbn'] = mi.isbn
- if mi.publisher:
- format_args['publisher'] = mi.publisher
if hasattr(mi.timestamp, 'timetuple'):
format_args['timestamp'] = strftime(timefmt, mi.timestamp.timetuple())
if hasattr(mi.pubdate, 'timetuple'):
format_args['pubdate'] = strftime(timefmt, mi.pubdate.timetuple())
format_args['id'] = str(id)
+
+ # These are not necessary any more. The values are set by
+ # 'format_args.update' above, and there is no special formatting
+# if mi.author_sort:
+# format_args['author_sort'] = mi.author_sort
+# if mi.isbn:
+# format_args['isbn'] = mi.isbn
+# if mi.publisher:
+# format_args['publisher'] = mi.publisher
+
components = [x.strip() for x in template.split('/') if x.strip()]
components = [safe_format(x, format_args) for x in components]
components = [sanitize_func(x) for x in components if x]
From b04faf70c2378c3569a4d1cf010da3d57a707c42 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Fri, 27 Aug 2010 08:22:08 +0100
Subject: [PATCH 002/207] Make Kobo driver use new metadata framework
---
src/calibre/devices/kobo/books.py | 80 ++----------------------------
src/calibre/devices/kobo/driver.py | 2 +-
2 files changed, 5 insertions(+), 77 deletions(-)
diff --git a/src/calibre/devices/kobo/books.py b/src/calibre/devices/kobo/books.py
index f0cf7c3763..1c3d05ea12 100644
--- a/src/calibre/devices/kobo/books.py
+++ b/src/calibre/devices/kobo/books.py
@@ -4,37 +4,15 @@ __copyright__ = '2010, Timothy Legge '
'''
import os
-import re
import time
-from calibre.ebooks.metadata.book.base import Metadata
-from calibre.constants import filesystem_encoding, preferred_encoding
-from calibre import isbytestring
+from calibre.devices.usbms.books import Book as Book_
-class Book(Metadata):
-
- BOOK_ATTRS = ['lpath', 'size', 'mime', 'device_collections', '_new_book']
-
- JSON_ATTRS = [
- 'lpath', 'title', 'authors', 'mime', 'size', 'tags', 'author_sort',
- 'title_sort', 'comments', 'category', 'publisher', 'series',
- 'series_index', 'rating', 'isbn', 'language', 'application_id',
- 'book_producer', 'lccn', 'lcc', 'ddc', 'rights', 'publication_type',
- 'uuid', 'device_collections',
- ]
+class Book(Book_):
def __init__(self, prefix, lpath, title, authors, mime, date, ContentType,
thumbnail_name, other=None):
- Metadata.__init__(self, '')
- self.device_collections = []
- self._new_book = False
-
- self.path = os.path.join(prefix, lpath)
- if os.sep == '\\':
- self.path = self.path.replace('/', '\\')
- self.lpath = lpath.replace('\\', '/')
- else:
- self.lpath = lpath
+ Book_.__init__(self, prefix, lpath)
self.title = title
if not authors:
@@ -59,57 +37,7 @@ class Book(Metadata):
if other:
self.smart_update(other)
- def __eq__(self, other):
- return self.path == getattr(other, 'path', None)
-
- @dynamic_property
- def db_id(self):
- doc = '''The database id in the application database that this file corresponds to'''
- def fget(self):
- match = re.search(r'_(\d+)$', self.lpath.rpartition('.')[0])
- if match:
- return int(match.group(1))
- return None
- return property(fget=fget, doc=doc)
-
- @dynamic_property
- def title_sorter(self):
- doc = '''String to sort the title. If absent, title is returned'''
- def fget(self):
- return re.sub('^\s*A\s+|^\s*The\s+|^\s*An\s+', '', self.title).rstrip()
- return property(doc=doc, fget=fget)
-
- @dynamic_property
- def thumbnail(self):
- return None
-
- def smart_update(self, other, replace_metadata=False):
- '''
- Merge the information in C{other} into self. In case of conflicts, the information
- in C{other} takes precedence, unless the information in C{other} is NULL.
- '''
-
- Metadata.smart_update(self, other)
-
- for attr in self.BOOK_ATTRS:
- if hasattr(other, attr):
- val = getattr(other, attr, None)
- setattr(self, attr, val)
-
- def to_json(self):
- json = {}
- for attr in self.JSON_ATTRS:
- val = getattr(self, attr)
- if isbytestring(val):
- enc = filesystem_encoding if attr == 'lpath' else preferred_encoding
- val = val.decode(enc, 'replace')
- elif isinstance(val, (list, tuple)):
- val = [x.decode(preferred_encoding, 'replace') if
- isbytestring(x) else x for x in val]
- json[attr] = val
- return json
-
class ImageWrapper(object):
def __init__(self, image_path):
- self.image_path = image_path
+ self.image_path = image_path
diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py
index 35fceb80f7..5f939a4498 100644
--- a/src/calibre/devices/kobo/driver.py
+++ b/src/calibre/devices/kobo/driver.py
@@ -132,7 +132,7 @@ class KOBO(USBMS):
changed = False
for i, row in enumerate(cursor):
- # self.report_progress((i+1) / float(numrows), _('Getting list of books on device...'))
+ # self.report_progress((i+1) / float(numrows), _('Getting list of books on device...'))
path = self.path_from_contentid(row[3], row[5], oncard)
mime = mime_type_ext(path_to_ext(row[3]))
From 47fedcee36db9d7f9f5ab39460a6eafcbe21df20 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Fri, 27 Aug 2010 10:26:36 +0100
Subject: [PATCH 003/207] Bug fixes: 1) Only reset to initial values when None
is assigned. Using 'if not var' is true for empty lists 2) Take unused values
out of the to_html and unicode functions 3) add 'language' as a valid
metadata field
---
src/calibre/ebooks/metadata/book/__init__.py | 68 +++++++-------------
src/calibre/ebooks/metadata/book/base.py | 31 +++++----
2 files changed, 42 insertions(+), 57 deletions(-)
diff --git a/src/calibre/ebooks/metadata/book/__init__.py b/src/calibre/ebooks/metadata/book/__init__.py
index 9de7ca1c6b..b1a322b143 100644
--- a/src/calibre/ebooks/metadata/book/__init__.py
+++ b/src/calibre/ebooks/metadata/book/__init__.py
@@ -11,50 +11,39 @@ an empty list/dictionary for complex types and (None, None) for cover_data
'''
SOCIAL_METADATA_FIELDS = frozenset([
- 'tags', # Ordered list
- # A floating point number between 0 and 10
- 'rating',
- # A simple HTML enabled string
- 'comments',
- # A simple string
- 'series',
- # A floating point number
- 'series_index',
+ 'tags', # Ordered list
+ 'rating', # A floating point number between 0 and 10
+ 'comments', # A simple HTML enabled string
+ 'series', # A simple string
+ 'series_index', # A floating point number
# Of the form { scheme1:value1, scheme2:value2}
# For example: {'isbn':'123456789', 'doi':'xxxx', ... }
'classifiers',
- 'isbn', # Pseudo field for convenience, should get/set isbn classifier
- # TODO: not sure what this is, but it is used by OPF
- 'category',
-
+ 'isbn', # Pseudo field for convenience, should get/set isbn classifier
+ 'category', # TODO: not sure what this is, but it is used by OPF
])
PUBLICATION_METADATA_FIELDS = frozenset([
- # title must never be None. Should be _('Unknown')
- 'title',
+ 'title', # title must never be None. Should be _('Unknown')
# Pseudo field that can be set, but if not set is auto generated
# from title and languages
'title_sort',
- # Ordered list of authors. Must never be None, can be [_('Unknown')]
- 'authors',
- # Map of sort strings for each author
- 'author_sort_map',
+ 'authors', # Ordered list. Must never be None, can be [_('Unknown')]
+ 'author_sort_map', # Map of sort strings for each author
# Pseudo field that can be set, but if not set is auto generated
# from authors and languages
'author_sort',
'book_producer',
- # Dates and times must be timezone aware
- 'timestamp',
+ 'timestamp', # Dates and times must be timezone aware
'pubdate',
'rights',
# So far only known publication type is periodical:calibre
# If None, means book
'publication_type',
- # A UUID usually of type 4
- 'uuid',
- 'languages', # ordered list
- # Simple string, no special semantics
- 'publisher',
+ 'uuid', # A UUID usually of type 4
+ 'language', # the primary language of this book
+ 'languages', # ordered list
+ 'publisher', # Simple string, no special semantics
# Absolute path to image file encoded in filesystem_encoding
'cover',
# Of the form (format, data) where format is, for e.g. 'jpeg', 'png', 'gif'...
@@ -77,22 +66,18 @@ USER_METADATA_FIELDS = frozenset([
])
DEVICE_METADATA_FIELDS = frozenset([
- # Ordered list of strings
- 'device_collections',
- 'lpath', # Unicode, / separated
- # In bytes
- 'size',
- # Mimetype of the book file being represented
- 'mime',
+ 'device_collections', # Ordered list of strings
+ 'lpath', # Unicode, / separated
+ 'size', # In bytes
+ 'mime', # Mimetype of the book file being represented
+
])
CALIBRE_METADATA_FIELDS = frozenset([
- # An application id
- # Semantics to be defined. Is it a db key? a db name + key? A uuid?
- # (It is currently set to the db_id.)
- 'application_id',
- # the calibre primary key of the item. May want to remove this once Sony's no longer use it
- 'db_id',
+ 'application_id', # An application id, currently set to the db_id.
+ # the calibre primary key of the item.
+ 'db_id', # the calibre primary key of the item.
+ # TODO: May want to remove once Sony's no longer use it
]
)
@@ -124,7 +109,4 @@ SERIALIZABLE_FIELDS = SOCIAL_METADATA_FIELDS.union(
CALIBRE_METADATA_FIELDS).union(
DEVICE_METADATA_FIELDS) - \
frozenset(['device_collections'])
- # I don't think we need device_collections
-
-# Serialization of covers/thumbnails will have to be handled carefully, maybe
-# as an option to the serializer class
+ # device_collections is rebuilt when needed
diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index 697de8d890..e352aecbf8 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -24,6 +24,7 @@ NULL_VALUES = {
'author_sort_map': {},
'authors' : [_('Unknown')],
'title' : _('Unknown'),
+ 'language' : 'und'
}
class Metadata(object):
@@ -68,14 +69,14 @@ class Metadata(object):
def __setattr__(self, field, val):
_data = object.__getattribute__(self, '_data')
if field in STANDARD_METADATA_FIELDS:
- if not val:
+ if val is None:
val = NULL_VALUES.get(field, None)
_data[field] = val
elif field in _data['user_metadata'].iterkeys():
_data['user_metadata'][field]['#value#'] = val
else:
# You are allowed to stick arbitrary attributes onto this object as
- # long as they dont conflict with global or user metadata names
+ # long as they don't conflict with global or user metadata names
# Don't abuse this privilege
self.__dict__[field] = val
@@ -294,12 +295,13 @@ class Metadata(object):
fmt('Published', isoformat(self.pubdate))
if self.rights is not None:
fmt('Rights', unicode(self.rights))
- if self.lccn:
- fmt('LCCN', unicode(self.lccn))
- if self.lcc:
- fmt('LCC', unicode(self.lcc))
- if self.ddc:
- fmt('DDC', unicode(self.ddc))
+# TODO: These are not in metadata. Should they be?
+# if self.lccn:
+# fmt('LCCN', unicode(self.lccn))
+# if self.lcc:
+# fmt('LCC', unicode(self.lcc))
+# if self.ddc:
+# fmt('DDC', unicode(self.ddc))
# CUSTFIELD: What to do about custom fields?
return u'\n'.join(ans)
@@ -311,12 +313,13 @@ class Metadata(object):
ans += [(_('Producer'), unicode(self.book_producer))]
ans += [(_('Comments'), unicode(self.comments))]
ans += [('ISBN', unicode(self.isbn))]
- if self.lccn:
- ans += [('LCCN', unicode(self.lccn))]
- if self.lcc:
- ans += [('LCC', unicode(self.lcc))]
- if self.ddc:
- ans += [('DDC', unicode(self.ddc))]
+# TODO: These are not in metadata. Should they be?
+# if self.lccn:
+# ans += [('LCCN', unicode(self.lccn))]
+# if self.lcc:
+# ans += [('LCC', unicode(self.lcc))]
+# if self.ddc:
+# ans += [('DDC', unicode(self.ddc))]
ans += [(_('Tags'), u', '.join([unicode(t) for t in self.tags]))]
if self.series:
ans += [(_('Series'), unicode(self.series)+ ' #%s'%self.format_series_index())]
From b4b0cb483df4a647de50145abf88a7b26ecd79c9 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sat, 28 Aug 2010 13:34:49 +0100
Subject: [PATCH 004/207] Changes to respond to Kovid's mail, and some
cleanups.
---
src/calibre/ebooks/metadata/book/__init__.py | 4 +-
src/calibre/ebooks/metadata/book/base.py | 132 +++++++++++-------
.../ebooks/metadata/book/json_codec.py | 2 +-
src/calibre/library/database2.py | 15 +-
src/calibre/library/save_to_disk.py | 13 +-
5 files changed, 100 insertions(+), 66 deletions(-)
diff --git a/src/calibre/ebooks/metadata/book/__init__.py b/src/calibre/ebooks/metadata/book/__init__.py
index b1a322b143..fbcca79aba 100644
--- a/src/calibre/ebooks/metadata/book/__init__.py
+++ b/src/calibre/ebooks/metadata/book/__init__.py
@@ -101,7 +101,9 @@ COPYABLE_METADATA_FIELDS = SOCIAL_METADATA_FIELDS.union(
BOOK_STRUCTURE_FIELDS).union(
DEVICE_METADATA_FIELDS).union(
CALIBRE_METADATA_FIELDS) - \
- frozenset(['title', 'authors', 'comments', 'cover_data'])
+ frozenset(['title', 'title_sort', 'authors',
+ 'author_sort', 'author_sort_map' 'comments',
+ 'cover_data', 'tags', 'language'])
SERIALIZABLE_FIELDS = SOCIAL_METADATA_FIELDS.union(
USER_METADATA_FIELDS).union(
diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index e352aecbf8..a81ce46c34 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -66,7 +66,7 @@ class Metadata(object):
raise AttributeError(
'Metadata object has no attribute named: '+ repr(field))
- def __setattr__(self, field, val):
+ def __setattr__(self, field, val, extra=None):
_data = object.__getattribute__(self, '_data')
if field in STANDARD_METADATA_FIELDS:
if val is None:
@@ -74,17 +74,23 @@ class Metadata(object):
_data[field] = val
elif field in _data['user_metadata'].iterkeys():
_data['user_metadata'][field]['#value#'] = val
+ _data['user_metadata'][field]['#extra#'] = extra
else:
# You are allowed to stick arbitrary attributes onto this object as
# long as they don't conflict with global or user metadata names
# Don't abuse this privilege
self.__dict__[field] = val
- def get(self, field):
+ def get(self, field, default=None):
+ if default is not None:
+ try:
+ return self.__getattribute__(field)
+ except AttributeError:
+ return default
return self.__getattribute__(field)
- def set(self, field, val):
- self.__setattr__(field, val)
+ def set(self, field, val, extra=None):
+ self.__setattr__(field, val, extra)
@property
def user_metadata_keys(self):
@@ -92,25 +98,25 @@ class Metadata(object):
_data = object.__getattribute__(self, '_data')
return frozenset(_data['user_metadata'].iterkeys())
- @property
- def all_user_metadata(self):
+ def get_all_user_metadata(self, make_copy):
'''
return a dict containing all the custom field metadata associated with
- the book. Return a deep copy, just in case the user wants to change
- values in the dict (json does).
+ the book.
'''
_data = object.__getattribute__(self, '_data')
- _data = _data['user_metadata']
+ user_metadata = _data['user_metadata']
+ if not make_copy:
+ return user_metadata
res = {}
- for k in _data:
- res[k] = copy.deepcopy(_data[k])
+ for k in user_metadata:
+ res[k] = copy.deepcopy(user_metadata[k])
return res
def get_user_metadata(self, field):
'''
return field metadata from the object if it is there. Otherwise return
- None. field is the key name, not the label. Return a shallow copy,
- just in case the user wants to change values in the dict (json does).
+ None. field is the key name, not the label. Return a copy, just in case
+ the user wants to change values in the dict (json does).
'''
_data = object.__getattribute__(self, '_data')
_data = _data['user_metadata']
@@ -118,6 +124,14 @@ class Metadata(object):
return copy.deepcopy(_data[field])
return None
+ @classmethod
+ def get_user_metadata_value(user_mi):
+ return user_mi['#value#']
+
+ @classmethod
+ def get_user_metadata_extra(user_mi):
+ return user_mi['#extra#']
+
def set_all_user_metadata(self, metadata):
'''
store custom field metadata into the object. Field is the key name
@@ -139,21 +153,30 @@ class Metadata(object):
traceback.print_stack()
metadata = copy.deepcopy(metadata)
if '#value#' not in metadata:
- metadata['#value#'] = None
+ if metadata['datatype'] == 'text' and metadata['is_multiple']:
+ metadata['#value#'] = []
+ else:
+ metadata['#value#'] = None
_data = object.__getattribute__(self, '_data')
_data['user_metadata'][field] = metadata
- @property
- def all_attributes(self):
+ def get_all_non_none_attributes(self):
+ '''
+ Return a dictionary containing all non-None metadata fields, including
+ the custom ones.
+ '''
result = {}
_data = object.__getattribute__(self, '_data')
for attr in STANDARD_METADATA_FIELDS:
v = _data.get(attr, None)
if v is not None:
result[attr] = v
- for attr in self.user_metadata_keys:
- if self.get(attr) is not None:
- result[attr] = self.get(attr)
+ for attr in _data['user_metadata'].iterkeys():
+ v = _data['user_metadata'][attr]['#value#']
+ if v is not None:
+ result[attr] = v
+ if _data['user_metadata'][attr]['datatype'] == 'series':
+ result[attr+'_index'] = _data['user_metadata'][attr]['#extra#']
return result
# Old Metadata API {{{
@@ -184,45 +207,49 @@ class Metadata(object):
'''
if other.title and other.title != _('Unknown'):
self.title = other.title
+ if hasattr(other, 'title_sort'):
+ self.title_sort = other.title_sort
if other.authors and other.authors[0] != _('Unknown'):
self.authors = other.authors
+ if hasattr(other, 'author_sort_map'):
+ self.author_sort_map = other.author_sort_map
+ if hasattr(other, 'author_sort'):
+ self.author_sort = other.author_sort
- for attr in COPYABLE_METADATA_FIELDS:
- if replace_metadata:
+ if replace_metadata:
+ for attr in COPYABLE_METADATA_FIELDS:
setattr(self, attr, getattr(other, attr, 1.0 if \
attr == 'series_index' else None))
- elif hasattr(other, attr):
- val = getattr(other, attr)
- if val is not None:
- setattr(self, attr, copy.deepcopy(val))
-
- if replace_metadata:
self.tags = other.tags
- elif other.tags:
- self.tags += other.tags
- self.tags = list(set(self.tags))
-
- if getattr(other, 'author_sort_map', None):
- self.author_sort_map.update(other.author_sort_map)
-
- if getattr(other, 'cover_data', False):
- other_cover = other.cover_data[-1]
- self_cover = self.cover_data[-1] if self.cover_data else ''
- if not self_cover: self_cover = ''
- if not other_cover: other_cover = ''
- if len(other_cover) > len(self_cover):
- self.cover_data = other.cover_data
-
- if getattr(other, 'user_metadata_keys', None):
- for x in other.user_metadata_keys:
- meta = other.get_user_metadata(x)
- if meta is not None or replace_metadata:
- self.set_user_metadata(x, meta) # get... did the deepcopy
-
- if replace_metadata:
+ self.cover_data = getattr(other, 'cover_data', '')
+ self.set_all_user_metadata(other.get_all_user_metadata(make_copy=True))
self.comments = getattr(other, 'comments', '')
+ self.language = getattr(other, 'language', None)
else:
+ for attr in COPYABLE_METADATA_FIELDS:
+ if hasattr(other, attr):
+ val = getattr(other, attr)
+ if val is not None:
+ setattr(self, attr, copy.deepcopy(val))
+
+ if other.tags:
+ self.tags += list(set(self.tags + other.tags))
+
+ if getattr(other, 'cover_data', False):
+ other_cover = other.cover_data[-1]
+ self_cover = self.cover_data[-1] if self.cover_data else ''
+ if not self_cover: self_cover = ''
+ if not other_cover: other_cover = ''
+ if len(other_cover) > len(self_cover):
+ self.cover_data = other.cover_data
+
+ if getattr(other, 'user_metadata_keys', None):
+ for x in other.user_metadata_keys:
+ meta = other.get_user_metadata(x)
+ if meta is not None:
+ self.set_user_metadata(x, meta) # get... did the deepcopy
+
my_comments = getattr(self, 'comments', '')
other_comments = getattr(other, 'comments', '')
if not my_comments:
@@ -232,10 +259,9 @@ class Metadata(object):
if len(other_comments.strip()) > len(my_comments.strip()):
self.comments = other_comments
- other_lang = getattr(other, 'language', None)
- if other_lang and other_lang.lower() != 'und':
- self.language = other_lang
-
+ other_lang = getattr(other, 'language', None)
+ if other_lang and other_lang.lower() != 'und':
+ self.language = other_lang
def format_series_index(self):
from calibre.ebooks.metadata import fmt_sidx
diff --git a/src/calibre/ebooks/metadata/book/json_codec.py b/src/calibre/ebooks/metadata/book/json_codec.py
index 5e13650f0e..7a80e16854 100644
--- a/src/calibre/ebooks/metadata/book/json_codec.py
+++ b/src/calibre/ebooks/metadata/book/json_codec.py
@@ -70,7 +70,7 @@ class JsonCodec(object):
def encode_metadata_attr(self, book, key):
if key == 'user_metadata':
- meta = book.all_user_metadata
+ meta = book.get_all_user_metadata(make_copy=True)
for k in meta:
if meta[k]['datatype'] == 'datetime':
meta[k]['#value#'] = datetime_to_string(meta[k]['#value#'])
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index 9f21fe0eda..935776f838 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -22,6 +22,7 @@ from calibre.library.sqlite import connect, IntegrityError, DBThread
from calibre.library.prefs import DBPrefs
from calibre.ebooks.metadata import string_to_authors, authors_to_string, \
MetaInformation
+from calibre.ebooks.metadata.book.base import Metadata
from calibre.ebooks.metadata.meta import get_metadata, metadata_from_formats
from calibre.constants import preferred_encoding, iswindows, isosx, filesystem_encoding
from calibre.ptempfile import PersistentTemporaryFile
@@ -537,7 +538,10 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
for key,meta in self.field_metadata.iteritems():
if meta['is_custom']:
mi.set_user_metadata(key, meta)
- mi.set(key, self.get_custom(idx, label=meta['label'], index_is_id=index_is_id))
+ mi.set(key, val=self.get_custom(idx, label=meta['label'],
+ index_is_id=index_is_id),
+ extra=self.get_custom_extra(idx, label=meta['label'],
+ index_is_id=index_is_id))
if get_cover:
mi.cover = self.cover(id, index_is_id=True, as_path=True)
return mi
@@ -1038,6 +1042,15 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
if getattr(mi, 'timestamp', None) is not None:
doit(self.set_timestamp, id, mi.timestamp, notify=False)
self.set_path(id, True)
+
+ user_mi = mi.get_all_user_metadata(make_copy=False)
+ for key in user_mi.iterkeys():
+ if key in self.field_metadata and \
+ user_mi[key]['datatype'] == self.field_metadata[key]['datatype']:
+ doit(self.set_custom, id,
+ val=Metadata.get_user_metadata_value(user_mi[key]),
+ extra=Metadata.get_user_metadata_extra(user_mi[key]),
+ label=user_mi[key]['label'])
self.notify('metadata', [id])
# Given a book, return the list of author sort strings for the book's authors
diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py
index 258ea7ba9e..7a3515305b 100644
--- a/src/calibre/library/save_to_disk.py
+++ b/src/calibre/library/save_to_disk.py
@@ -115,7 +115,7 @@ def get_components(template, mi, id, timefmt='%b %Y', length=250,
library_order = tweaks['save_template_title_series_sorting'] == 'library_order'
tsfmt = title_sort if library_order else lambda x: x
format_args = dict(**FORMAT_ARGS)
- format_args.update(mi.all_attributes)
+ format_args.update(mi.get_all_non_none_attributes())
if mi.title:
format_args['title'] = tsfmt(mi.title)
if mi.authors:
@@ -131,6 +131,8 @@ def get_components(template, mi, id, timefmt='%b %Y', length=250,
format_args['series_index'] = mi.format_series_index()
else:
template = re.sub(r'\{series_index[^}]*?\}', '', template)
+ ## TODO: format custom values. Check all the datatypes.
+
if mi.rating is not None:
format_args['rating'] = mi.format_rating()
if hasattr(mi.timestamp, 'timetuple'):
@@ -139,15 +141,6 @@ def get_components(template, mi, id, timefmt='%b %Y', length=250,
format_args['pubdate'] = strftime(timefmt, mi.pubdate.timetuple())
format_args['id'] = str(id)
- # These are not necessary any more. The values are set by
- # 'format_args.update' above, and there is no special formatting
-# if mi.author_sort:
-# format_args['author_sort'] = mi.author_sort
-# if mi.isbn:
-# format_args['isbn'] = mi.isbn
-# if mi.publisher:
-# format_args['publisher'] = mi.publisher
-
components = [x.strip() for x in template.split('/') if x.strip()]
components = [safe_format(x, format_args) for x in components]
components = [sanitize_func(x) for x in components if x]
From 65f8767057afa440c143353c52e038a092da6783 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sun, 29 Aug 2010 11:18:09 +0100
Subject: [PATCH 005/207] New metadata: 1) remove 'category' from standard
metadata fields 2) make isbn a classifier. Add ability to add more
classifiers 3) fixup TODO: to make them easier to find.
---
src/calibre/devices/usbms/books.py | 1 +
src/calibre/ebooks/metadata/book/__init__.py | 12 ++++++---
src/calibre/ebooks/metadata/book/base.py | 25 ++++++-------------
src/calibre/ebooks/metadata/opf2.py | 2 +-
.../gui2/dialogs/config/save_template.py | 4 +--
src/calibre/library/save_to_disk.py | 2 +-
6 files changed, 21 insertions(+), 25 deletions(-)
diff --git a/src/calibre/devices/usbms/books.py b/src/calibre/devices/usbms/books.py
index e3c405ee4e..0efa507e09 100644
--- a/src/calibre/devices/usbms/books.py
+++ b/src/calibre/devices/usbms/books.py
@@ -135,6 +135,7 @@ class CollectionsBookList(BookList):
elif isinstance(val, unicode):
val = [val]
for category in val:
+ # TODO: NEWMETA: format the custom fields
if attr == 'tags' and len(category) > 1 and \
category[0] == '[' and category[-1] == ']':
continue
diff --git a/src/calibre/ebooks/metadata/book/__init__.py b/src/calibre/ebooks/metadata/book/__init__.py
index fbcca79aba..47eb616394 100644
--- a/src/calibre/ebooks/metadata/book/__init__.py
+++ b/src/calibre/ebooks/metadata/book/__init__.py
@@ -19,8 +19,14 @@ SOCIAL_METADATA_FIELDS = frozenset([
# Of the form { scheme1:value1, scheme2:value2}
# For example: {'isbn':'123456789', 'doi':'xxxx', ... }
'classifiers',
- 'isbn', # Pseudo field for convenience, should get/set isbn classifier
- 'category', # TODO: not sure what this is, but it is used by OPF
+])
+
+'''
+The list of names that convert to classifiers when in get and set.
+'''
+
+TOP_LEVEL_CLASSIFIERS = frozenset([
+ 'isbn',
])
PUBLICATION_METADATA_FIELDS = frozenset([
@@ -77,7 +83,7 @@ CALIBRE_METADATA_FIELDS = frozenset([
'application_id', # An application id, currently set to the db_id.
# the calibre primary key of the item.
'db_id', # the calibre primary key of the item.
- # TODO: May want to remove once Sony's no longer use it
+ # TODO: NEWMETA: May want to remove once Sony's no longer use it
]
)
diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index a81ce46c34..6d89049bfb 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -11,6 +11,7 @@ import traceback
from calibre import prints
from calibre.ebooks.metadata.book import COPYABLE_METADATA_FIELDS
from calibre.ebooks.metadata.book import STANDARD_METADATA_FIELDS
+from calibre.ebooks.metadata.book import TOP_LEVEL_CLASSIFIERS
from calibre.utils.date import isoformat
@@ -55,6 +56,8 @@ class Metadata(object):
def __getattribute__(self, field):
_data = object.__getattribute__(self, '_data')
+ if field in TOP_LEVEL_CLASSIFIERS:
+ return _data.get('classifiers').get(field, None)
if field in STANDARD_METADATA_FIELDS:
return _data.get(field, None)
try:
@@ -68,7 +71,9 @@ class Metadata(object):
def __setattr__(self, field, val, extra=None):
_data = object.__getattribute__(self, '_data')
- if field in STANDARD_METADATA_FIELDS:
+ if field in TOP_LEVEL_CLASSIFIERS:
+ _data['classifiers'].update({field: val})
+ elif field in STANDARD_METADATA_FIELDS:
if val is None:
val = NULL_VALUES.get(field, None)
_data[field] = val
@@ -301,8 +306,6 @@ class Metadata(object):
fmt('Publisher', self.publisher)
if getattr(self, 'book_producer', False):
fmt('Book Producer', self.book_producer)
- if self.category:
- fmt('Category', self.category)
if self.comments:
fmt('Comments', self.comments)
if self.isbn:
@@ -321,14 +324,7 @@ class Metadata(object):
fmt('Published', isoformat(self.pubdate))
if self.rights is not None:
fmt('Rights', unicode(self.rights))
-# TODO: These are not in metadata. Should they be?
-# if self.lccn:
-# fmt('LCCN', unicode(self.lccn))
-# if self.lcc:
-# fmt('LCC', unicode(self.lcc))
-# if self.ddc:
-# fmt('DDC', unicode(self.ddc))
- # CUSTFIELD: What to do about custom fields?
+ # TODO: NEWMETA: What to do about custom fields?
return u'\n'.join(ans)
def to_html(self):
@@ -339,13 +335,6 @@ class Metadata(object):
ans += [(_('Producer'), unicode(self.book_producer))]
ans += [(_('Comments'), unicode(self.comments))]
ans += [('ISBN', unicode(self.isbn))]
-# TODO: These are not in metadata. Should they be?
-# if self.lccn:
-# ans += [('LCCN', unicode(self.lccn))]
-# if self.lcc:
-# ans += [('LCC', unicode(self.lcc))]
-# if self.ddc:
-# ans += [('DDC', unicode(self.ddc))]
ans += [(_('Tags'), u', '.join([unicode(t) for t in self.tags]))]
if self.series:
ans += [(_('Series'), unicode(self.series)+ ' #%s'%self.format_series_index())]
diff --git a/src/calibre/ebooks/metadata/opf2.py b/src/calibre/ebooks/metadata/opf2.py
index 0ab6d3bbc0..54d97fc157 100644
--- a/src/calibre/ebooks/metadata/opf2.py
+++ b/src/calibre/ebooks/metadata/opf2.py
@@ -1188,7 +1188,7 @@ def metadata_to_opf(mi, as_string=True):
factory(DC('contributor'), mi.book_producer, __appname__, 'bkp')
if hasattr(mi.pubdate, 'isoformat'):
factory(DC('date'), isoformat(mi.pubdate))
- if mi.category:
+ if hasattr(mi, 'category') and mi.category:
factory(DC('type'), mi.category)
if mi.comments:
factory(DC('description'), mi.comments)
diff --git a/src/calibre/gui2/dialogs/config/save_template.py b/src/calibre/gui2/dialogs/config/save_template.py
index 2157d5b3bf..7e49e86d29 100644
--- a/src/calibre/gui2/dialogs/config/save_template.py
+++ b/src/calibre/gui2/dialogs/config/save_template.py
@@ -34,8 +34,8 @@ class SaveTemplate(QWidget, Ui_Form):
self.option_name = name
def validate(self):
- # TODO: I haven't figured out how to get the custom columns into here,
- # so for the moment make all templates valid.
+ # TODO: NEWMETA: I haven't figured out how to get the custom columns
+ # into here, so for the moment make all templates valid.
return True
# tmpl = preprocess_template(self.opt_template.text())
# fa = {}
diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py
index 7a3515305b..963afd4085 100644
--- a/src/calibre/library/save_to_disk.py
+++ b/src/calibre/library/save_to_disk.py
@@ -131,7 +131,7 @@ def get_components(template, mi, id, timefmt='%b %Y', length=250,
format_args['series_index'] = mi.format_series_index()
else:
template = re.sub(r'\{series_index[^}]*?\}', '', template)
- ## TODO: format custom values. Check all the datatypes.
+ ## TODO: NEWMETA: format custom values. Check all the datatypes.
if mi.rating is not None:
format_args['rating'] = mi.format_rating()
From 6eb7383e82b564a146cd9e719d6424ec8f79c355 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Mon, 30 Aug 2010 12:16:37 +0100
Subject: [PATCH 006/207] Remove unused (and unworking) copy method
---
src/calibre/ebooks/metadata/book/base.py | 13 +------------
1 file changed, 1 insertion(+), 12 deletions(-)
diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index 6d89049bfb..dcb31c3ecc 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -185,17 +185,6 @@ class Metadata(object):
return result
# Old Metadata API {{{
- @staticmethod
- def copy(mi):
- ans = Metadata(mi.title, mi.authors)
- for attr in STANDARD_METADATA_FIELDS:
- if hasattr(mi, attr):
- setattr(ans, attr, copy.deepcopy(getattr(mi, attr)))
- for x in mi.user_metadata_keys:
- meta = mi.get_user_metadata(x)
- if meta is not None:
- ans.set_user_metadata(x, meta) # get... did the deep copy
-
def print_all_attributes(self):
for x in STANDARD_METADATA_FIELDS:
prints('%s:'%x, getattr(self, x, 'None'))
@@ -347,7 +336,7 @@ class Metadata(object):
ans += [(_('Rights'), unicode(self.rights))]
for i, x in enumerate(ans):
ans[i] = u'
%s
%s
'%x
- # CUSTFIELD: What to do about custom fields
+ # TODO: NEWMETA: What to do about custom fields
return u'
%s
'%u'\n'.join(ans)
def __str__(self):
From 11f7bd06a82a0b1d03afffd374344593dfb0c5dc Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Mon, 30 Aug 2010 17:47:25 +0100
Subject: [PATCH 007/207] Format custom fields in save_to_disk.
---
src/calibre/library/save_to_disk.py | 16 ++++++++++++++--
1 file changed, 14 insertions(+), 2 deletions(-)
diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py
index 963afd4085..2bc71cde9c 100644
--- a/src/calibre/library/save_to_disk.py
+++ b/src/calibre/library/save_to_disk.py
@@ -14,6 +14,7 @@ from calibre.utils.filenames import shorten_components_to, supports_long_names,
from calibre.ebooks.metadata.opf2 import metadata_to_opf
from calibre.ebooks.metadata.meta import set_metadata
from calibre.constants import preferred_encoding, filesystem_encoding
+from calibre.ebooks.metadata import fmt_sidx
from calibre.ebooks.metadata import title_sort
from calibre import strftime
@@ -131,8 +132,6 @@ def get_components(template, mi, id, timefmt='%b %Y', length=250,
format_args['series_index'] = mi.format_series_index()
else:
template = re.sub(r'\{series_index[^}]*?\}', '', template)
- ## TODO: NEWMETA: format custom values. Check all the datatypes.
-
if mi.rating is not None:
format_args['rating'] = mi.format_rating()
if hasattr(mi.timestamp, 'timetuple'):
@@ -140,6 +139,19 @@ def get_components(template, mi, id, timefmt='%b %Y', length=250,
if hasattr(mi.pubdate, 'timetuple'):
format_args['pubdate'] = strftime(timefmt, mi.pubdate.timetuple())
format_args['id'] = str(id)
+ # Now format the custom fields
+ custom_metadata = mi.get_all_user_metadata(make_copy=False)
+ for key in custom_metadata:
+ if key in format_args:
+ ## TODO: NEWMETA: should ratings be divided by 2? The standard rating isn't...
+ if custom_metadata[key]['datatype'] == 'series':
+ format_args[key] = tsfmt(format_args[key])
+ if key+'_index' in format_args:
+ format_args[key+'_index'] = fmt_sidx(format_args[key+'_index'])
+ elif custom_metadata[key]['datatype'] == 'datetime':
+ format_args[key] = strftime(timefmt, format_args[key].timetuple())
+ elif custom_metadata[key]['datatype'] == 'bool':
+ format_args[key] = _('yes') if format_args[key] else _('no')
components = [x.strip() for x in template.split('/') if x.strip()]
components = [safe_format(x, format_args) for x in components]
From 16e5b2f3b0204efefe1c73531c4ce98376342fc7 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Wed, 1 Sep 2010 16:44:12 +0100
Subject: [PATCH 008/207] Add code for Sony collections
---
resources/default_tweaks.py | 32 ++++++-
src/calibre/devices/usbms/books.py | 95 ++++++++++++++------
src/calibre/ebooks/metadata/book/__init__.py | 2 +-
src/calibre/ebooks/metadata/book/base.py | 17 ++--
4 files changed, 111 insertions(+), 35 deletions(-)
diff --git a/resources/default_tweaks.py b/resources/default_tweaks.py
index e03b0680be..e68096ecd5 100644
--- a/resources/default_tweaks.py
+++ b/resources/default_tweaks.py
@@ -90,4 +90,34 @@ save_template_title_series_sorting = 'library_order'
# Examples:
# auto_connect_to_folder = 'C:\\Users\\someone\\Desktop\\testlib'
# auto_connect_to_folder = '/home/dropbox/My Dropbox/someone/library'
-auto_connect_to_folder = ''
\ No newline at end of file
+auto_connect_to_folder = ''
+
+# Specify renaming rules for sony collections. Collections on Sonys are named
+# depending upon whether the field is standard or custom. A collection derived
+# from a standard field is named for the value in that field. For example, if
+# the standard 'series' column contains the name 'Darkover', then the series
+# will be named 'Darkover'. A collection derived from a custom field will have
+# the name of the field added to the value. For example, if a custom series
+# column named 'My Series' contains the name 'Darkover', then the collection
+# will be named 'Darkover (My Series)'. If two books have fields that generate
+# the same collection name, then both books will be in that collection. This
+# tweak lets you specify for a standard or custom field the value to be put
+# inside the parentheses. You can use it to add a parenthetical description to a
+# standard field, for example 'Foo (Tag)' instead of the 'Foo'. You can also use
+# it to force multiple fields to end up in the same collection. For example, you
+# could force the values in 'series', '#my_series_1', and '#my_series_2' to
+# appear in collections named 'some_value (Series)', thereby merging all of the
+# fields into one set of collections. The syntax of this tweak is
+# {'field_lookup_name':'name_to_use', 'lookup_name':'name', ...}
+# Example 1: I want three series columns to be merged into one set of
+# collections. If the column lookup names are 'series', '#series_1' and
+# '#series_2', and if I want nothing in the parenthesis, then the value to use
+# in the tweak value would be:
+# sony_collection_renaming_rules={'series':'', '#series_1':'', '#series_2':''}
+# Example 2: I want the word '(Series)' to appear on collections made from
+# series, and the word '(Tag)' to appear on collections made from tags. Use:
+# sony_collection_renaming_rules={'series':'Series', 'tags':'Tag'}
+# Example 3: I want 'series' and '#myseries' to be merged, and for the
+# collection name to have '(Series)' appended. The renaming rule is:
+# sony_collection_renaming_rules={'series':'Series', '#myseries':'Series'}
+sony_collection_renaming_rules={}
\ No newline at end of file
diff --git a/src/calibre/devices/usbms/books.py b/src/calibre/devices/usbms/books.py
index 0efa507e09..3e13527bd0 100644
--- a/src/calibre/devices/usbms/books.py
+++ b/src/calibre/devices/usbms/books.py
@@ -9,9 +9,10 @@ import os, re, time, sys
from calibre.ebooks.metadata.book.base import Metadata
from calibre.devices.mime import mime_type_ext
from calibre.devices.interface import BookList as _BookList
-from calibre.constants import filesystem_encoding, preferred_encoding
+from calibre.constants import preferred_encoding
from calibre import isbytestring
-from calibre.utils.config import prefs
+from calibre.utils.config import prefs, tweaks
+from calibre.utils.date import format_date
class Book(Metadata):
def __init__(self, prefix, lpath, size=None, other=None):
@@ -94,11 +95,38 @@ class CollectionsBookList(BookList):
def supports_collections(self):
return True
+ def compute_category_name(self, attr, category, cust_field_meta):
+ renames = tweaks['sony_collection_renaming_rules']
+ attr_name = renames.get(attr, None)
+ if attr_name is None:
+ if attr in cust_field_meta:
+ attr_name = '(%s)'%cust_field_meta[attr]['name']
+ else:
+ attr_name = ''
+ elif attr_name != '':
+ attr_name = '(%s)'%attr_name
+
+ if attr not in cust_field_meta:
+ cat_name = '%s %s'%(category, attr_name)
+ else:
+ fm = cust_field_meta[attr]
+ if fm['datatype'] == 'bool':
+ if category:
+ cat_name = '%s %s'%(_('Yes'), attr_name)
+ else:
+ cat_name = '%s %s'%(_('No'), attr_name)
+ elif fm['datatype'] == 'datetime':
+ cat_name = '%s %s'%(format_date(category,
+ fm['display'].get('date_format','dd MMM yyyy')), attr_name)
+ else:
+ cat_name = '%s %s'%(category, attr_name)
+ return cat_name.strip()
+
def get_collections(self, collection_attributes):
from calibre.devices.usbms.driver import debug_print
debug_print('Starting get_collections:', prefs['manage_device_metadata'])
+ debug_print('Renaming rules:', tweaks['sony_collection_renaming_rules'])
collections = {}
- series_categories = set([])
# This map of sets is used to avoid linear searches when testing for
# book equality
collections_lpaths = {}
@@ -124,42 +152,55 @@ class CollectionsBookList(BookList):
# For existing books, modify the collections only if the user
# specified 'on_connect'
attrs = collection_attributes
+ meta_vals = book.get_all_non_none_attributes()
+ cust_field_meta = book.get_all_user_metadata(make_copy=False)
for attr in attrs:
attr = attr.strip()
- val = getattr(book, attr, None)
+ val = meta_vals.get(attr, None)
if not val: continue
if isbytestring(val):
val = val.decode(preferred_encoding, 'replace')
if isinstance(val, (list, tuple)):
val = list(val)
- elif isinstance(val, unicode):
+ else:
val = [val]
for category in val:
- # TODO: NEWMETA: format the custom fields
- if attr == 'tags' and len(category) > 1 and \
- category[0] == '[' and category[-1] == ']':
+ is_series = False
+ if attr in cust_field_meta: # is a custom field
+ fm = cust_field_meta[attr]
+ if fm['datatype'] == 'text' and len(category) > 1 and \
+ category[0] == '[' and category[-1] == ']':
+ continue
+ if fm['datatype'] == 'series':
+ is_series = True
+ else: # is a standard field
+ if attr == 'tags' and len(category) > 1 and \
+ category[0] == '[' and category[-1] == ']':
+ continue
+ if attr == 'series' or \
+ ('series' in collection_attributes and
+ meta_vals.get('series', None) == category):
+ is_series = True
+ cat_name = self.compute_category_name(attr, category,
+ cust_field_meta)
+ if cat_name not in collections:
+ collections[cat_name] = []
+ collections_lpaths[cat_name] = set()
+ if lpath in collections_lpaths[cat_name]:
continue
- if category not in collections:
- collections[category] = []
- collections_lpaths[category] = set()
- if lpath not in collections_lpaths[category]:
- collections_lpaths[category].add(lpath)
- collections[category].append(book)
- if attr == 'series' or \
- ('series' in collection_attributes and
- getattr(book, 'series', None) == category):
- series_categories.add(category)
+ collections_lpaths[cat_name].add(lpath)
+ if is_series:
+ collections[cat_name].append(
+ (book, meta_vals.get(attr+'_index', sys.maxint)))
+ else:
+ collections[cat_name].append(
+ (book, meta_vals.get('title_sort', 'zzzz')))
# Sort collections
+ result = {}
for category, books in collections.items():
- def tgetter(x):
- return getattr(x, 'title_sort', 'zzzz')
- books.sort(cmp=lambda x,y:cmp(tgetter(x), tgetter(y)))
- if category in series_categories:
- # Ensures books are sub sorted by title
- def getter(x):
- return getattr(x, 'series_index', sys.maxint)
- books.sort(cmp=lambda x,y:cmp(getter(x), getter(y)))
- return collections
+ books.sort(cmp=lambda x,y:cmp(x[1], y[1]))
+ result[category] = [x[0] for x in books]
+ return result
def rebuild_collections(self, booklist, oncard):
'''
diff --git a/src/calibre/ebooks/metadata/book/__init__.py b/src/calibre/ebooks/metadata/book/__init__.py
index 47eb616394..ca7f4f7074 100644
--- a/src/calibre/ebooks/metadata/book/__init__.py
+++ b/src/calibre/ebooks/metadata/book/__init__.py
@@ -109,7 +109,7 @@ COPYABLE_METADATA_FIELDS = SOCIAL_METADATA_FIELDS.union(
CALIBRE_METADATA_FIELDS) - \
frozenset(['title', 'title_sort', 'authors',
'author_sort', 'author_sort_map' 'comments',
- 'cover_data', 'tags', 'language'])
+ 'cover_data', 'tags', 'language', 'lpath'])
SERIALIZABLE_FIELDS = SOCIAL_METADATA_FIELDS.union(
USER_METADATA_FIELDS).union(
diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index dcb31c3ecc..3f5507d676 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -117,16 +117,18 @@ class Metadata(object):
res[k] = copy.deepcopy(user_metadata[k])
return res
- def get_user_metadata(self, field):
+ def get_user_metadata(self, field, make_copy):
'''
return field metadata from the object if it is there. Otherwise return
- None. field is the key name, not the label. Return a copy, just in case
- the user wants to change values in the dict (json does).
+ None. field is the key name, not the label. Return a copy if requested,
+ just in case the user wants to change values in the dict.
'''
_data = object.__getattribute__(self, '_data')
_data = _data['user_metadata']
if field in _data:
- return copy.deepcopy(_data[field])
+ if make_copy:
+ return copy.deepcopy(_data[field])
+ return _data[field]
return None
@classmethod
@@ -189,7 +191,7 @@ class Metadata(object):
for x in STANDARD_METADATA_FIELDS:
prints('%s:'%x, getattr(self, x, 'None'))
for x in self.user_metadata_keys:
- meta = self.get_user_metadata(x)
+ meta = self.get_user_metadata(x, make_copy=False)
if meta is not None:
prints(x, meta)
prints('--------------')
@@ -220,6 +222,9 @@ class Metadata(object):
self.set_all_user_metadata(other.get_all_user_metadata(make_copy=True))
self.comments = getattr(other, 'comments', '')
self.language = getattr(other, 'language', None)
+ lpath = getattr(other, 'lpath', None)
+ if lpath is not None:
+ self.lpath = lpath
else:
for attr in COPYABLE_METADATA_FIELDS:
if hasattr(other, attr):
@@ -240,7 +245,7 @@ class Metadata(object):
if getattr(other, 'user_metadata_keys', None):
for x in other.user_metadata_keys:
- meta = other.get_user_metadata(x)
+ meta = other.get_user_metadata(x, make_copy=True)
if meta is not None:
self.set_user_metadata(x, meta) # get... did the deepcopy
From 0606afc8ff6edb0dfb6042bbc6cde3f891a068e9 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Thu, 2 Sep 2010 13:19:01 +0100
Subject: [PATCH 009/207] 1) make to_html support custom fields 2) clean up the
_extra code 3) add a format_custom_field method to avoid duplicating code 4)
pass custom metadata to book_details
---
src/calibre/ebooks/metadata/book/base.py | 43 ++++++++++++++++++------
src/calibre/gui2/book_details.py | 2 ++
src/calibre/gui2/library/models.py | 6 +++-
src/calibre/library/database2.py | 4 +--
4 files changed, 42 insertions(+), 13 deletions(-)
diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index 3f5507d676..caaaccb3d0 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -12,7 +12,8 @@ from calibre import prints
from calibre.ebooks.metadata.book import COPYABLE_METADATA_FIELDS
from calibre.ebooks.metadata.book import STANDARD_METADATA_FIELDS
from calibre.ebooks.metadata.book import TOP_LEVEL_CLASSIFIERS
-from calibre.utils.date import isoformat
+from calibre.utils.date import isoformat, format_date
+
NULL_VALUES = {
@@ -94,6 +95,13 @@ class Metadata(object):
return default
return self.__getattribute__(field)
+ def get_extra(self, field):
+ _data = object.__getattribute__(self, '_data')
+ if field in _data['user_metadata'].iterkeys():
+ return _data['user_metadata'][field]['#extra#']
+ raise AttributeError(
+ 'Metadata object has no attribute named: '+ repr(field))
+
def set(self, field, val, extra=None):
self.__setattr__(field, val, extra)
@@ -131,14 +139,6 @@ class Metadata(object):
return _data[field]
return None
- @classmethod
- def get_user_metadata_value(user_mi):
- return user_mi['#value#']
-
- @classmethod
- def get_user_metadata_extra(user_mi):
- return user_mi['#extra#']
-
def set_all_user_metadata(self, metadata):
'''
store custom field metadata into the object. Field is the key name
@@ -284,6 +284,25 @@ class Metadata(object):
def format_rating(self):
return unicode(self.rating)
+ def format_custom_field(self, key):
+ '''
+ returns the tuple (field_name, formatted_value)
+ '''
+ cmeta = self.get_user_metadata(key, make_copy=False)
+ name = unicode(cmeta['name'])
+ res = self.get(key, None)
+ if res is not None:
+ datatype = cmeta['datatype']
+ if datatype == 'text' and cmeta['is_multiple']:
+ res = u', '.join(res)
+ elif datatype == 'series':
+ res = res + ' [%s]'%self.format_series_index(self.get_extra(key))
+ elif datatype == 'datetime':
+ res = format_date(res, cmeta['display'].get('date_format','dd MMM yyyy'))
+ elif datatype == 'bool':
+ res = _('Yes') if res else _('No')
+ return (name, unicode(res))
+
def __unicode__(self):
from calibre.ebooks.metadata import authors_to_string
ans = []
@@ -339,9 +358,13 @@ class Metadata(object):
ans += [(_('Published'), unicode(self.pubdate.isoformat(' ')))]
if self.rights is not None:
ans += [(_('Rights'), unicode(self.rights))]
+ for key in self.user_metadata_keys:
+ val = self.get(key, None)
+ if val is not None:
+ (name, val) = self.format_custom_field(key)
+ ans += [(name, val)]
for i, x in enumerate(ans):
ans[i] = u'
%s
%s
'%x
- # TODO: NEWMETA: What to do about custom fields
return u'
%s
'%u'\n'.join(ans)
def __str__(self):
diff --git a/src/calibre/gui2/book_details.py b/src/calibre/gui2/book_details.py
index f08dd09429..4e11e0c84f 100644
--- a/src/calibre/gui2/book_details.py
+++ b/src/calibre/gui2/book_details.py
@@ -28,6 +28,8 @@ WEIGHTS[_('Tags')] = 4
def render_rows(data):
keys = data.keys()
+ # First sort by name. The WEIGHTS sort will preserve this sub-order
+ keys.sort(cmp=lambda x, y: cmp(x.lower(), y.lower()))
keys.sort(cmp=lambda x, y: cmp(WEIGHTS[x], WEIGHTS[y]))
rows = []
for key in keys:
diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index fdf21ecc23..2711756856 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -323,7 +323,11 @@ class BooksModel(QAbstractTableModel): # {{{
data[_('Series')] = \
_('Book %s of %s.')%\
(sidx, prepare_string_for_xml(series))
-
+ mi = self.db.get_metadata(idx)
+ for key in mi.user_metadata_keys:
+ name, val = mi.format_custom_field(key)
+ if val is not None:
+ data[name] = val
return data
def set_cache(self, idx):
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index 6457e12905..bb6d72bcff 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -1052,8 +1052,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
if key in self.field_metadata and \
user_mi[key]['datatype'] == self.field_metadata[key]['datatype']:
doit(self.set_custom, id,
- val=Metadata.get_user_metadata_value(user_mi[key]),
- extra=Metadata.get_user_metadata_extra(user_mi[key]),
+ val=mi.get(key),
+ extra=mi.get_extra(key),
label=user_mi[key]['label'])
self.notify('metadata', [id])
From 7ff7da0fbb2485301142a6ba2e6c04923a6d4a59 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Fri, 3 Sep 2010 18:43:17 +0100
Subject: [PATCH 010/207] Remove relative import
---
src/calibre/ebooks/metadata/book/json_codec.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/calibre/ebooks/metadata/book/json_codec.py b/src/calibre/ebooks/metadata/book/json_codec.py
index 7a80e16854..96178c4a63 100644
--- a/src/calibre/ebooks/metadata/book/json_codec.py
+++ b/src/calibre/ebooks/metadata/book/json_codec.py
@@ -9,7 +9,7 @@ import json
import traceback
from PIL import Image
-from . import SERIALIZABLE_FIELDS
+from calibre.ebooks.metadata.book import SERIALIZABLE_FIELDS
from calibre.constants import filesystem_encoding, preferred_encoding
from calibre.library.field_metadata import FieldMetadata
from calibre.utils.date import parse_date, isoformat, UNDEFINED_DATE
From 0ba513e2879b5e476d4787b333fb82b8dbe2e914 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sun, 5 Sep 2010 11:01:55 +0100
Subject: [PATCH 011/207] Add a comment about not using the Metadata class, and
why
---
src/calibre/library/server/mobile.py | 4 ++++
src/calibre/library/server/xml.py | 7 ++++---
2 files changed, 8 insertions(+), 3 deletions(-)
diff --git a/src/calibre/library/server/mobile.py b/src/calibre/library/server/mobile.py
index 229e0c21c4..6e08581aed 100644
--- a/src/calibre/library/server/mobile.py
+++ b/src/calibre/library/server/mobile.py
@@ -199,6 +199,10 @@ class MobileServer(object):
CKEYS = [key for key in sorted(CFM.get_custom_fields(),
cmp=lambda x,y: cmp(CFM[x]['name'].lower(),
CFM[y]['name'].lower()))]
+ # This method uses its own book dict, not the Metadata dict. The loop
+ # below could be changed to use db.get_metadata instead of reading
+ # info directly from the record made by the view, but it doesn't seem
+ # worth it at the moment.
books = []
for record in items[(start-1):(start-1)+num]:
book = {'formats':record[FM['formats']], 'size':record[FM['size']]}
diff --git a/src/calibre/library/server/xml.py b/src/calibre/library/server/xml.py
index ed8479980e..5bf2783f96 100644
--- a/src/calibre/library/server/xml.py
+++ b/src/calibre/library/server/xml.py
@@ -66,6 +66,10 @@ class XMLServer(object):
return x.decode(preferred_encoding, 'replace')
return unicode(x)
+ # This method uses its own book dict, not the Metadata dict. The loop
+ # below could be changed to use db.get_metadata instead of reading
+ # info directly from the record made by the view, but it doesn't seem
+ # worth it at the moment.
for record in items[start:start+num]:
kwargs = {}
aus = record[FM['authors']] if record[FM['authors']] else __builtin__._('Unknown')
@@ -138,6 +142,3 @@ class XMLServer(object):
return etree.tostring(ans, encoding='utf-8', pretty_print=True,
xml_declaration=True)
-
-
-
From 38c6199c7b54b243d039aba25cad86125326a582 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sun, 5 Sep 2010 11:06:54 +0100
Subject: [PATCH 012/207] Change some comments from 'class:MetaInformation' to
'class:Metadata'
---
src/calibre/customize/__init__.py | 4 ++--
src/calibre/devices/apple/driver.py | 2 +-
src/calibre/ebooks/metadata/epub.py | 2 +-
src/calibre/ebooks/metadata/fetch.py | 2 +-
4 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/src/calibre/customize/__init__.py b/src/calibre/customize/__init__.py
index 27e319de14..8ddc791b2f 100644
--- a/src/calibre/customize/__init__.py
+++ b/src/calibre/customize/__init__.py
@@ -218,7 +218,7 @@ class MetadataReaderPlugin(Plugin): # {{{
with the input data.
:param type: The type of file. Guaranteed to be one of the entries
in :attr:`file_types`.
- :return: A :class:`calibre.ebooks.metadata.MetaInformation` object
+ :return: A :class:`calibre.ebooks.metadata.book.Metadata` object
'''
return None
# }}}
@@ -248,7 +248,7 @@ class MetadataWriterPlugin(Plugin): # {{{
with the input data.
:param type: The type of file. Guaranteed to be one of the entries
in :attr:`file_types`.
- :param mi: A :class:`calibre.ebooks.metadata.MetaInformation` object
+ :param mi: A :class:`calibre.ebooks.metadata.book.Metadata` object
'''
pass
diff --git a/src/calibre/devices/apple/driver.py b/src/calibre/devices/apple/driver.py
index 75517e9df7..94aea1e79d 100644
--- a/src/calibre/devices/apple/driver.py
+++ b/src/calibre/devices/apple/driver.py
@@ -872,7 +872,7 @@ class ITUNES(DriverBase):
once uploaded to the device. len(names) == len(files)
:return: A list of 3-element tuples. The list is meant to be passed
to L{add_books_to_metadata}.
- :metadata: If not None, it is a list of :class:`MetaInformation` objects.
+ :metadata: If not None, it is a list of :class:`Metadata` objects.
The idea is to use the metadata to determine where on the device to
put the book. len(metadata) == len(files). Apart from the regular
cover (path to cover), there may also be a thumbnail attribute, which should
diff --git a/src/calibre/ebooks/metadata/epub.py b/src/calibre/ebooks/metadata/epub.py
index 041a1ee603..ac6b5feebe 100644
--- a/src/calibre/ebooks/metadata/epub.py
+++ b/src/calibre/ebooks/metadata/epub.py
@@ -164,7 +164,7 @@ def get_cover(opf, opf_path, stream, reader=None):
return render_html_svg_workaround(cpage, default_log)
def get_metadata(stream, extract_cover=True):
- """ Return metadata as a :class:`MetaInformation` object """
+ """ Return metadata as a :class:`Metadata` object """
stream.seek(0)
reader = OCFZipReader(stream)
mi = MetaInformation(reader.opf)
diff --git a/src/calibre/ebooks/metadata/fetch.py b/src/calibre/ebooks/metadata/fetch.py
index 96807c06ae..9b8a42e482 100644
--- a/src/calibre/ebooks/metadata/fetch.py
+++ b/src/calibre/ebooks/metadata/fetch.py
@@ -29,7 +29,7 @@ class MetadataSource(Plugin): # {{{
future use.
The fetch method must store the results in `self.results` as a list of
- :class:`MetaInformation` objects. If there is an error, it should be stored
+ :class:`Metadata` objects. If there is an error, it should be stored
in `self.exception` and `self.tb` (for the traceback).
'''
From 06b266840c8785aa93aa265c0c20308c1c7286f7 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sun, 5 Sep 2010 12:35:35 +0100
Subject: [PATCH 013/207] Make gui version of content server able to show both
abbreviated and full lists of tags.
---
resources/content_server/gui.js | 34 ++++++++++++++++++++++++
src/calibre/ebooks/metadata/book/base.py | 7 ++---
src/calibre/library/server/utils.py | 9 ++++---
src/calibre/library/server/xml.py | 6 +++--
4 files changed, 47 insertions(+), 9 deletions(-)
diff --git a/resources/content_server/gui.js b/resources/content_server/gui.js
index 8368866a5e..abbd409dc8 100644
--- a/resources/content_server/gui.js
+++ b/resources/content_server/gui.js
@@ -59,14 +59,44 @@ function render_book(book) {
title = title.slice(0, title.length-2);
title += ' ({0} MB) '.format(size);
}
+ title += ''
+ if (tags) {
+ t = tags.split(':&:', 2);
+ m = parseInt(t[0]);
+ t = t[1].split(',', m);
+ if (t.length == m) t[m] = '...'
+ title += 'Tags=[{0}] '.format(t.join(','));
+ }
+ custcols = book.attr("custcols").split(',')
+ for ( i = 0; i < custcols.length; i++) {
+ if (custcols[i].length > 0) {
+ vals = book.attr(custcols[i]).split(':#:', 2);
+ if (vals[0].indexOf('#T#') == 0) { //startswith
+ vals[0] = vals[0].substr(3, vals[0].length)
+ t = vals[1].split(':&:', 2);
+ m = parseInt(t[0]);
+ t = t[1].split(',', m);
+ if (t.length == m) t[m] = '...';
+ vals[1] = t.join(',');
+ }
+ title += '{0}=[{1}] '.format(vals[0], vals[1]);
+ }
+ }
+ title += ''
+ title += ''
if (tags) title += 'Tags=[{0}] '.format(tags);
custcols = book.attr("custcols").split(',')
for ( i = 0; i < custcols.length; i++) {
if (custcols[i].length > 0) {
vals = book.attr(custcols[i]).split(':#:', 2);
+ if (vals[0].indexOf('#T#') == 0) { //startswith
+ vals[0] = vals[0].substr(3, vals[0].length)
+ vals[1] = (vals[1].split(':&:', 2))[1];
+ }
title += '{0}=[{1}] '.format(vals[0], vals[1]);
}
}
+ title += ''
title += ''.format(id);
title += '
{0}
'.format(comments)
// Render authors cell
@@ -170,11 +200,15 @@ function fetch_library_books(start, num, timeout, sort, order, search) {
var cover = row.find('img').attr('src');
var collapsed = row.find('.comments').css('display') == 'none';
$("#book_list tbody tr * .comments").css('display', 'none');
+ $("#book_list tbody tr * .tagdata_short").css('display', 'inherit');
+ $("#book_list tbody tr * .tagdata_long").css('display', 'none');
$('#cover_pane').css('visibility', 'hidden');
if (collapsed) {
row.find('.comments').css('display', 'inherit');
$('#cover_pane img').attr('src', cover);
$('#cover_pane').css('visibility', 'visible');
+ row.find(".tagdata_short").css('display', 'none');
+ row.find(".tagdata_long").css('display', 'inherit');
}
});
diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index caaaccb3d0..2ff24b0ddc 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -262,10 +262,11 @@ class Metadata(object):
if other_lang and other_lang.lower() != 'und':
self.language = other_lang
- def format_series_index(self):
+ def format_series_index(self, val=None):
from calibre.ebooks.metadata import fmt_sidx
+ v = self.series_index if val is None else val
try:
- x = float(self.series_index)
+ x = float(v)
except ValueError:
x = 1
return fmt_sidx(x)
@@ -296,7 +297,7 @@ class Metadata(object):
if datatype == 'text' and cmeta['is_multiple']:
res = u', '.join(res)
elif datatype == 'series':
- res = res + ' [%s]'%self.format_series_index(self.get_extra(key))
+ res = res + ' [%s]'%self.format_series_index(val=self.get_extra(key))
elif datatype == 'datetime':
res = format_date(res, cmeta['display'].get('date_format','dd MMM yyyy'))
elif datatype == 'bool':
diff --git a/src/calibre/library/server/utils.py b/src/calibre/library/server/utils.py
index 23916aa75c..373653c15f 100644
--- a/src/calibre/library/server/utils.py
+++ b/src/calibre/library/server/utils.py
@@ -5,7 +5,7 @@ __license__ = 'GPL v3'
__copyright__ = '2010, Kovid Goyal '
__docformat__ = 'restructuredtext en'
-import time
+import time, sys
import cherrypy
@@ -44,8 +44,8 @@ def strftime(fmt='%Y/%m/%d %H:%M:%S', dt=None):
except:
return _strftime(fmt, nowf().timetuple())
-def format_tag_string(tags, sep):
- MAX = tweaks['max_content_server_tags_shown']
+def format_tag_string(tags, sep, ignore_max=False):
+ MAX = sys.maxint if ignore_max else tweaks['max_content_server_tags_shown']
if tags:
tlist = [t.strip() for t in tags.split(sep)]
else:
@@ -53,5 +53,6 @@ def format_tag_string(tags, sep):
tlist.sort(cmp=lambda x,y:cmp(x.lower(), y.lower()))
if len(tlist) > MAX:
tlist = tlist[:MAX]+['...']
- return u'%s'%(', '.join(tlist)) if tlist else ''
+ return u'%s:&:%s'%(tweaks['max_content_server_tags_shown'],
+ ', '.join(tlist)) if tlist else ''
diff --git a/src/calibre/library/server/xml.py b/src/calibre/library/server/xml.py
index 5bf2783f96..8715dda7d0 100644
--- a/src/calibre/library/server/xml.py
+++ b/src/calibre/library/server/xml.py
@@ -89,7 +89,7 @@ class XMLServer(object):
'comments'):
y = record[FM[x]]
if x == 'tags':
- y = format_tag_string(y, ',')
+ y = format_tag_string(y, ',', ignore_max=True)
kwargs[x] = serialize(y) if y else ''
c = kwargs.pop('comments')
@@ -111,7 +111,9 @@ class XMLServer(object):
name = CFM[key]['name']
custcols.append(k)
if datatype == 'text' and CFM[key]['is_multiple']:
- kwargs[k] = concat(name, format_tag_string(val,'|'))
+ kwargs[k] = concat('#T#'+name,
+ format_tag_string(val,'|',
+ ignore_max=True))
elif datatype == 'series':
kwargs[k] = concat(name, '%s [%s]'%(val,
fmt_sidx(record[CFM.cc_series_index_column_for(key)])))
From 6bad998546cc2c9891bdf48a5ff89ebe8a86b0cd Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Wed, 8 Sep 2010 16:52:40 +0100
Subject: [PATCH 014/207] Add custom fields to the 'unicode' function
---
src/calibre/ebooks/metadata/book/base.py | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index 2ff24b0ddc..648beb7b5c 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -338,7 +338,11 @@ class Metadata(object):
fmt('Published', isoformat(self.pubdate))
if self.rights is not None:
fmt('Rights', unicode(self.rights))
- # TODO: NEWMETA: What to do about custom fields?
+ for key in self.user_metadata_keys:
+ val = self.get(key, None)
+ if val is not None:
+ (name, val) = self.format_custom_field(key)
+ fmt(name, unicode(val))
return u'\n'.join(ans)
def to_html(self):
From 9155f838f1f43d878c083bd1a9b9ce955d1c32c8 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Wed, 8 Sep 2010 19:39:57 +0100
Subject: [PATCH 015/207] Fix some bugs found after or introduced by trunk
merges
---
src/calibre/ebooks/metadata/book/__init__.py | 3 ++-
src/calibre/ebooks/metadata/book/base.py | 26 ++++++++++----------
src/calibre/library/save_to_disk.py | 25 ++++++++++---------
3 files changed, 29 insertions(+), 25 deletions(-)
diff --git a/src/calibre/ebooks/metadata/book/__init__.py b/src/calibre/ebooks/metadata/book/__init__.py
index ca7f4f7074..e7f58ce858 100644
--- a/src/calibre/ebooks/metadata/book/__init__.py
+++ b/src/calibre/ebooks/metadata/book/__init__.py
@@ -109,7 +109,8 @@ COPYABLE_METADATA_FIELDS = SOCIAL_METADATA_FIELDS.union(
CALIBRE_METADATA_FIELDS) - \
frozenset(['title', 'title_sort', 'authors',
'author_sort', 'author_sort_map' 'comments',
- 'cover_data', 'tags', 'language', 'lpath'])
+ 'cover_data', 'tags', 'language', 'lpath',
+ 'size'])
SERIALIZABLE_FIELDS = SOCIAL_METADATA_FIELDS.union(
USER_METADATA_FIELDS).union(
diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index 648beb7b5c..69a3c42f4d 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -201,6 +201,11 @@ class Metadata(object):
Merge the information in C{other} into self. In case of conflicts, the information
in C{other} takes precedence, unless the information in other is NULL.
'''
+ def copy_not_none(dest, src, attr):
+ v = getattr(src, attr, None)
+ if v is not None:
+ setattr(dest, attr, copy.deepcopy(v))
+
if other.title and other.title != _('Unknown'):
self.title = other.title
if hasattr(other, 'title_sort'):
@@ -220,21 +225,19 @@ class Metadata(object):
self.tags = other.tags
self.cover_data = getattr(other, 'cover_data', '')
self.set_all_user_metadata(other.get_all_user_metadata(make_copy=True))
- self.comments = getattr(other, 'comments', '')
- self.language = getattr(other, 'language', None)
- lpath = getattr(other, 'lpath', None)
- if lpath is not None:
- self.lpath = lpath
+ copy_not_none(self, other, 'lpath')
+ copy_not_none(self, other, 'size')
+ copy_not_none(self, other, 'comments')
+ # language is handled below
else:
for attr in COPYABLE_METADATA_FIELDS:
if hasattr(other, attr):
+ copy_not_none(self, other, attr)
val = getattr(other, attr)
if val is not None:
setattr(self, attr, copy.deepcopy(val))
-
if other.tags:
self.tags += list(set(self.tags + other.tags))
-
if getattr(other, 'cover_data', False):
other_cover = other.cover_data[-1]
self_cover = self.cover_data[-1] if self.cover_data else ''
@@ -242,13 +245,11 @@ class Metadata(object):
if not other_cover: other_cover = ''
if len(other_cover) > len(self_cover):
self.cover_data = other.cover_data
-
if getattr(other, 'user_metadata_keys', None):
for x in other.user_metadata_keys:
meta = other.get_user_metadata(x, make_copy=True)
if meta is not None:
self.set_user_metadata(x, meta) # get... did the deepcopy
-
my_comments = getattr(self, 'comments', '')
other_comments = getattr(other, 'comments', '')
if not my_comments:
@@ -257,10 +258,9 @@ class Metadata(object):
other_comments = ''
if len(other_comments.strip()) > len(my_comments.strip()):
self.comments = other_comments
-
- other_lang = getattr(other, 'language', None)
- if other_lang and other_lang.lower() != 'und':
- self.language = other_lang
+ other_lang = getattr(other, 'language', None)
+ if other_lang and other_lang.lower() != 'und':
+ self.language = other_lang
def format_series_index(self, val=None):
from calibre.ebooks.metadata import fmt_sidx
diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py
index c940cc006b..d33cbb04b5 100644
--- a/src/calibre/library/save_to_disk.py
+++ b/src/calibre/library/save_to_disk.py
@@ -6,7 +6,7 @@ __license__ = 'GPL v3'
__copyright__ = '2009, Kovid Goyal '
__docformat__ = 'restructuredtext en'
-import os, traceback, cStringIO, re
+import os, traceback, cStringIO, re, string
from calibre.utils.config import Config, StringConfig, tweaks
from calibre.utils.filenames import shorten_components_to, supports_long_names, \
@@ -98,17 +98,20 @@ def preprocess_template(template):
template = template.decode(preferred_encoding, 'replace')
return template
+class SafeFormat(string.Formatter):
+ '''
+ Provides a format function that substitutes '' for any missing value
+ '''
+ def get_value(self, key, args, kwargs):
+ try:
+ return kwargs[key]
+ except:
+ return ''
+safe_formatter = SafeFormat()
+
def safe_format(x, format_args):
- try:
- ans = x.format(**format_args).strip()
- return re.sub(r'\s+', ' ', ans)
- except IndexError: # Thrown if user used [] and index is out of bounds
- pass
- except AttributeError: # Thrown if user used a non existing attribute
- pass
- except KeyError: # Thrown if user used custom field w/value None
- pass
- return ''
+ ans = safe_formatter.vformat(x, [], format_args).strip()
+ return re.sub(r'\s+', ' ', ans)
def get_components(template, mi, id, timefmt='%b %Y', length=250,
sanitize_func=ascii_filename, replace_whitespace=False,
From 7603f208b39032cbcdc42939c0753ed8ddbbe987 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Wed, 8 Sep 2010 20:29:42 +0100
Subject: [PATCH 016/207] SLight cleanup
---
src/calibre/library/save_to_disk.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py
index d33cbb04b5..3fa40c68b2 100644
--- a/src/calibre/library/save_to_disk.py
+++ b/src/calibre/library/save_to_disk.py
@@ -118,7 +118,7 @@ def get_components(template, mi, id, timefmt='%b %Y', length=250,
to_lowercase=False):
library_order = tweaks['save_template_title_series_sorting'] == 'library_order'
tsfmt = title_sort if library_order else lambda x: x
- format_args = dict(**FORMAT_ARGS)
+ format_args = FORMAT_ARGS.copy()
format_args.update(mi.get_all_non_none_attributes())
if mi.title:
format_args['title'] = tsfmt(mi.title)
From f175b8bf6d75ac8615227c2557dcccc341f493d4 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Fri, 10 Sep 2010 20:12:12 -0600
Subject: [PATCH 017/207] ...
---
src/calibre/ebooks/metadata/__init__.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/calibre/ebooks/metadata/__init__.py b/src/calibre/ebooks/metadata/__init__.py
index f64a269fd1..429ba06c6e 100644
--- a/src/calibre/ebooks/metadata/__init__.py
+++ b/src/calibre/ebooks/metadata/__init__.py
@@ -231,7 +231,7 @@ def MetaInformation(title, authors=(_('Unknown'),)):
mi = title
title = mi.title
authors = mi.authors
- return Metadata(title, authors, mi)
+ return Metadata(title, authors, other=mi)
def check_isbn10(isbn):
try:
From 4f01b09ded8a0aa21f273596051ca8f32426bef2 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sat, 11 Sep 2010 09:21:29 +0100
Subject: [PATCH 018/207] Check in the metadata class that custom field names
begin with '#'
---
src/calibre/ebooks/metadata/book/base.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index 69a3c42f4d..d0b428bf96 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -156,6 +156,9 @@ class Metadata(object):
the key name not the label
'''
if field is not None:
+ if not field.startswith('#'):
+ raise AttributeError(
+ 'Custom field name %s must begin with \'#\''%repr(field))
if metadata is None:
traceback.print_stack()
metadata = copy.deepcopy(metadata)
From afe5546a15aa34fec849c875cb2b9b139c36ebd6 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sat, 11 Sep 2010 09:23:03 +0100
Subject: [PATCH 019/207] Avoid spurious exceptions when adding None custom
metadata
---
src/calibre/ebooks/metadata/book/base.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index d0b428bf96..be9a4675c0 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -161,6 +161,7 @@ class Metadata(object):
'Custom field name %s must begin with \'#\''%repr(field))
if metadata is None:
traceback.print_stack()
+ return
metadata = copy.deepcopy(metadata)
if '#value#' not in metadata:
if metadata['datatype'] == 'text' and metadata['is_multiple']:
From db54abd2b659290ba8a8d598f99fc16c853bf2f7 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Sun, 12 Sep 2010 12:08:18 -0600
Subject: [PATCH 020/207] Various tweaks to the way smart_update works
---
src/calibre/ebooks/metadata/book/base.py | 34 +++++++++++++++---------
1 file changed, 21 insertions(+), 13 deletions(-)
diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index be9a4675c0..f52c41e4c5 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -202,12 +202,12 @@ class Metadata(object):
def smart_update(self, other, replace_metadata=False):
'''
- Merge the information in C{other} into self. In case of conflicts, the information
- in C{other} takes precedence, unless the information in other is NULL.
+ Merge the information in `other` into self. In case of conflicts, the information
+ in `other` takes precedence, unless the information in `other` is NULL.
'''
def copy_not_none(dest, src, attr):
v = getattr(src, attr, None)
- if v is not None:
+ if v not in (None, NULL_VALUES.get(attr, None)):
setattr(dest, attr, copy.deepcopy(v))
if other.title and other.title != _('Unknown'):
@@ -216,32 +216,38 @@ class Metadata(object):
self.title_sort = other.title_sort
if other.authors and other.authors[0] != _('Unknown'):
- self.authors = other.authors
+ self.authors = list(other.authors)
if hasattr(other, 'author_sort_map'):
- self.author_sort_map = other.author_sort_map
+ self.author_sort_map = dict(other.author_sort_map)
if hasattr(other, 'author_sort'):
self.author_sort = other.author_sort
if replace_metadata:
+ SPECIAL_FIELDS = frozenset(['lpath', 'size', 'comments'])
for attr in COPYABLE_METADATA_FIELDS:
setattr(self, attr, getattr(other, attr, 1.0 if \
attr == 'series_index' else None))
self.tags = other.tags
- self.cover_data = getattr(other, 'cover_data', '')
+ self.cover_data = getattr(other, 'cover_data',
+ NULL_VALUES['cover_data'])
self.set_all_user_metadata(other.get_all_user_metadata(make_copy=True))
- copy_not_none(self, other, 'lpath')
- copy_not_none(self, other, 'size')
- copy_not_none(self, other, 'comments')
+ for x in SPECIAL_FIELDS:
+ copy_not_none(self, other, x)
# language is handled below
else:
for attr in COPYABLE_METADATA_FIELDS:
if hasattr(other, attr):
copy_not_none(self, other, attr)
- val = getattr(other, attr)
- if val is not None:
- setattr(self, attr, copy.deepcopy(val))
if other.tags:
- self.tags += list(set(self.tags + other.tags))
+ # Case-insensitive but case preserving merging
+ lotags = [t.lower() for t in other.tags]
+ lstags = [t.lower() for t in self.tags]
+ ot, st = map(frozenset, (lotags, lstags))
+ for t in st.interection(ot):
+ sidx = lstags.index(t)
+ oidx = lotags.index(t)
+ self.tags[sidx] = other.tags[oidx]
+ self.tags += [t for t in other.tags if t.lower() in ot-st]
if getattr(other, 'cover_data', False):
other_cover = other.cover_data[-1]
self_cover = self.cover_data[-1] if self.cover_data else ''
@@ -262,6 +268,7 @@ class Metadata(object):
other_comments = ''
if len(other_comments.strip()) > len(my_comments.strip()):
self.comments = other_comments
+
other_lang = getattr(other, 'language', None)
if other_lang and other_lang.lower() != 'und':
self.language = other_lang
@@ -383,3 +390,4 @@ class Metadata(object):
return bool(self.title or self.author or self.comments or self.tags)
# }}}
+
From b01b603358c0332d481a3c572fcd44d63d5cccdf Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Sun, 12 Sep 2010 13:28:22 -0600
Subject: [PATCH 021/207] json_codec: Handle dictionaries with bytsestring
keys/vals as well
---
.../ebooks/metadata/book/json_codec.py | 35 ++++++++++++-------
1 file changed, 22 insertions(+), 13 deletions(-)
diff --git a/src/calibre/ebooks/metadata/book/json_codec.py b/src/calibre/ebooks/metadata/book/json_codec.py
index 0e205c52b0..ea0de07342 100644
--- a/src/calibre/ebooks/metadata/book/json_codec.py
+++ b/src/calibre/ebooks/metadata/book/json_codec.py
@@ -12,23 +12,18 @@ from calibre.ebooks.metadata.book import SERIALIZABLE_FIELDS
from calibre.constants import filesystem_encoding, preferred_encoding
from calibre.library.field_metadata import FieldMetadata
from calibre.utils.date import parse_date, isoformat, UNDEFINED_DATE
+from calibre import isbytestring
# Translate datetimes to and from strings. The string form is the datetime in
# UTC. The returned date is also UTC
def string_to_datetime(src):
if src == "None":
return None
-# dt = strptime(src, '%d %m %Y %H:%M:%S', assume_utc=True, as_utc=True)
-# if dt == UNDEFINED_DATE:
-# return None
return parse_date(src)
def datetime_to_string(dateval):
if dateval is None or dateval == UNDEFINED_DATE:
return "None"
-# tt = date_to_utc(dateval).timetuple()
-# res = "%02d %02d %04d %02d:%02d:%02d"%(tt.tm_mday, tt.tm_mon, tt.tm_year,
-# tt.tm_hour, tt.tm_min, tt.tm_sec)
return isoformat(dateval)
def encode_thumbnail(thumbnail):
@@ -47,6 +42,24 @@ def decode_thumbnail(tup):
return None
return (tup[0], tup[1], b64decode(tup[2]))
+def object_to_unicode(obj, enc=preferred_encoding):
+
+ def dec(x):
+ return x.decode(enc, 'replace')
+
+ if isbytestring(obj):
+ return dec(obj)
+ if isinstance(obj, (list, tuple)):
+ return [dec(x) if isbytestring(x) else x for x in obj]
+ if isinstance(obj, dict):
+ ans = {}
+ for k, v in obj.items():
+ k = object_to_unicode(k)
+ v = object_to_unicode(v)
+ ans[k] = v
+ return ans
+ return obj
+
class JsonCodec(object):
def __init__(self):
@@ -81,16 +94,13 @@ class JsonCodec(object):
value = book.get(key)
if key == 'thumbnail':
return encode_thumbnail(value)
- elif isinstance(value, str): # str includes bytes
+ elif isbytestring(value): # str includes bytes
enc = filesystem_encoding if key == 'lpath' else preferred_encoding
- return value.decode(enc, 'replace')
- elif isinstance(value, (list, tuple)):
- return [x.decode(preferred_encoding, 'replace') if
- isinstance(x, str) else x for x in value]
+ return object_to_unicode(value, enc=enc)
elif datatype == 'datetime':
return datetime_to_string(value)
else:
- return value
+ return object_to_unicode(value)
def decode_from_file(self, file, booklist, book_class, prefix):
js = []
@@ -108,7 +118,6 @@ class JsonCodec(object):
except:
print 'exception during JSON decoding'
traceback.print_exc()
- booklist = []
def decode_metadata(self, key, value):
if key == 'user_metadata':
From ea3719c7737b4c4df60840ed0b86bed872e07a6c Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sun, 12 Sep 2010 20:52:26 +0100
Subject: [PATCH 022/207] Content server fixes
---
src/calibre/library/server/mobile.py | 8 ++++++--
src/calibre/library/server/opds.py | 10 +++++++---
src/calibre/library/server/utils.py | 7 +++++--
3 files changed, 18 insertions(+), 7 deletions(-)
diff --git a/src/calibre/library/server/mobile.py b/src/calibre/library/server/mobile.py
index 6e08581aed..ab5b39eed8 100644
--- a/src/calibre/library/server/mobile.py
+++ b/src/calibre/library/server/mobile.py
@@ -124,6 +124,7 @@ def build_index(books, num, search, sort, order, start, total, url_base, CKEYS):
series = u'[%s - %s]'%(book['series'], book['series_index']) \
if book['series'] else ''
tags = u'Tags=[%s]'%book['tags'] if book['tags'] else ''
+ print tags
ctext = ''
for key in CKEYS:
@@ -217,7 +218,8 @@ class MobileServer(object):
book['authors'] = authors
book['series_index'] = fmt_sidx(float(record[FM['series_index']]))
book['series'] = record[FM['series']]
- book['tags'] = format_tag_string(record[FM['tags']], ',')
+ book['tags'] = format_tag_string(record[FM['tags']], ',',
+ no_tag_count=True)
book['title'] = record[FM['title']]
for x in ('timestamp', 'pubdate'):
book[x] = strftime('%Y/%m/%d %H:%M:%S', record[FM[x]])
@@ -233,7 +235,9 @@ class MobileServer(object):
continue
name = CFM[key]['name']
if datatype == 'text' and CFM[key]['is_multiple']:
- book[key] = concat(name, format_tag_string(val, '|'))
+ book[key] = concat(name,
+ format_tag_string(val, '|',
+ no_tag_count=True))
elif datatype == 'series':
book[key] = concat(name, '%s [%s]'%(val,
fmt_sidx(record[CFM.cc_series_index_column_for(key)])))
diff --git a/src/calibre/library/server/opds.py b/src/calibre/library/server/opds.py
index c3a1d68749..e495598a2f 100644
--- a/src/calibre/library/server/opds.py
+++ b/src/calibre/library/server/opds.py
@@ -17,6 +17,7 @@ import routes
from calibre.constants import __appname__
from calibre.ebooks.metadata import fmt_sidx
from calibre.library.comments import comments_to_html
+from calibre.library.server.utils import format_tag_string
from calibre import guess_type
from calibre.utils.ordered_dict import OrderedDict
from calibre.utils.date import format_date
@@ -147,8 +148,9 @@ def ACQUISITION_ENTRY(item, version, FM, updated, CFM, CKEYS):
extra.append(_('RATING: %s ')%rating)
tags = item[FM['tags']]
if tags:
- extra.append(_('TAGS: %s ')%\
- ', '.join(tags.split(',')))
+ extra.append(_('TAGS: %s ')%format_tag_string(tags, ',',
+ ignore_max=True,
+ no_tag_count=True))
series = item[FM['series']]
if series:
extra.append(_('SERIES: %s [%s] ')%\
@@ -160,7 +162,9 @@ def ACQUISITION_ENTRY(item, version, FM, updated, CFM, CKEYS):
name = CFM[key]['name']
datatype = CFM[key]['datatype']
if datatype == 'text' and CFM[key]['is_multiple']:
- extra.append('%s: %s '%(name, ', '.join(val.split('|'))))
+ extra.append('%s: %s '%(name, format_tag_string(val, '|',
+ ignore_max=True,
+ no_tag_count=True)))
elif datatype == 'series':
extra.append('%s: %s [%s] '%(name, val,
fmt_sidx(item[CFM.cc_series_index_column_for(key)])))
diff --git a/src/calibre/library/server/utils.py b/src/calibre/library/server/utils.py
index 373653c15f..9a64948a3d 100644
--- a/src/calibre/library/server/utils.py
+++ b/src/calibre/library/server/utils.py
@@ -44,7 +44,7 @@ def strftime(fmt='%Y/%m/%d %H:%M:%S', dt=None):
except:
return _strftime(fmt, nowf().timetuple())
-def format_tag_string(tags, sep, ignore_max=False):
+def format_tag_string(tags, sep, ignore_max=False, no_tag_count=False):
MAX = sys.maxint if ignore_max else tweaks['max_content_server_tags_shown']
if tags:
tlist = [t.strip() for t in tags.split(sep)]
@@ -53,6 +53,9 @@ def format_tag_string(tags, sep, ignore_max=False):
tlist.sort(cmp=lambda x,y:cmp(x.lower(), y.lower()))
if len(tlist) > MAX:
tlist = tlist[:MAX]+['...']
- return u'%s:&:%s'%(tweaks['max_content_server_tags_shown'],
+ if no_tag_count:
+ return ', '.join(tlist) if tlist else ''
+ else:
+ return u'%s:&:%s'%(tweaks['max_content_server_tags_shown'],
', '.join(tlist)) if tlist else ''
From 240c9428f5da2c5452dd7f0ab6cc8a6c8cd67afe Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Sun, 12 Sep 2010 13:57:42 -0600
Subject: [PATCH 023/207] Revert change to how Metadata.thumbnail is
interpreted in library.models
---
src/calibre/gui2/library/models.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index b0aec7446a..b81628cd27 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -1055,8 +1055,8 @@ class DeviceBooksModel(BooksModel): # {{{
img = QImage()
if hasattr(cdata, 'image_path'):
img.load(cdata.image_path)
- else:
- img.loadFromData(cdata[2])
+ elif cdata:
+ img.loadFromData(cdata)
if img.isNull():
img = self.default_image
data['cover'] = img
From 0a2f80fdbff1b5fb541e567a8735ba5b8ca20ffa Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sun, 12 Sep 2010 21:13:35 +0100
Subject: [PATCH 024/207] Merge from trunk
---
src/calibre/devices/usbms/books.py | 2 +-
src/calibre/ebooks/metadata/book/__init__.py | 2 +-
src/calibre/ebooks/metadata/book/base.py | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/src/calibre/devices/usbms/books.py b/src/calibre/devices/usbms/books.py
index 3e13527bd0..4d5110b049 100644
--- a/src/calibre/devices/usbms/books.py
+++ b/src/calibre/devices/usbms/books.py
@@ -59,7 +59,7 @@ class Book(Metadata):
return property(doc=doc, fget=fget)
@dynamic_property
- def thumbnail(self):
+ def thumbnail(self):'
return None
class BookList(_BookList):
diff --git a/src/calibre/ebooks/metadata/book/__init__.py b/src/calibre/ebooks/metadata/book/__init__.py
index e7f58ce858..84a88606f2 100644
--- a/src/calibre/ebooks/metadata/book/__init__.py
+++ b/src/calibre/ebooks/metadata/book/__init__.py
@@ -110,7 +110,7 @@ COPYABLE_METADATA_FIELDS = SOCIAL_METADATA_FIELDS.union(
frozenset(['title', 'title_sort', 'authors',
'author_sort', 'author_sort_map' 'comments',
'cover_data', 'tags', 'language', 'lpath',
- 'size'])
+ 'size', 'thumbnail'])
SERIALIZABLE_FIELDS = SOCIAL_METADATA_FIELDS.union(
USER_METADATA_FIELDS).union(
diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index f52c41e4c5..647a9f467e 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -243,7 +243,7 @@ class Metadata(object):
lotags = [t.lower() for t in other.tags]
lstags = [t.lower() for t in self.tags]
ot, st = map(frozenset, (lotags, lstags))
- for t in st.interection(ot):
+ for t in st.intersection(ot):
sidx = lstags.index(t)
oidx = lotags.index(t)
self.tags[sidx] = other.tags[oidx]
From db4b8d8216bd2299d471be43b114e3e05edf1f26 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sun, 12 Sep 2010 21:15:31 +0100
Subject: [PATCH 025/207] Fix some inadvertent changes
---
src/calibre/devices/usbms/books.py | 2 +-
src/calibre/ebooks/metadata/book/__init__.py | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/calibre/devices/usbms/books.py b/src/calibre/devices/usbms/books.py
index 4d5110b049..3e13527bd0 100644
--- a/src/calibre/devices/usbms/books.py
+++ b/src/calibre/devices/usbms/books.py
@@ -59,7 +59,7 @@ class Book(Metadata):
return property(doc=doc, fget=fget)
@dynamic_property
- def thumbnail(self):'
+ def thumbnail(self):
return None
class BookList(_BookList):
diff --git a/src/calibre/ebooks/metadata/book/__init__.py b/src/calibre/ebooks/metadata/book/__init__.py
index 84a88606f2..e7f58ce858 100644
--- a/src/calibre/ebooks/metadata/book/__init__.py
+++ b/src/calibre/ebooks/metadata/book/__init__.py
@@ -110,7 +110,7 @@ COPYABLE_METADATA_FIELDS = SOCIAL_METADATA_FIELDS.union(
frozenset(['title', 'title_sort', 'authors',
'author_sort', 'author_sort_map' 'comments',
'cover_data', 'tags', 'language', 'lpath',
- 'size', 'thumbnail'])
+ 'size'])
SERIALIZABLE_FIELDS = SOCIAL_METADATA_FIELDS.union(
USER_METADATA_FIELDS).union(
From 6ca5263d005a10218008ba74a1a8942ca5c6f74f Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sun, 12 Sep 2010 21:36:23 +0100
Subject: [PATCH 026/207] Fix typo, add merge is_multiple
---
src/calibre/ebooks/metadata/book/base.py | 13 +++++++++++++
1 file changed, 13 insertions(+)
diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index 647a9f467e..f2031afd0e 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -259,7 +259,20 @@ class Metadata(object):
for x in other.user_metadata_keys:
meta = other.get_user_metadata(x, make_copy=True)
if meta is not None:
+ self_tags = self.get(x, [])
self.set_user_metadata(x, meta) # get... did the deepcopy
+ other_tags = other.get(x, [])
+ if meta['is_multiple']:
+ # Case-insensitive but case preserving merging
+ lotags = [t.lower() for t in other_tags]
+ lstags = [t.lower() for t in self_tags]
+ ot, st = map(frozenset, (lotags, lstags))
+ for t in st.intersection(ot):
+ sidx = lstags.index(t)
+ oidx = lotags.index(t)
+ self_tags[sidx] = other.tags[oidx]
+ self_tags += [t for t in other.tags if t.lower() in ot-st]
+ setattr(self, x, self_tags)
my_comments = getattr(self, 'comments', '')
other_comments = getattr(other, 'comments', '')
if not my_comments:
From 8b554ee0cda99dd11c50ba6def1634ca2416396b Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sun, 12 Sep 2010 22:04:10 +0100
Subject: [PATCH 027/207] Fixes for thumbnails.
---
src/calibre/ebooks/metadata/book/__init__.py | 2 +-
src/calibre/ebooks/metadata/book/base.py | 3 ++-
2 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/src/calibre/ebooks/metadata/book/__init__.py b/src/calibre/ebooks/metadata/book/__init__.py
index e7f58ce858..84a88606f2 100644
--- a/src/calibre/ebooks/metadata/book/__init__.py
+++ b/src/calibre/ebooks/metadata/book/__init__.py
@@ -110,7 +110,7 @@ COPYABLE_METADATA_FIELDS = SOCIAL_METADATA_FIELDS.union(
frozenset(['title', 'title_sort', 'authors',
'author_sort', 'author_sort_map' 'comments',
'cover_data', 'tags', 'language', 'lpath',
- 'size'])
+ 'size', 'thumbnail'])
SERIALIZABLE_FIELDS = SOCIAL_METADATA_FIELDS.union(
USER_METADATA_FIELDS).union(
diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index f2031afd0e..7812f81180 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -223,7 +223,7 @@ class Metadata(object):
self.author_sort = other.author_sort
if replace_metadata:
- SPECIAL_FIELDS = frozenset(['lpath', 'size', 'comments'])
+ SPECIAL_FIELDS = frozenset(['lpath', 'size', 'comments', 'thumbnail'])
for attr in COPYABLE_METADATA_FIELDS:
setattr(self, attr, getattr(other, attr, 1.0 if \
attr == 'series_index' else None))
@@ -238,6 +238,7 @@ class Metadata(object):
for attr in COPYABLE_METADATA_FIELDS:
if hasattr(other, attr):
copy_not_none(self, other, attr)
+ copy_not_none(self, other, 'thumbnail')
if other.tags:
# Case-insensitive but case preserving merging
lotags = [t.lower() for t in other.tags]
From 171ac9488aeadfe1b6fd0605c17bbc71662726ed Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sun, 12 Sep 2010 22:32:20 +0100
Subject: [PATCH 028/207] An attempt to make covers work.
---
src/calibre/gui2/library/models.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index b81628cd27..4e8e9a10bd 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -1056,7 +1056,10 @@ class DeviceBooksModel(BooksModel): # {{{
if hasattr(cdata, 'image_path'):
img.load(cdata.image_path)
elif cdata:
- img.loadFromData(cdata)
+ if isinstance(cdata, tuple):
+ img.loadFromData(cdata[2])
+ else:
+ img.loadFromData(cdata)
if img.isNull():
img = self.default_image
data['cover'] = img
From e73b688ca8ee61b39a65dbfb40d36863b74800f2 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sun, 12 Sep 2010 22:44:36 +0100
Subject: [PATCH 029/207] Deal with the two thumbnail formats
---
src/calibre/ebooks/metadata/book/json_codec.py | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/src/calibre/ebooks/metadata/book/json_codec.py b/src/calibre/ebooks/metadata/book/json_codec.py
index ea0de07342..a6235e64d5 100644
--- a/src/calibre/ebooks/metadata/book/json_codec.py
+++ b/src/calibre/ebooks/metadata/book/json_codec.py
@@ -12,6 +12,7 @@ from calibre.ebooks.metadata.book import SERIALIZABLE_FIELDS
from calibre.constants import filesystem_encoding, preferred_encoding
from calibre.library.field_metadata import FieldMetadata
from calibre.utils.date import parse_date, isoformat, UNDEFINED_DATE
+from calibre.utils.magick.draw import identify_data
from calibre import isbytestring
# Translate datetimes to and from strings. The string form is the datetime in
@@ -32,7 +33,12 @@ def encode_thumbnail(thumbnail):
'''
if thumbnail is None:
return None
- return (thumbnail[0], thumbnail[1], b64encode(str(thumbnail[2])))
+ if isinstance(thumbnail, tuple):
+ try:
+ thumbnail = identify_data(thumbnail)
+ except:
+ return None
+ return (0, 0, b64encode(str(thumbnail)))
def decode_thumbnail(tup):
'''
From e0261c2ba1e53300275849818df0b78729f99b98 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sun, 12 Sep 2010 22:50:34 +0100
Subject: [PATCH 030/207] This time, do json thumbnails right.
---
src/calibre/ebooks/metadata/book/json_codec.py | 11 +++++++----
1 file changed, 7 insertions(+), 4 deletions(-)
diff --git a/src/calibre/ebooks/metadata/book/json_codec.py b/src/calibre/ebooks/metadata/book/json_codec.py
index a6235e64d5..51b9722803 100644
--- a/src/calibre/ebooks/metadata/book/json_codec.py
+++ b/src/calibre/ebooks/metadata/book/json_codec.py
@@ -12,7 +12,7 @@ from calibre.ebooks.metadata.book import SERIALIZABLE_FIELDS
from calibre.constants import filesystem_encoding, preferred_encoding
from calibre.library.field_metadata import FieldMetadata
from calibre.utils.date import parse_date, isoformat, UNDEFINED_DATE
-from calibre.utils.magick.draw import identify_data
+from calibre.utils.magick import Image
from calibre import isbytestring
# Translate datetimes to and from strings. The string form is the datetime in
@@ -33,12 +33,15 @@ def encode_thumbnail(thumbnail):
'''
if thumbnail is None:
return None
- if isinstance(thumbnail, tuple):
+ if not isinstance(thumbnail, tuple):
try:
- thumbnail = identify_data(thumbnail)
+ img = Image()
+ img.load(thumbnail)
+ width, height = img.size
+ thumbnail = (width, height, thumbnail)
except:
return None
- return (0, 0, b64encode(str(thumbnail)))
+ return (thumbnail[0], thumbnail[1], b64encode(str(thumbnail[2])))
def decode_thumbnail(tup):
'''
From be95815b6ce234602f1fe6f76a51a3571db06535 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Sun, 12 Sep 2010 18:10:52 -0600
Subject: [PATCH 031/207] ...
---
src/calibre/gui2/library/models.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index 4e8e9a10bd..09a28fb04e 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -1056,8 +1056,8 @@ class DeviceBooksModel(BooksModel): # {{{
if hasattr(cdata, 'image_path'):
img.load(cdata.image_path)
elif cdata:
- if isinstance(cdata, tuple):
- img.loadFromData(cdata[2])
+ if isinstance(cdata, (tuple, list)):
+ img.loadFromData(cdata[-1])
else:
img.loadFromData(cdata)
if img.isNull():
From 43adf4226a6d381cf121ad5871b9237af57c58af Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Mon, 13 Sep 2010 08:18:09 +0100
Subject: [PATCH 032/207] Rationalize how smart_update knows what to do.
Introduced 2 more metadata groups. One is the list of attributes that
smart_update is to process specially. The other is the list of attributes
that are to be copied if not none (what you called SPECIAL_FIELDS). These two
help keep the replace and merge branches in sync.
---
src/calibre/ebooks/metadata/book/__init__.py | 17 ++++++++++-----
src/calibre/ebooks/metadata/book/base.py | 23 ++++++++++++--------
2 files changed, 26 insertions(+), 14 deletions(-)
diff --git a/src/calibre/ebooks/metadata/book/__init__.py b/src/calibre/ebooks/metadata/book/__init__.py
index 84a88606f2..e087f8072d 100644
--- a/src/calibre/ebooks/metadata/book/__init__.py
+++ b/src/calibre/ebooks/metadata/book/__init__.py
@@ -101,16 +101,23 @@ STANDARD_METADATA_FIELDS = SOCIAL_METADATA_FIELDS.union(
DEVICE_METADATA_FIELDS).union(
CALIBRE_METADATA_FIELDS)
+# Metadata fields that smart update must do special processing to copy.
+
+SC_FIELDS_NOT_COPIED = frozenset(['title', 'title_sort', 'authors',
+ 'author_sort', 'author_sort_map',
+ 'cover_data', 'tags', 'language'])
+
+# Metadata fields that smart update should copy only if the source is not None
+SC_FIELDS_COPY_NOT_NULL = frozenset(['lpath', 'size', 'comments', 'thumbnail'])
+
# Metadata fields that smart update should copy without special handling
-COPYABLE_METADATA_FIELDS = SOCIAL_METADATA_FIELDS.union(
+SC_COPYABLE_FIELDS = SOCIAL_METADATA_FIELDS.union(
PUBLICATION_METADATA_FIELDS).union(
BOOK_STRUCTURE_FIELDS).union(
DEVICE_METADATA_FIELDS).union(
CALIBRE_METADATA_FIELDS) - \
- frozenset(['title', 'title_sort', 'authors',
- 'author_sort', 'author_sort_map' 'comments',
- 'cover_data', 'tags', 'language', 'lpath',
- 'size', 'thumbnail'])
+ SC_FIELDS_NOT_COPIED.union(
+ SC_FIELDS_COPY_NOT_NULL)
SERIALIZABLE_FIELDS = SOCIAL_METADATA_FIELDS.union(
USER_METADATA_FIELDS).union(
diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index 7812f81180..8538ed886c 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -9,7 +9,8 @@ import copy
import traceback
from calibre import prints
-from calibre.ebooks.metadata.book import COPYABLE_METADATA_FIELDS
+from calibre.ebooks.metadata.book import SC_COPYABLE_FIELDS
+from calibre.ebooks.metadata.book import SC_FIELDS_COPY_NOT_NULL
from calibre.ebooks.metadata.book import STANDARD_METADATA_FIELDS
from calibre.ebooks.metadata.book import TOP_LEVEL_CLASSIFIERS
from calibre.utils.date import isoformat, format_date
@@ -223,22 +224,23 @@ class Metadata(object):
self.author_sort = other.author_sort
if replace_metadata:
- SPECIAL_FIELDS = frozenset(['lpath', 'size', 'comments', 'thumbnail'])
- for attr in COPYABLE_METADATA_FIELDS:
+ # SPECIAL_FIELDS = frozenset(['lpath', 'size', 'comments', 'thumbnail'])
+ for attr in SC_COPYABLE_FIELDS:
setattr(self, attr, getattr(other, attr, 1.0 if \
attr == 'series_index' else None))
self.tags = other.tags
self.cover_data = getattr(other, 'cover_data',
- NULL_VALUES['cover_data'])
+ NULL_VALUES['cover_data'])
self.set_all_user_metadata(other.get_all_user_metadata(make_copy=True))
- for x in SPECIAL_FIELDS:
+ for x in SC_FIELDS_COPY_NOT_NULL:
copy_not_none(self, other, x)
# language is handled below
else:
- for attr in COPYABLE_METADATA_FIELDS:
- if hasattr(other, attr):
- copy_not_none(self, other, attr)
- copy_not_none(self, other, 'thumbnail')
+ for attr in SC_COPYABLE_FIELDS:
+ copy_not_none(self, other, attr)
+ for x in SC_FIELDS_COPY_NOT_NULL:
+ copy_not_none(self, other, x)
+
if other.tags:
# Case-insensitive but case preserving merging
lotags = [t.lower() for t in other.tags]
@@ -249,6 +251,7 @@ class Metadata(object):
oidx = lotags.index(t)
self.tags[sidx] = other.tags[oidx]
self.tags += [t for t in other.tags if t.lower() in ot-st]
+
if getattr(other, 'cover_data', False):
other_cover = other.cover_data[-1]
self_cover = self.cover_data[-1] if self.cover_data else ''
@@ -256,6 +259,7 @@ class Metadata(object):
if not other_cover: other_cover = ''
if len(other_cover) > len(self_cover):
self.cover_data = other.cover_data
+
if getattr(other, 'user_metadata_keys', None):
for x in other.user_metadata_keys:
meta = other.get_user_metadata(x, make_copy=True)
@@ -274,6 +278,7 @@ class Metadata(object):
self_tags[sidx] = other.tags[oidx]
self_tags += [t for t in other.tags if t.lower() in ot-st]
setattr(self, x, self_tags)
+
my_comments = getattr(self, 'comments', '')
other_comments = getattr(other, 'comments', '')
if not my_comments:
From a85be2ba3229f2e27221fb7f64d25a7e48977ab7 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Mon, 13 Sep 2010 11:59:45 +0100
Subject: [PATCH 033/207] Several changes: 1) allow use of unusual standard
fields in get_collections. Format them appropriately 2) change
metadata.book.base.format_field to handle standard fields. 3) add standard
metadata access methods to metadata.book.base.
---
src/calibre/devices/usbms/books.py | 18 +-----
src/calibre/ebooks/metadata/book/base.py | 71 +++++++++++++++++++++---
src/calibre/gui2/library/models.py | 2 +-
src/calibre/library/field_metadata.py | 17 ++++--
4 files changed, 76 insertions(+), 32 deletions(-)
diff --git a/src/calibre/devices/usbms/books.py b/src/calibre/devices/usbms/books.py
index 3e13527bd0..2b19027df4 100644
--- a/src/calibre/devices/usbms/books.py
+++ b/src/calibre/devices/usbms/books.py
@@ -105,21 +105,7 @@ class CollectionsBookList(BookList):
attr_name = ''
elif attr_name != '':
attr_name = '(%s)'%attr_name
-
- if attr not in cust_field_meta:
- cat_name = '%s %s'%(category, attr_name)
- else:
- fm = cust_field_meta[attr]
- if fm['datatype'] == 'bool':
- if category:
- cat_name = '%s %s'%(_('Yes'), attr_name)
- else:
- cat_name = '%s %s'%(_('No'), attr_name)
- elif fm['datatype'] == 'datetime':
- cat_name = '%s %s'%(format_date(category,
- fm['display'].get('date_format','dd MMM yyyy')), attr_name)
- else:
- cat_name = '%s %s'%(category, attr_name)
+ cat_name = '%s %s'%(category, attr_name)
return cat_name.strip()
def get_collections(self, collection_attributes):
@@ -156,7 +142,7 @@ class CollectionsBookList(BookList):
cust_field_meta = book.get_all_user_metadata(make_copy=False)
for attr in attrs:
attr = attr.strip()
- val = meta_vals.get(attr, None)
+ ign, val = book.format_field(attr, ignore_series_index=True)
if not val: continue
if isbytestring(val):
val = val.decode(preferred_encoding, 'replace')
diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index 8538ed886c..6e0351353f 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -13,6 +13,7 @@ from calibre.ebooks.metadata.book import SC_COPYABLE_FIELDS
from calibre.ebooks.metadata.book import SC_FIELDS_COPY_NOT_NULL
from calibre.ebooks.metadata.book import STANDARD_METADATA_FIELDS
from calibre.ebooks.metadata.book import TOP_LEVEL_CLASSIFIERS
+from calibre.library.field_metadata import FieldMetadata
from calibre.utils.date import isoformat, format_date
@@ -30,6 +31,8 @@ NULL_VALUES = {
'language' : 'und'
}
+field_metadata = FieldMetadata()
+
class Metadata(object):
'''
@@ -112,6 +115,31 @@ class Metadata(object):
_data = object.__getattribute__(self, '_data')
return frozenset(_data['user_metadata'].iterkeys())
+ def get_standard_metadata(self, field, make_copy):
+ '''
+ return field metadata from the field if it is there. Otherwise return
+ None. field is the key name, not the label. Return a copy if requested,
+ just in case the user wants to change values in the dict.
+ '''
+ if field in field_metadata and field_metadata[field]['kind'] == 'field':
+ if make_copy:
+ return copy.deepcopy(field_metadata[field])
+ return field_metadata[field]
+ return None
+
+ def get_all_standard_metadata(self, make_copy):
+ '''
+ return a dict containing all the standard field metadata associated with
+ the book.
+ '''
+ if not make_copy:
+ return field_metadata
+ res = {}
+ for k in field_metadata:
+ if field_metadata[k]['kind'] == 'field':
+ res[k] = copy.deepcopy(field_metadata[k])
+ return res
+
def get_all_user_metadata(self, make_copy):
'''
return a dict containing all the custom field metadata associated with
@@ -315,24 +343,49 @@ class Metadata(object):
def format_rating(self):
return unicode(self.rating)
- def format_custom_field(self, key):
+ def format_field(self, key, ignore_series_index=False):
+ from calibre.ebooks.metadata import authors_to_string
'''
returns the tuple (field_name, formatted_value)
'''
- cmeta = self.get_user_metadata(key, make_copy=False)
- name = unicode(cmeta['name'])
- res = self.get(key, None)
- if res is not None:
+ if key in self.user_metadata_keys:
+ res = self.get(key, None)
+ if res is None or res == '':
+ return (None, None)
+ cmeta = self.get_user_metadata(key, make_copy=False)
+ name = unicode(cmeta['name'])
datatype = cmeta['datatype']
if datatype == 'text' and cmeta['is_multiple']:
res = u', '.join(res)
elif datatype == 'series':
- res = res + ' [%s]'%self.format_series_index(val=self.get_extra(key))
+ if not ignore_series_index:
+ res = res + \
+ ' [%s]'%self.format_series_index(val=self.get_extra(key))
elif datatype == 'datetime':
res = format_date(res, cmeta['display'].get('date_format','dd MMM yyyy'))
elif datatype == 'bool':
res = _('Yes') if res else _('No')
- return (name, unicode(res))
+ return (name, unicode(res))
+
+ if key in field_metadata and field_metadata[key]['kind'] == 'field':
+ res = self.get(key, None)
+ if res is None or res == '':
+ return (None, None)
+ fmeta = field_metadata[key]
+ name = unicode(fmeta['name'])
+ datatype = fmeta['datatype']
+ if key == 'authors':
+ res = authors_to_string(res)
+ elif datatype == 'text' and fmeta['is_multiple']:
+ res = u', '.join(res)
+ elif datatype == 'series':
+ if not ignore_series_index:
+ res = res + ' [%s]'%self.format_series_index()
+ elif datatype == 'datetime':
+ res = format_date(res, fmeta['display'].get('date_format','dd MMM yyyy'))
+ return (name, unicode(res))
+
+ return (None, None)
def __unicode__(self):
from calibre.ebooks.metadata import authors_to_string
@@ -371,7 +424,7 @@ class Metadata(object):
for key in self.user_metadata_keys:
val = self.get(key, None)
if val is not None:
- (name, val) = self.format_custom_field(key)
+ (name, val) = self.format_field(key)
fmt(name, unicode(val))
return u'\n'.join(ans)
@@ -396,7 +449,7 @@ class Metadata(object):
for key in self.user_metadata_keys:
val = self.get(key, None)
if val is not None:
- (name, val) = self.format_custom_field(key)
+ (name, val) = self.format_field(key)
ans += [(name, val)]
for i, x in enumerate(ans):
ans[i] = u'
%s
%s
'%x
diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index 09a28fb04e..5fa514ae8a 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -320,7 +320,7 @@ class BooksModel(QAbstractTableModel): # {{{
(sidx, prepare_string_for_xml(series))
mi = self.db.get_metadata(idx)
for key in mi.user_metadata_keys:
- name, val = mi.format_custom_field(key)
+ name, val = mi.format_field(key)
if val is not None:
data[name] = val
return data
diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py
index 276a6ba971..2773f573b2 100644
--- a/src/calibre/library/field_metadata.py
+++ b/src/calibre/library/field_metadata.py
@@ -5,6 +5,7 @@ Created on 25 May 2010
'''
from calibre.utils.ordered_dict import OrderedDict
+from calibre.utils.config import tweaks
class TagsIcons(dict):
'''
@@ -213,7 +214,7 @@ class FieldMetadata(dict):
'datatype':'text',
'is_multiple':None,
'kind':'field',
- 'name':None,
+ 'name':_('On Device'),
'search_terms':['ondevice'],
'is_custom':False,
'is_category':False}),
@@ -231,7 +232,7 @@ class FieldMetadata(dict):
'datatype':'datetime',
'is_multiple':None,
'kind':'field',
- 'name':None,
+ 'name':_('Published'),
'search_terms':['pubdate'],
'is_custom':False,
'is_category':False}),
@@ -258,7 +259,7 @@ class FieldMetadata(dict):
'datatype':'float',
'is_multiple':None,
'kind':'field',
- 'name':None,
+ 'name':_('Size (MB)'),
'search_terms':['size'],
'is_custom':False,
'is_category':False}),
@@ -267,7 +268,7 @@ class FieldMetadata(dict):
'datatype':'datetime',
'is_multiple':None,
'kind':'field',
- 'name':None,
+ 'name':_('Date'),
'search_terms':['date'],
'is_custom':False,
'is_category':False}),
@@ -276,7 +277,7 @@ class FieldMetadata(dict):
'datatype':'text',
'is_multiple':None,
'kind':'field',
- 'name':None,
+ 'name':_('Title'),
'search_terms':['title'],
'is_custom':False,
'is_category':False}),
@@ -310,6 +311,10 @@ class FieldMetadata(dict):
self._tb_cats[k]['display'] = {}
self._tb_cats[k]['is_editable'] = True
self._add_search_terms_to_map(k, v['search_terms'])
+ self._tb_cats['timestamp']['display'] = {
+ 'date_format': tweaks['gui_timestamp_display_format']}
+ self._tb_cats['pubdate']['display'] = {
+ 'date_format': tweaks['gui_pubdate_display_format']}
self.custom_field_prefix = '#'
self.get = self._tb_cats.get
@@ -410,7 +415,7 @@ class FieldMetadata(dict):
if datatype == 'series':
key += '_index'
self._tb_cats[key] = {'table':None, 'column':None,
- 'datatype':'float', 'is_multiple':False,
+ 'datatype':'float', 'is_multiple':None,
'kind':'field', 'name':'',
'search_terms':[key], 'label':label+'_index',
'colnum':None, 'display':{},
From 2a654f3062401e864cdd357850033ad05937f34e Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Tue, 14 Sep 2010 07:41:06 +0100
Subject: [PATCH 034/207] Fix stupidity in collectiions_management where I
broke tag splitting
---
src/calibre/devices/usbms/books.py | 4 +++-
src/calibre/ebooks/metadata/book/base.py | 13 ++++++++-----
2 files changed, 11 insertions(+), 6 deletions(-)
diff --git a/src/calibre/devices/usbms/books.py b/src/calibre/devices/usbms/books.py
index cf60f1311c..d25787fc89 100644
--- a/src/calibre/devices/usbms/books.py
+++ b/src/calibre/devices/usbms/books.py
@@ -141,7 +141,9 @@ class CollectionsBookList(BookList):
cust_field_meta = book.get_all_user_metadata(make_copy=False)
for attr in attrs:
attr = attr.strip()
- ign, val = book.format_field(attr, ignore_series_index=True)
+ ign, val = book.format_field(attr,
+ ignore_series_index=True,
+ return_multiples_as_list=True)
if not val: continue
if isbytestring(val):
val = val.decode(preferred_encoding, 'replace')
diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index 6e0351353f..7405f20a7c 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -343,7 +343,8 @@ class Metadata(object):
def format_rating(self):
return unicode(self.rating)
- def format_field(self, key, ignore_series_index=False):
+ def format_field(self, key, ignore_series_index=False,
+ return_multiples_as_list=False):
from calibre.ebooks.metadata import authors_to_string
'''
returns the tuple (field_name, formatted_value)
@@ -356,7 +357,8 @@ class Metadata(object):
name = unicode(cmeta['name'])
datatype = cmeta['datatype']
if datatype == 'text' and cmeta['is_multiple']:
- res = u', '.join(res)
+ if not return_multiples_as_list:
+ res = u', '.join(res)
elif datatype == 'series':
if not ignore_series_index:
res = res + \
@@ -365,7 +367,7 @@ class Metadata(object):
res = format_date(res, cmeta['display'].get('date_format','dd MMM yyyy'))
elif datatype == 'bool':
res = _('Yes') if res else _('No')
- return (name, unicode(res))
+ return (name, res)
if key in field_metadata and field_metadata[key]['kind'] == 'field':
res = self.get(key, None)
@@ -377,13 +379,14 @@ class Metadata(object):
if key == 'authors':
res = authors_to_string(res)
elif datatype == 'text' and fmeta['is_multiple']:
- res = u', '.join(res)
+ if not return_multiples_as_list:
+ res = u', '.join(res)
elif datatype == 'series':
if not ignore_series_index:
res = res + ' [%s]'%self.format_series_index()
elif datatype == 'datetime':
res = format_date(res, fmeta['display'].get('date_format','dd MMM yyyy'))
- return (name, unicode(res))
+ return (name, res)
return (None, None)
From 6653fff4cd3228629c879e0e534024a9cc203fd2 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Wed, 15 Sep 2010 21:20:55 -0600
Subject: [PATCH 035/207] OPF serialization of user metadata
---
.../ebooks/metadata/book/json_codec.py | 2 +-
src/calibre/ebooks/metadata/opf2.py | 108 ++++++++++++++++--
2 files changed, 98 insertions(+), 12 deletions(-)
diff --git a/src/calibre/ebooks/metadata/book/json_codec.py b/src/calibre/ebooks/metadata/book/json_codec.py
index 51b9722803..2550089473 100644
--- a/src/calibre/ebooks/metadata/book/json_codec.py
+++ b/src/calibre/ebooks/metadata/book/json_codec.py
@@ -33,7 +33,7 @@ def encode_thumbnail(thumbnail):
'''
if thumbnail is None:
return None
- if not isinstance(thumbnail, tuple):
+ if not isinstance(thumbnail, (tuple, list)):
try:
img = Image()
img.load(thumbnail)
diff --git a/src/calibre/ebooks/metadata/opf2.py b/src/calibre/ebooks/metadata/opf2.py
index be8507f478..236b2fa18f 100644
--- a/src/calibre/ebooks/metadata/opf2.py
+++ b/src/calibre/ebooks/metadata/opf2.py
@@ -7,7 +7,7 @@ __docformat__ = 'restructuredtext en'
lxml based OPF parser.
'''
-import re, sys, unittest, functools, os, mimetypes, uuid, glob, cStringIO
+import re, sys, unittest, functools, os, mimetypes, uuid, glob, cStringIO, json
from urllib import unquote
from urlparse import urlparse
@@ -20,8 +20,9 @@ from calibre.ebooks.metadata import string_to_authors, MetaInformation
from calibre.ebooks.metadata.book.base import Metadata
from calibre.utils.date import parse_date, isoformat
from calibre.utils.localization import get_lang
+from calibre import prints
-class Resource(object):
+class Resource(object): # {{{
'''
Represents a resource (usually a file on the filesystem or a URL pointing
to the web. Such resources are commonly referred to in OPF files.
@@ -102,8 +103,9 @@ class Resource(object):
def __repr__(self):
return 'Resource(%s, %s)'%(repr(self.path), repr(self.href()))
+# }}}
-class ResourceCollection(object):
+class ResourceCollection(object): # {{{
def __init__(self):
self._resources = []
@@ -154,10 +156,9 @@ class ResourceCollection(object):
for res in self:
res.set_basedir(path)
+# }}}
-
-
-class ManifestItem(Resource):
+class ManifestItem(Resource): # {{{
@staticmethod
def from_opf_manifest_item(item, basedir):
@@ -195,8 +196,9 @@ class ManifestItem(Resource):
return self.media_type
raise IndexError('%d out of bounds.'%index)
+# }}}
-class Manifest(ResourceCollection):
+class Manifest(ResourceCollection): # {{{
@staticmethod
def from_opf_manifest_element(items, dir):
@@ -263,7 +265,9 @@ class Manifest(ResourceCollection):
if i.id == id:
return i.mime_type
-class Spine(ResourceCollection):
+# }}}
+
+class Spine(ResourceCollection): # {{{
class Item(Resource):
@@ -335,7 +339,9 @@ class Spine(ResourceCollection):
for i in self:
yield i.path
-class Guide(ResourceCollection):
+# }}}
+
+class Guide(ResourceCollection): # {{{
class Reference(Resource):
@@ -372,6 +378,7 @@ class Guide(ResourceCollection):
self[-1].type = type
self[-1].title = ''
+# }}}
class MetadataField(object):
@@ -413,7 +420,29 @@ class MetadataField(object):
elem = obj.create_metadata_element(self.name, is_dc=self.is_dc)
obj.set_text(elem, unicode(val))
-class OPF(object):
+
+def serialize_user_metadata(metadata_elem, all_user_metadata, tail='\n'+(' '*8)):
+ from calibre.utils.config import to_json
+ from calibre.ebooks.metadata.book.json_codec import object_to_unicode
+
+ for name, fm in all_user_metadata.items():
+ try:
+ fm = object_to_unicode(fm)
+ fm = json.dumps(fm, default=to_json, ensure_ascii=False)
+ except:
+ prints('Failed to write user metadata:', name)
+ import traceback
+ traceback.print_exc()
+ continue
+ meta = metadata_elem.makeelement('meta')
+ meta.set('name', 'calibre:user_metadata:'+name)
+ meta.set('content', fm)
+ meta.tail = tail
+ metadata_elem.append(meta)
+
+
+class OPF(object): # {{{
+
MIMETYPE = 'application/oebps-package+xml'
PARSER = etree.XMLParser(recover=True)
NAMESPACES = {
@@ -498,6 +527,34 @@ class OPF(object):
self.guide = Guide.from_opf_guide(guide, basedir) if guide else None
self.cover_data = (None, None)
self.find_toc()
+ self.read_user_metadata()
+
+ def read_user_metadata(self):
+ self.user_metadata = {}
+ from calibre.utils.config import from_json
+ elems = self.root.xpath('//*[name() = "meta" and starts-with(@name,'
+ '"calibre:user_metadata:") and @content]')
+ for elem in elems:
+ name = elem.get('name')
+ name = ':'.join(name.split(':')[2:])
+ if not name or not name.startswith('#'):
+ continue
+ fm = elem.get('content')
+ try:
+ fm = json.loads(fm, object_hook=from_json)
+ except:
+ prints('Failed to read user metadata:', name)
+ import traceback
+ traceback.print_exc()
+ continue
+ self.user_metadata[name] = fm
+
+
+ def write_user_metadata(self):
+ for elem in self.user_metadata_path(self.root):
+ elem.getparent().remove(elem)
+ serialize_user_metadata(self.metadata,
+ self.user_metadata)
def find_toc(self):
self.toc = None
@@ -912,6 +969,7 @@ class OPF(object):
return elem
def render(self, encoding='utf-8'):
+ self.write_user_metadata()
raw = etree.tostring(self.root, encoding=encoding, pretty_print=True)
if not raw.lstrip().startswith('\n'%encoding.upper()+raw
@@ -926,6 +984,7 @@ class OPF(object):
if val is not None and val != [] and val != (None, None):
setattr(self, attr, val)
+# }}}
class OPFCreator(Metadata):
@@ -1116,6 +1175,8 @@ class OPFCreator(Metadata):
item.set('title', ref.title)
guide.append(item)
+ serialize_user_metadata(metadata, self.get_all_user_metadata(False))
+
root = E.package(
metadata,
manifest,
@@ -1218,6 +1279,8 @@ def metadata_to_opf(mi, as_string=True):
if mi.title_sort:
meta('title_sort', mi.title_sort)
+ serialize_user_metadata(metadata, mi.get_all_user_metadata(False))
+
metadata[-1].tail = '\n' +(' '*4)
if mi.cover:
@@ -1335,5 +1398,28 @@ def suite():
def test():
unittest.TextTestRunner(verbosity=2).run(suite())
+def test_user_metadata():
+ from cStringIO import StringIO
+ mi = Metadata('Test title', ['test author1', 'test author2'])
+ um = {
+ '#myseries': { '#value#': u'test series\xe4', 'datatype':'text',
+ 'is_multiple': False, 'name': u'My Series'},
+ '#myseries_index': { '#value#': 2.45, 'datatype': 'float',
+ 'is_multiple': False}
+ }
+ mi.set_all_user_metadata(um)
+ raw = metadata_to_opf(mi)
+ opfc = OPFCreator(os.getcwd(), other=mi)
+ out = StringIO()
+ opfc.render(out)
+ raw2 = out.getvalue()
+ f = StringIO(raw)
+ opf = OPF(f)
+ f2 = StringIO(raw2)
+ opf2 = OPF(f2)
+ assert um == opf.user_metadata
+ assert um == opf2.user_metadata
+ print raw
+
if __name__ == '__main__':
- test()
+ test_user_metadata()
From 420db7851b8650ae7e61f2b441b9a7822dddbd8b Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Wed, 15 Sep 2010 21:50:06 -0600
Subject: [PATCH 036/207] Fix use of OPF class to generate a Metadata object
and have OPF.smart_update also update user metadata
---
src/calibre/customize/builtins.py | 3 +--
src/calibre/ebooks/conversion/plumber.py | 2 +-
src/calibre/ebooks/metadata/cli.py | 2 +-
src/calibre/ebooks/metadata/epub.py | 2 +-
src/calibre/ebooks/metadata/lit.py | 3 +--
src/calibre/ebooks/metadata/meta.py | 2 +-
src/calibre/ebooks/metadata/opf2.py | 24 +++++++++++++++++-------
src/calibre/ebooks/mobi/reader.py | 2 +-
src/calibre/ebooks/oeb/reader.py | 3 +--
src/calibre/gui2/add.py | 2 +-
src/calibre/library/cli.py | 2 +-
11 files changed, 27 insertions(+), 20 deletions(-)
diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py
index 68df832048..1ddb2843a1 100644
--- a/src/calibre/customize/builtins.py
+++ b/src/calibre/customize/builtins.py
@@ -226,8 +226,7 @@ class OPFMetadataReader(MetadataReaderPlugin):
def get_metadata(self, stream, ftype):
from calibre.ebooks.metadata.opf2 import OPF
- from calibre.ebooks.metadata import MetaInformation
- return MetaInformation(OPF(stream, os.getcwd()))
+ return OPF(stream, os.getcwd()).to_book_metadata()
class PDBMetadataReader(MetadataReaderPlugin):
diff --git a/src/calibre/ebooks/conversion/plumber.py b/src/calibre/ebooks/conversion/plumber.py
index 16282dd28d..38e47f6bf7 100644
--- a/src/calibre/ebooks/conversion/plumber.py
+++ b/src/calibre/ebooks/conversion/plumber.py
@@ -692,7 +692,7 @@ OptionRecommendation(name='timestamp',
self.opts.read_metadata_from_opf)
opf = OPF(open(self.opts.read_metadata_from_opf, 'rb'),
os.path.dirname(self.opts.read_metadata_from_opf))
- mi = MetaInformation(opf)
+ mi = opf.to_book_metadata()
self.opts_to_mi(mi)
if mi.cover:
if mi.cover.startswith('http:') or mi.cover.startswith('https:'):
diff --git a/src/calibre/ebooks/metadata/cli.py b/src/calibre/ebooks/metadata/cli.py
index 780d3febcf..a0be187512 100644
--- a/src/calibre/ebooks/metadata/cli.py
+++ b/src/calibre/ebooks/metadata/cli.py
@@ -109,7 +109,7 @@ def do_set_metadata(opts, mi, stream, stream_type):
from_opf = getattr(opts, 'from_opf', None)
if from_opf is not None:
from calibre.ebooks.metadata.opf2 import OPF
- opf_mi = MetaInformation(OPF(open(from_opf, 'rb')))
+ opf_mi = OPF(open(from_opf, 'rb')).to_book_metadata()
mi.smart_update(opf_mi)
for pref in config().option_set.preferences:
diff --git a/src/calibre/ebooks/metadata/epub.py b/src/calibre/ebooks/metadata/epub.py
index ac6b5feebe..8984a252a3 100644
--- a/src/calibre/ebooks/metadata/epub.py
+++ b/src/calibre/ebooks/metadata/epub.py
@@ -167,7 +167,7 @@ def get_metadata(stream, extract_cover=True):
""" Return metadata as a :class:`Metadata` object """
stream.seek(0)
reader = OCFZipReader(stream)
- mi = MetaInformation(reader.opf)
+ mi = reader.opf.to_book_metadata()
if extract_cover:
try:
cdata = get_cover(reader.opf, reader.opf_path, stream, reader=reader)
diff --git a/src/calibre/ebooks/metadata/lit.py b/src/calibre/ebooks/metadata/lit.py
index 1a267b6858..3be1f22632 100644
--- a/src/calibre/ebooks/metadata/lit.py
+++ b/src/calibre/ebooks/metadata/lit.py
@@ -6,7 +6,6 @@ Support for reading the metadata from a LIT file.
import cStringIO, os
-from calibre.ebooks.metadata import MetaInformation
from calibre.ebooks.metadata.opf2 import OPF
def get_metadata(stream):
@@ -16,7 +15,7 @@ def get_metadata(stream):
src = litfile.get_metadata().encode('utf-8')
litfile = litfile._litfile
opf = OPF(cStringIO.StringIO(src), os.getcwd())
- mi = MetaInformation(opf)
+ mi = opf.to_book_metadata()
covers = []
for item in opf.iterguide():
if 'cover' not in item.get('type', '').lower():
diff --git a/src/calibre/ebooks/metadata/meta.py b/src/calibre/ebooks/metadata/meta.py
index eae8171362..68deca5e10 100644
--- a/src/calibre/ebooks/metadata/meta.py
+++ b/src/calibre/ebooks/metadata/meta.py
@@ -194,7 +194,7 @@ def opf_metadata(opfpath):
try:
opf = OPF(f, os.path.dirname(opfpath))
if opf.application_id is not None:
- mi = MetaInformation(opf)
+ mi = opf.to_book_metadata()
if hasattr(opf, 'cover') and opf.cover:
cpath = os.path.join(os.path.dirname(opfpath), opf.cover)
if os.access(cpath, os.R_OK):
diff --git a/src/calibre/ebooks/metadata/opf2.py b/src/calibre/ebooks/metadata/opf2.py
index 236b2fa18f..96f1fa4832 100644
--- a/src/calibre/ebooks/metadata/opf2.py
+++ b/src/calibre/ebooks/metadata/opf2.py
@@ -530,7 +530,7 @@ class OPF(object): # {{{
self.read_user_metadata()
def read_user_metadata(self):
- self.user_metadata = {}
+ self._user_metadata_ = {}
from calibre.utils.config import from_json
elems = self.root.xpath('//*[name() = "meta" and starts-with(@name,'
'"calibre:user_metadata:") and @content]')
@@ -547,14 +547,21 @@ class OPF(object): # {{{
import traceback
traceback.print_exc()
continue
- self.user_metadata[name] = fm
+ self._user_metadata_[name] = fm
+ def to_book_metadata(self):
+ ans = MetaInformation(self)
+ for n, v in self._user_metadata_.items():
+ ans.set_user_metadata(n, v)
+ return ans
def write_user_metadata(self):
- for elem in self.user_metadata_path(self.root):
+ elems = self.root.xpath('//*[name() = "meta" and starts-with(@name,'
+ '"calibre:user_metadata:") and @content]')
+ for elem in elems:
elem.getparent().remove(elem)
serialize_user_metadata(self.metadata,
- self.user_metadata)
+ self._user_metadata_)
def find_toc(self):
self.toc = None
@@ -983,6 +990,9 @@ class OPF(object): # {{{
val = getattr(mi, attr, None)
if val is not None and val != [] and val != (None, None):
setattr(self, attr, val)
+ temp = self.to_book_metadata()
+ temp.smart_update(mi, replace_metadata=replace_metadata)
+ self._user_metadata_ = temp.get_all_user_metadata(True)
# }}}
@@ -1417,9 +1427,9 @@ def test_user_metadata():
opf = OPF(f)
f2 = StringIO(raw2)
opf2 = OPF(f2)
- assert um == opf.user_metadata
- assert um == opf2.user_metadata
- print raw
+ assert um == opf._user_metadata_
+ assert um == opf2._user_metadata_
+ print opf.render()
if __name__ == '__main__':
test_user_metadata()
diff --git a/src/calibre/ebooks/mobi/reader.py b/src/calibre/ebooks/mobi/reader.py
index 2a35c7cb45..6a44c2aa77 100644
--- a/src/calibre/ebooks/mobi/reader.py
+++ b/src/calibre/ebooks/mobi/reader.py
@@ -441,7 +441,7 @@ class MobiReader(object):
html.tostring(elem, encoding='utf-8') + ''
stream = cStringIO.StringIO(raw)
opf = OPF(stream)
- self.embedded_mi = MetaInformation(opf)
+ self.embedded_mi = opf.to_book_metadata()
if guide is not None:
for ref in guide.xpath('descendant::reference'):
if 'cover' in ref.get('type', '').lower():
diff --git a/src/calibre/ebooks/oeb/reader.py b/src/calibre/ebooks/oeb/reader.py
index d7d7bbf725..559421326c 100644
--- a/src/calibre/ebooks/oeb/reader.py
+++ b/src/calibre/ebooks/oeb/reader.py
@@ -126,10 +126,9 @@ class OEBReader(object):
def _metadata_from_opf(self, opf):
from calibre.ebooks.metadata.opf2 import OPF
- from calibre.ebooks.metadata import MetaInformation
from calibre.ebooks.oeb.transforms.metadata import meta_info_to_oeb_metadata
stream = cStringIO.StringIO(etree.tostring(opf))
- mi = MetaInformation(OPF(stream))
+ mi = OPF(stream).to_book_metadata()
if not mi.language:
mi.language = get_lang().replace('_', '-')
self.oeb.metadata.add('language', mi.language)
diff --git a/src/calibre/gui2/add.py b/src/calibre/gui2/add.py
index 5b9fb35be3..9f246aeb93 100644
--- a/src/calibre/gui2/add.py
+++ b/src/calibre/gui2/add.py
@@ -138,7 +138,7 @@ class DBAdder(Thread): # {{{
self.critical[name] = open(opf, 'rb').read().decode('utf-8', 'replace')
else:
try:
- mi = MetaInformation(OPF(opf))
+ mi = OPF(opf).to_book_metadata()
except:
import traceback
mi = MetaInformation('', [_('Unknown')])
diff --git a/src/calibre/library/cli.py b/src/calibre/library/cli.py
index 9a2d0b0a62..cd4e472807 100644
--- a/src/calibre/library/cli.py
+++ b/src/calibre/library/cli.py
@@ -448,7 +448,7 @@ def command_show_metadata(args, dbpath):
return 0
def do_set_metadata(db, id, stream):
- mi = OPF(stream)
+ mi = OPF(stream).to_book_metadata()
db.set_metadata(id, mi)
db.clean()
do_show_metadata(db, id, False)
From 4bc7aa1b710e00bdefdd82bed915c8a977b2523e Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Wed, 15 Sep 2010 21:56:02 -0600
Subject: [PATCH 037/207] More robust reading of user metadata from OPF
---
src/calibre/ebooks/metadata/opf2.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/src/calibre/ebooks/metadata/opf2.py b/src/calibre/ebooks/metadata/opf2.py
index 96f1fa4832..ecbef3194d 100644
--- a/src/calibre/ebooks/metadata/opf2.py
+++ b/src/calibre/ebooks/metadata/opf2.py
@@ -531,6 +531,7 @@ class OPF(object): # {{{
def read_user_metadata(self):
self._user_metadata_ = {}
+ temp = Metadata('x', ['x'])
from calibre.utils.config import from_json
elems = self.root.xpath('//*[name() = "meta" and starts-with(@name,'
'"calibre:user_metadata:") and @content]')
@@ -542,12 +543,13 @@ class OPF(object): # {{{
fm = elem.get('content')
try:
fm = json.loads(fm, object_hook=from_json)
+ temp.set_user_metadata(name, fm)
except:
prints('Failed to read user metadata:', name)
import traceback
traceback.print_exc()
continue
- self._user_metadata_[name] = fm
+ self._user_metadata_ = temp.get_all_user_metadata(True)
def to_book_metadata(self):
ans = MetaInformation(self)
From 56023722709d35816424cd3c77f5afe972103d0c Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Thu, 16 Sep 2010 12:24:56 +0100
Subject: [PATCH 038/207] Minor changes to OPF testing
---
src/calibre/ebooks/metadata/opf2.py | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/src/calibre/ebooks/metadata/opf2.py b/src/calibre/ebooks/metadata/opf2.py
index ecbef3194d..8a4ff6a5bd 100644
--- a/src/calibre/ebooks/metadata/opf2.py
+++ b/src/calibre/ebooks/metadata/opf2.py
@@ -1415,9 +1415,11 @@ def test_user_metadata():
mi = Metadata('Test title', ['test author1', 'test author2'])
um = {
'#myseries': { '#value#': u'test series\xe4', 'datatype':'text',
- 'is_multiple': False, 'name': u'My Series'},
+ 'is_multiple': None, 'name': u'My Series'},
'#myseries_index': { '#value#': 2.45, 'datatype': 'float',
- 'is_multiple': False}
+ 'is_multiple': None},
+ '#mytags': {'#value#':['t1','t2','t3'], 'datatype':'text',
+ 'is_multiple': '|', 'name': u'My Tags'}
}
mi.set_all_user_metadata(um)
raw = metadata_to_opf(mi)
From e1dd08acef1b14c73a3da542f974acd9a10a1f79 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Thu, 16 Sep 2010 14:02:49 +0100
Subject: [PATCH 039/207] Several changes: 1) Add an option to specify the time
format when sending to device. This is the analog of the same option that
already exists for save to disk. 2) refactor the format_field code. Remove
the special parameters on format_field. Add format_field_extended that
returns a 4-element tuple (name, formatted val, original val, field
metadata). 3) change (simplify) usbms collections management to use new
format_field_extended method. 4) change device.py to not call sync_booklists
twice. 5) add the fix for gui-not-updating, in hopes that we can avoid merge
conflicts
---
src/calibre/devices/usbms/books.py | 21 +++++++--------
src/calibre/devices/usbms/device.py | 4 ++-
src/calibre/ebooks/metadata/book/base.py | 33 ++++++++++++------------
src/calibre/gui2/actions/add.py | 2 +-
src/calibre/gui2/device.py | 28 ++++++++++++--------
src/calibre/gui2/preferences/sending.py | 3 +++
src/calibre/gui2/preferences/sending.ui | 15 ++++++++++-
src/calibre/library/save_to_disk.py | 3 +++
8 files changed, 68 insertions(+), 41 deletions(-)
diff --git a/src/calibre/devices/usbms/books.py b/src/calibre/devices/usbms/books.py
index d25787fc89..eab625f7be 100644
--- a/src/calibre/devices/usbms/books.py
+++ b/src/calibre/devices/usbms/books.py
@@ -94,12 +94,12 @@ class CollectionsBookList(BookList):
def supports_collections(self):
return True
- def compute_category_name(self, attr, category, cust_field_meta):
+ def compute_category_name(self, attr, category, field_meta):
renames = tweaks['sony_collection_renaming_rules']
attr_name = renames.get(attr, None)
if attr_name is None:
- if attr in cust_field_meta:
- attr_name = '(%s)'%cust_field_meta[attr]['name']
+ if field_meta['is_custom']:
+ attr_name = '(%s)'%field_meta['name']
else:
attr_name = ''
elif attr_name != '':
@@ -138,23 +138,23 @@ class CollectionsBookList(BookList):
# specified 'on_connect'
attrs = collection_attributes
meta_vals = book.get_all_non_none_attributes()
- cust_field_meta = book.get_all_user_metadata(make_copy=False)
for attr in attrs:
attr = attr.strip()
- ign, val = book.format_field(attr,
- ignore_series_index=True,
- return_multiples_as_list=True)
+ ign, val, orig_val, fm = book.format_field_extended(attr)
if not val: continue
if isbytestring(val):
val = val.decode(preferred_encoding, 'replace')
if isinstance(val, (list, tuple)):
val = list(val)
+ elif fm['datatype'] == 'series':
+ val = [orig_val]
+ elif fm['datatype'] == 'text' and fm['is_multiple']:
+ val = orig_val
else:
val = [val]
for category in val:
is_series = False
- if attr in cust_field_meta: # is a custom field
- fm = cust_field_meta[attr]
+ if fm['is_custom']: # is a custom field
if fm['datatype'] == 'text' and len(category) > 1 and \
category[0] == '[' and category[-1] == ']':
continue
@@ -168,8 +168,7 @@ class CollectionsBookList(BookList):
('series' in collection_attributes and
meta_vals.get('series', None) == category):
is_series = True
- cat_name = self.compute_category_name(attr, category,
- cust_field_meta)
+ cat_name = self.compute_category_name(attr, category, fm)
if cat_name not in collections:
collections[cat_name] = []
collections_lpaths[cat_name] = set()
diff --git a/src/calibre/devices/usbms/device.py b/src/calibre/devices/usbms/device.py
index b954911242..928d00ad4a 100644
--- a/src/calibre/devices/usbms/device.py
+++ b/src/calibre/devices/usbms/device.py
@@ -829,12 +829,14 @@ class Device(DeviceConfig, DevicePlugin):
ext = os.path.splitext(fname)[1]
from calibre.library.save_to_disk import get_components
+ from calibre.library.save_to_disk import config
+ opts = config().parse()
if not isinstance(template, unicode):
template = template.decode('utf-8')
app_id = str(getattr(mdata, 'application_id', ''))
# The db id will be in the created filename
extra_components = get_components(template, mdata, fname,
- length=250-len(app_id)-1)
+ timefmt=opts.send_timefmt, length=250-len(app_id)-1)
if not extra_components:
extra_components.append(sanitize(self.filename_callback(fname,
mdata)))
diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index 7405f20a7c..b252f518da 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -343,8 +343,11 @@ class Metadata(object):
def format_rating(self):
return unicode(self.rating)
- def format_field(self, key, ignore_series_index=False,
- return_multiples_as_list=False):
+ def format_field(self, key):
+ name, val, ign, ign = self.format_field_extended(key)
+ return (name, val)
+
+ def format_field_extended(self, key):
from calibre.ebooks.metadata import authors_to_string
'''
returns the tuple (field_name, formatted_value)
@@ -352,43 +355,41 @@ class Metadata(object):
if key in self.user_metadata_keys:
res = self.get(key, None)
if res is None or res == '':
- return (None, None)
+ return (None, None, None, None)
+ orig_res = res
cmeta = self.get_user_metadata(key, make_copy=False)
name = unicode(cmeta['name'])
datatype = cmeta['datatype']
if datatype == 'text' and cmeta['is_multiple']:
- if not return_multiples_as_list:
- res = u', '.join(res)
+ res = u', '.join(res)
elif datatype == 'series':
- if not ignore_series_index:
- res = res + \
- ' [%s]'%self.format_series_index(val=self.get_extra(key))
+ res = res + \
+ ' [%s]'%self.format_series_index(val=self.get_extra(key))
elif datatype == 'datetime':
res = format_date(res, cmeta['display'].get('date_format','dd MMM yyyy'))
elif datatype == 'bool':
res = _('Yes') if res else _('No')
- return (name, res)
+ return (name, res, orig_res, cmeta)
if key in field_metadata and field_metadata[key]['kind'] == 'field':
res = self.get(key, None)
if res is None or res == '':
- return (None, None)
+ return (None, None, None, None)
+ orig_res = res
fmeta = field_metadata[key]
name = unicode(fmeta['name'])
datatype = fmeta['datatype']
if key == 'authors':
res = authors_to_string(res)
elif datatype == 'text' and fmeta['is_multiple']:
- if not return_multiples_as_list:
- res = u', '.join(res)
+ res = u', '.join(res)
elif datatype == 'series':
- if not ignore_series_index:
- res = res + ' [%s]'%self.format_series_index()
+ res = res + ' [%s]'%self.format_series_index()
elif datatype == 'datetime':
res = format_date(res, fmeta['display'].get('date_format','dd MMM yyyy'))
- return (name, res)
+ return (name, res, orig_res, fmeta)
- return (None, None)
+ return (None, None, None, None)
def __unicode__(self):
from calibre.ebooks.metadata import authors_to_string
diff --git a/src/calibre/gui2/actions/add.py b/src/calibre/gui2/actions/add.py
index add7bf1d5b..aa20b8bc16 100644
--- a/src/calibre/gui2/actions/add.py
+++ b/src/calibre/gui2/actions/add.py
@@ -230,7 +230,7 @@ class AddAction(InterfaceAction):
self._files_added(paths, names, infos, on_card=on_card)
# set the in-library flags, and as a consequence send the library's
# metadata for this book to the device. This sets the uuid to the
- # correct value.
+ # correct value. Note that set_books_in_library might sync_booklists
self.gui.set_books_in_library(booklists=[model.db], reset=True)
model.reset()
diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py
index f839e1d519..196e97f2a3 100644
--- a/src/calibre/gui2/device.py
+++ b/src/calibre/gui2/device.py
@@ -745,6 +745,7 @@ class DeviceMixin(object): # {{{
if job.failed:
self.device_job_exception(job)
return
+ # set_books_in_library might schedule a sync_booklists job
self.set_books_in_library(job.result, reset=True)
mainlist, cardalist, cardblist = job.result
self.memory_view.set_database(mainlist)
@@ -789,11 +790,12 @@ class DeviceMixin(object): # {{{
self.device_manager.remove_books_from_metadata(paths,
self.booklists())
model.paths_deleted(paths)
- self.upload_booklists()
# Force recomputation the library's ondevice info. We need to call
# set_books_in_library even though books were not added because
- # the deleted book might have been an exact match.
- self.set_books_in_library(self.booklists(), reset=True)
+ # the deleted book might have been an exact match. Upload the booklists
+ # if set_books_in_library did not.
+ if not self.set_books_in_library(self.booklists(), reset=True):
+ self.upload_booklists()
self.book_on_device(None, None, reset=True)
# We need to reset the ondevice flags in the library. Use a big hammer,
# so we don't need to worry about whether some succeeded or not.
@@ -1280,8 +1282,6 @@ class DeviceMixin(object): # {{{
self.device_manager.add_books_to_metadata(job.result,
metadata, self.booklists())
- self.upload_booklists()
-
books_to_be_deleted = []
if memory and memory[1]:
books_to_be_deleted = memory[1]
@@ -1291,12 +1291,15 @@ class DeviceMixin(object): # {{{
# book already there with a different book. This happens frequently in
# news. When this happens, the book match indication will be wrong
# because the UUID changed. Force both the device and the library view
- # to refresh the flags.
- self.set_books_in_library(self.booklists(), reset=True)
+ # to refresh the flags. Set_books_in_library could upload the booklists.
+ # If it does not, then do it here.
+ if not self.set_books_in_library(self.booklists(), reset=True):
+ self.upload_booklists()
self.book_on_device(None, reset=True)
self.refresh_ondevice_info(device_connected = True)
- view = self.card_a_view if on_card == 'carda' else self.card_b_view if on_card == 'cardb' else self.memory_view
+ view = self.card_a_view if on_card == 'carda' else \
+ self.card_b_view if on_card == 'cardb' else self.memory_view
view.model().resort(reset=False)
view.model().research()
for f in files:
@@ -1371,7 +1374,7 @@ class DeviceMixin(object): # {{{
try:
db = self.library_view.model().db
except:
- return
+ return False
# Build a cache (map) of the library, so the search isn't On**2
self.db_book_title_cache = {}
self.db_book_uuid_cache = {}
@@ -1466,10 +1469,13 @@ class DeviceMixin(object): # {{{
# Set author_sort if it isn't already
asort = getattr(book, 'author_sort', None)
if not asort and book.authors:
- book.author_sort = self.library_view.model().db.author_sort_from_authors(book.authors)
+ book.author_sort = self.library_view.model().db.\
+ author_sort_from_authors(book.authors)
if update_metadata:
if self.device_manager.is_device_connected:
- self.device_manager.sync_booklists(None, booklists)
+ self.device_manager.sync_booklists(
+ Dispatcher(self.metadata_synced), booklists)
+ return update_metadata
# }}}
diff --git a/src/calibre/gui2/preferences/sending.py b/src/calibre/gui2/preferences/sending.py
index 748c6b2a2d..ac4abbcf41 100644
--- a/src/calibre/gui2/preferences/sending.py
+++ b/src/calibre/gui2/preferences/sending.py
@@ -22,6 +22,9 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
r = self.register
+ for x in ('send_timefmt',):
+ r(x, self.proxy)
+
choices = [(_('Manual management'), 'manual'),
(_('Only on send'), 'on_send'),
(_('Automatic management'), 'on_connect')]
diff --git a/src/calibre/gui2/preferences/sending.ui b/src/calibre/gui2/preferences/sending.ui
index e064646afd..b9d1d1e1d2 100644
--- a/src/calibre/gui2/preferences/sending.ui
+++ b/src/calibre/gui2/preferences/sending.ui
@@ -80,7 +80,20 @@
-
+
+
+
+ Format &dates as:
+
+
+ opt_send_timefmt
+
+
+
+
+
+
+ Here you can control how calibre will save your books when you click the Send to Device button. This setting can be overriden for individual devices by customizing the device interface plugins in Preferences->Advanced->Plugins
diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py
index 3fa40c68b2..71850abcd5 100644
--- a/src/calibre/library/save_to_disk.py
+++ b/src/calibre/library/save_to_disk.py
@@ -84,6 +84,9 @@ def config(defaults=None):
x('timefmt', default='%b, %Y',
help=_('The format in which to display dates. %d - day, %b - month, '
'%Y - year. Default is: %b, %Y'))
+ x('send_timefmt', default='%b, %Y',
+ help=_('The format in which to display dates. %d - day, %b - month, '
+ '%Y - year. Default is: %b, %Y'))
x('to_lowercase', default=False,
help=_('Convert paths to lowercase.'))
x('replace_whitespace', default=False,
From 4645138a67193537ba3e91fc3e4942017d72e1de Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Thu, 16 Sep 2010 16:03:07 +0100
Subject: [PATCH 040/207] 1) Re-enable syntactic validation of save templates.
2) fix row numbering on send_to_device preferences ui template.
---
src/calibre/gui2/preferences/save_template.py | 37 +++++++++++--------
src/calibre/gui2/preferences/sending.ui | 2 +-
2 files changed, 22 insertions(+), 17 deletions(-)
diff --git a/src/calibre/gui2/preferences/save_template.py b/src/calibre/gui2/preferences/save_template.py
index 26dc02f259..0dbee5bf21 100644
--- a/src/calibre/gui2/preferences/save_template.py
+++ b/src/calibre/gui2/preferences/save_template.py
@@ -8,8 +8,10 @@ __docformat__ = 'restructuredtext en'
from PyQt4.Qt import QWidget, pyqtSignal
+from calibre.gui2 import error_dialog
from calibre.gui2.preferences.save_template_ui import Ui_Form
-from calibre.library.save_to_disk import FORMAT_ARG_DESCS
+from calibre.library.save_to_disk import FORMAT_ARG_DESCS, preprocess_template,\
+ safe_format
class SaveTemplate(QWidget, Ui_Form):
@@ -24,8 +26,11 @@ class SaveTemplate(QWidget, Ui_Form):
variables = sorted(FORMAT_ARG_DESCS.keys())
rows = []
for var in variables:
- rows.append(u'
%s
%s
'%
+ rows.append(u'
%s
%s
'%
(var, FORMAT_ARG_DESCS[var]))
+ rows.append(u'
%s
%s
'%(
+ _('Any custom field'),
+ _('The lookup name of any custom field. These names begin with "#")')))
table = u'
%s
'%(u'\n'.join(rows))
self.template_variables.setText(table)
@@ -39,21 +44,21 @@ class SaveTemplate(QWidget, Ui_Form):
self.changed_signal.emit()
def validate(self):
- # TODO: NEWMETA: I haven't figured out how to get the custom columns
- # into here, so for the moment make all templates valid.
+ '''
+ Do a syntax check on the format string. Doing a semantic check
+ (verifying that the fields exist) is not useful in the presence of
+ custom fields, because they may or may not exist.
+ '''
+ tmpl = preprocess_template(self.opt_template.text())
+ fa = {}
+ try:
+ safe_format(tmpl, fa)
+ except Exception, err:
+ error_dialog(self, _('Invalid template'),
+ '
'+_('The template %s is invalid:')%tmpl + \
+ ' '+str(err), show=True)
+ return False
return True
-# tmpl = preprocess_template(self.opt_template.text())
-# fa = {}
-# for x in FORMAT_ARG_DESCS.keys():
-# fa[x]='random long string'
-# try:
-# tmpl.format(**fa)
-# except Exception, err:
-# error_dialog(self, _('Invalid template'),
-# '
-
+
From 788128627459035cf0e87fc6b0baa4da708cef3d Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sat, 18 Sep 2010 11:08:33 +0100
Subject: [PATCH 041/207] 1) add the composite field custom datatype 2) clean
up content server code so it uses the new formatting facilities
---
src/calibre/devices/usbms/books.py | 7 ++-
src/calibre/ebooks/metadata/book/__init__.py | 7 ++-
src/calibre/ebooks/metadata/book/base.py | 35 +++++++++---
src/calibre/gui2/library/models.py | 33 ++++++++++--
src/calibre/gui2/preferences/columns.py | 3 +-
.../gui2/preferences/create_custom_column.py | 30 ++++++++---
.../gui2/preferences/create_custom_column.ui | 53 ++++++++++++++++++-
src/calibre/library/custom_columns.py | 6 +--
src/calibre/library/database2.py | 1 +
src/calibre/library/field_metadata.py | 2 +-
src/calibre/library/server/mobile.py | 36 +++++--------
src/calibre/library/server/opds.py | 24 ++++-----
src/calibre/library/server/xml.py | 40 ++++++--------
13 files changed, 181 insertions(+), 96 deletions(-)
diff --git a/src/calibre/devices/usbms/books.py b/src/calibre/devices/usbms/books.py
index eab625f7be..13fcb90b49 100644
--- a/src/calibre/devices/usbms/books.py
+++ b/src/calibre/devices/usbms/books.py
@@ -137,7 +137,6 @@ class CollectionsBookList(BookList):
# For existing books, modify the collections only if the user
# specified 'on_connect'
attrs = collection_attributes
- meta_vals = book.get_all_non_none_attributes()
for attr in attrs:
attr = attr.strip()
ign, val, orig_val, fm = book.format_field_extended(attr)
@@ -166,7 +165,7 @@ class CollectionsBookList(BookList):
continue
if attr == 'series' or \
('series' in collection_attributes and
- meta_vals.get('series', None) == category):
+ book.get('series', None) == category):
is_series = True
cat_name = self.compute_category_name(attr, category, fm)
if cat_name not in collections:
@@ -177,10 +176,10 @@ class CollectionsBookList(BookList):
collections_lpaths[cat_name].add(lpath)
if is_series:
collections[cat_name].append(
- (book, meta_vals.get(attr+'_index', sys.maxint)))
+ (book, book.get(attr+'_index', sys.maxint)))
else:
collections[cat_name].append(
- (book, meta_vals.get('title_sort', 'zzzz')))
+ (book, book.get('title_sort', 'zzzz')))
# Sort collections
result = {}
for category, books in collections.items():
diff --git a/src/calibre/ebooks/metadata/book/__init__.py b/src/calibre/ebooks/metadata/book/__init__.py
index e087f8072d..e6dff9110b 100644
--- a/src/calibre/ebooks/metadata/book/__init__.py
+++ b/src/calibre/ebooks/metadata/book/__init__.py
@@ -81,9 +81,8 @@ DEVICE_METADATA_FIELDS = frozenset([
CALIBRE_METADATA_FIELDS = frozenset([
'application_id', # An application id, currently set to the db_id.
- # the calibre primary key of the item.
'db_id', # the calibre primary key of the item.
- # TODO: NEWMETA: May want to remove once Sony's no longer use it
+ 'formats', # list of formats (extensions) for this book
]
)
@@ -124,5 +123,5 @@ SERIALIZABLE_FIELDS = SOCIAL_METADATA_FIELDS.union(
PUBLICATION_METADATA_FIELDS).union(
CALIBRE_METADATA_FIELDS).union(
DEVICE_METADATA_FIELDS) - \
- frozenset(['device_collections'])
- # device_collections is rebuilt when needed
+ frozenset(['device_collections', 'formats'])
+ # these are rebuilt when needed
diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index b252f518da..31485dfe1b 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -5,8 +5,7 @@ __license__ = 'GPL v3'
__copyright__ = '2010, Kovid Goyal '
__docformat__ = 'restructuredtext en'
-import copy
-import traceback
+import copy, re, string, traceback
from calibre import prints
from calibre.ebooks.metadata.book import SC_COPYABLE_FIELDS
@@ -33,6 +32,23 @@ NULL_VALUES = {
field_metadata = FieldMetadata()
+class SafeFormat(string.Formatter):
+ '''
+ Provides a format function that substitutes '' for any missing value
+ '''
+ def get_value(self, key, args, mi):
+ ign, v = mi.format_field(key, series_with_index=False)
+ if v is None:
+ return ''
+ return v
+
+composite_formatter = SafeFormat()
+compress_spaces = re.compile(r'\s+')
+
+def format_composite(x, mi):
+ ans = composite_formatter.vformat(x, [], mi).strip()
+ return compress_spaces.sub(' ', ans)
+
class Metadata(object):
'''
@@ -343,18 +359,19 @@ class Metadata(object):
def format_rating(self):
return unicode(self.rating)
- def format_field(self, key):
- name, val, ign, ign = self.format_field_extended(key)
+ def format_field(self, key, series_with_index=True):
+ name, val, ign, ign = self.format_field_extended(key, series_with_index)
return (name, val)
- def format_field_extended(self, key):
+ def format_field_extended(self, key, series_with_index=True):
from calibre.ebooks.metadata import authors_to_string
'''
returns the tuple (field_name, formatted_value)
'''
if key in self.user_metadata_keys:
res = self.get(key, None)
- if res is None or res == '':
+ cmeta = self.get_user_metadata(key, make_copy=False)
+ if cmeta['datatype'] != 'composite' and (res is None or res == ''):
return (None, None, None, None)
orig_res = res
cmeta = self.get_user_metadata(key, make_copy=False)
@@ -362,13 +379,15 @@ class Metadata(object):
datatype = cmeta['datatype']
if datatype == 'text' and cmeta['is_multiple']:
res = u', '.join(res)
- elif datatype == 'series':
+ elif datatype == 'series' and series_with_index:
res = res + \
' [%s]'%self.format_series_index(val=self.get_extra(key))
elif datatype == 'datetime':
res = format_date(res, cmeta['display'].get('date_format','dd MMM yyyy'))
elif datatype == 'bool':
res = _('Yes') if res else _('No')
+ elif datatype == 'composite':
+ res = format_composite(cmeta['display']['composite_template'], self)
return (name, res, orig_res, cmeta)
if key in field_metadata and field_metadata[key]['kind'] == 'field':
@@ -383,7 +402,7 @@ class Metadata(object):
res = authors_to_string(res)
elif datatype == 'text' and fmeta['is_multiple']:
res = u', '.join(res)
- elif datatype == 'series':
+ elif datatype == 'series' and series_with_index:
res = res + ' [%s]'%self.format_series_index()
elif datatype == 'datetime':
res = format_date(res, fmeta['display'].get('date_format','dd MMM yyyy'))
diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index e9e688c93b..7839b89d7e 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -86,6 +86,7 @@ class BooksModel(QAbstractTableModel): # {{{
self.last_search = '' # The last search performed on this model
self.column_map = []
self.headers = {}
+ self.metadata_cache = {}
self.alignment_map = {}
self.buffer_size = buffer
self.cover_cache = None
@@ -114,6 +115,16 @@ class BooksModel(QAbstractTableModel): # {{{
def clear_caches(self):
if self.cover_cache:
self.cover_cache.clear_cache()
+ self.metadata_cache = {}
+
+ def get_cached_metadata(self, idx):
+ if idx not in self.metadata_cache:
+ self.metadata_cache[idx] = self.db.get_metadata(idx)
+ return self.metadata_cache[idx]
+
+ def remove_cached_metadata(self, idx):
+ if idx in self.metadata_cache:
+ del self.metadata_cache[idx]
def read_config(self):
self.use_roman_numbers = config['use_roman_numerals_for_series_number']
@@ -146,6 +157,7 @@ class BooksModel(QAbstractTableModel): # {{{
elif col in self.custom_columns:
self.headers[col] = self.custom_columns[col]['name']
+ self.metadata_cache = {}
self.build_data_convertors()
self.reset()
self.database_changed.emit(db)
@@ -159,11 +171,13 @@ class BooksModel(QAbstractTableModel): # {{{
db.add_listener(refresh_cover)
def refresh_ids(self, ids, current_row=-1):
+ self.metadata_cache = {}
rows = self.db.refresh_ids(ids)
if rows:
self.refresh_rows(rows, current_row=current_row)
def refresh_rows(self, rows, current_row=-1):
+ self.metadata_cache = {}
for row in rows:
if row == current_row:
self.new_bookdisplay_data.emit(
@@ -193,6 +207,7 @@ class BooksModel(QAbstractTableModel): # {{{
return ret
def count_changed(self, *args):
+ self.metadata_cache = {}
self.count_changed_signal.emit(self.db.count())
def row_indices(self, index):
@@ -262,6 +277,7 @@ class BooksModel(QAbstractTableModel): # {{{
self.sorting_done.emit(self.db.index)
def refresh(self, reset=True):
+ self.metadata_cache = {}
self.db.refresh(field=None)
self.resort(reset=reset)
@@ -318,7 +334,7 @@ class BooksModel(QAbstractTableModel): # {{{
data[_('Series')] = \
_('Book %s of %s.')%\
(sidx, prepare_string_for_xml(series))
- mi = self.db.get_metadata(idx)
+ mi = self.get_cached_metadata(idx)
for key in mi.user_metadata_keys:
name, val = mi.format_field(key)
if val is not None:
@@ -327,6 +343,7 @@ class BooksModel(QAbstractTableModel): # {{{
def set_cache(self, idx):
l, r = 0, self.count()-1
+ self.remove_cached_metadata(idx)
if self.cover_cache is not None:
l = max(l, idx-self.buffer_size)
r = min(r, idx+self.buffer_size)
@@ -586,6 +603,10 @@ class BooksModel(QAbstractTableModel): # {{{
def number_type(r, idx=-1):
return QVariant(self.db.data[r][idx])
+ def composite_type(r, key=None):
+ mi = self.get_cached_metadata(r)
+ return QVariant(mi.format_field(key)[1])
+
self.dc = {
'title' : functools.partial(text_type,
idx=self.db.field_metadata['title']['rec_index'], mult=False),
@@ -620,7 +641,8 @@ class BooksModel(QAbstractTableModel): # {{{
idx = self.custom_columns[col]['rec_index']
datatype = self.custom_columns[col]['datatype']
if datatype in ('text', 'comments'):
- self.dc[col] = functools.partial(text_type, idx=idx, mult=self.custom_columns[col]['is_multiple'])
+ self.dc[col] = functools.partial(text_type, idx=idx,
+ mult=self.custom_columns[col]['is_multiple'])
elif datatype in ('int', 'float'):
self.dc[col] = functools.partial(number_type, idx=idx)
elif datatype == 'datetime':
@@ -628,13 +650,15 @@ class BooksModel(QAbstractTableModel): # {{{
elif datatype == 'bool':
self.dc[col] = functools.partial(bool_type, idx=idx)
self.dc_decorator[col] = functools.partial(
- bool_type_decorator, idx=idx,
- bool_cols_are_tristate=tweaks['bool_custom_columns_are_tristate'] == 'yes')
+ bool_type_decorator, idx=idx,
+ bool_cols_are_tristate=tweaks['bool_custom_columns_are_tristate'] == 'yes')
elif datatype == 'rating':
self.dc[col] = functools.partial(rating_type, idx=idx)
elif datatype == 'series':
self.dc[col] = functools.partial(series_type, idx=idx,
siix=self.db.field_metadata.cc_series_index_column_for(col))
+ elif datatype == 'composite':
+ self.dc[col] = functools.partial(composite_type, key=col)
else:
print 'What type is this?', col, datatype
# build a index column to data converter map, to remove the string lookup in the data loop
@@ -729,6 +753,7 @@ class BooksModel(QAbstractTableModel): # {{{
if role == Qt.EditRole:
row, col = index.row(), index.column()
column = self.column_map[col]
+ self.remove_cached_metadata(row)
if self.is_custom_column(column):
if not self.set_custom_column_data(row, column, value):
return False
diff --git a/src/calibre/gui2/preferences/columns.py b/src/calibre/gui2/preferences/columns.py
index c1b9230f42..761a9880b1 100644
--- a/src/calibre/gui2/preferences/columns.py
+++ b/src/calibre/gui2/preferences/columns.py
@@ -155,7 +155,8 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
name=self.custcols[c]['name'],
datatype=self.custcols[c]['datatype'],
is_multiple=self.custcols[c]['is_multiple'],
- display = self.custcols[c]['display'])
+ display = self.custcols[c]['display'],
+ editable = self.custcols[c]['editable'])
must_restart = True
elif '*deleteme' in self.custcols[c]:
db.delete_custom_column(label=self.custcols[c]['label'])
diff --git a/src/calibre/gui2/preferences/create_custom_column.py b/src/calibre/gui2/preferences/create_custom_column.py
index e8ab8707e2..4b21301ccd 100644
--- a/src/calibre/gui2/preferences/create_custom_column.py
+++ b/src/calibre/gui2/preferences/create_custom_column.py
@@ -38,6 +38,8 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
'is_multiple':False},
8:{'datatype':'bool',
'text':_('Yes/No'), 'is_multiple':False},
+ 8:{'datatype':'composite',
+ 'text':_('Field built from other fields'), 'is_multiple':False},
}
def __init__(self, parent, editing, standard_colheads, standard_colnames):
@@ -86,6 +88,8 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
if ct == 'datetime':
if c['display'].get('date_format', None):
self.date_format_box.setText(c['display'].get('date_format', ''))
+ elif ct == 'composite':
+ self.composite_box.setText(c['display'].get('composite_template', ''))
self.datatype_changed()
self.exec_()
@@ -94,9 +98,10 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
col_type = self.column_types[self.column_type_box.currentIndex()]['datatype']
except:
col_type = None
- df_visible = col_type == 'datetime'
for x in ('box', 'default_label', 'label'):
- getattr(self, 'date_format_'+x).setVisible(df_visible)
+ getattr(self, 'date_format_'+x).setVisible(col_type == 'datetime')
+ for x in ('box', 'default_label', 'label'):
+ getattr(self, 'composite_'+x).setVisible(col_type == 'composite')
def accept(self):
@@ -122,6 +127,7 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
bad_col = True
if bad_col:
return self.simple_error('', _('The lookup name %s is already used')%col)
+
bad_head = False
for t in self.parent.custcols:
if self.parent.custcols[t]['name'] == col_heading:
@@ -133,12 +139,20 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
if bad_head:
return self.simple_error('', _('The heading %s is already used')%col_heading)
- date_format = {}
+ display_dict = {}
if col_type == 'datetime':
if self.date_format_box.text():
- date_format = {'date_format':unicode(self.date_format_box.text())}
+ display_dict = {'date_format':unicode(self.date_format_box.text())}
else:
- date_format = {'date_format': None}
+ display_dict = {'date_format': None}
+
+ if col_type == 'composite':
+ if not self.composite_box.text():
+ return self.simple_error('', _('You must enter a template for composite fields')%col_heading)
+ display_dict = {'composite_template':unicode(self.composite_box.text())}
+ is_editable = False
+ else:
+ is_editable = True
db = self.parent.gui.library_view.model().db
key = db.field_metadata.custom_field_prefix+col
@@ -148,8 +162,8 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
'label':col,
'name':col_heading,
'datatype':col_type,
- 'editable':True,
- 'display':date_format,
+ 'editable':is_editable,
+ 'display':display_dict,
'normalized':None,
'colnum':None,
'is_multiple':is_multiple,
@@ -164,7 +178,7 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
item.setText(col_heading)
self.parent.custcols[self.orig_column_name]['label'] = col
self.parent.custcols[self.orig_column_name]['name'] = col_heading
- self.parent.custcols[self.orig_column_name]['display'].update(date_format)
+ self.parent.custcols[self.orig_column_name]['display'].update(display_dict)
self.parent.custcols[self.orig_column_name]['*edited'] = True
self.parent.custcols[self.orig_column_name]['*must_restart'] = True
QDialog.accept(self)
diff --git a/src/calibre/gui2/preferences/create_custom_column.ui b/src/calibre/gui2/preferences/create_custom_column.ui
index 5cb9494845..640becca8c 100644
--- a/src/calibre/gui2/preferences/create_custom_column.ui
+++ b/src/calibre/gui2/preferences/create_custom_column.ui
@@ -147,9 +147,59 @@
+
+
+
+
+
+
+ 0
+ 0
+
+
+
+ <p>Field template. Uses the same syntax as save templates.
+
+
+
+
+
+
+ Similar to save templates. For example, {title} {isbn}
+
+
+ Default: (nothing)
+
+
+
+
+
+
+
+
+ &Template
+
+
+ composite_box
+
+
+
+
+
+
+ Qt::Vertical
+
+
+
+ 20
+ 40
+
+
+
+
-
+ Qt::Horizontal
@@ -184,6 +234,7 @@
column_heading_boxcolumn_type_boxdate_format_box
+ composite_boxbutton_box
diff --git a/src/calibre/library/custom_columns.py b/src/calibre/library/custom_columns.py
index 4ba664dadc..d74024280e 100644
--- a/src/calibre/library/custom_columns.py
+++ b/src/calibre/library/custom_columns.py
@@ -18,7 +18,7 @@ from calibre.utils.date import parse_date
class CustomColumns(object):
CUSTOM_DATA_TYPES = frozenset(['rating', 'text', 'comments', 'datetime',
- 'int', 'float', 'bool', 'series'])
+ 'int', 'float', 'bool', 'series', 'composite'])
def custom_table_names(self, num):
return 'custom_column_%d'%num, 'books_custom_column_%d_link'%num
@@ -540,7 +540,7 @@ class CustomColumns(object):
if datatype not in self.CUSTOM_DATA_TYPES:
raise ValueError('%r is not a supported data type'%datatype)
normalized = datatype not in ('datetime', 'comments', 'int', 'bool',
- 'float')
+ 'float', 'composite')
is_multiple = is_multiple and datatype in ('text',)
num = self.conn.execute(
('INSERT INTO '
@@ -551,7 +551,7 @@ class CustomColumns(object):
if datatype in ('rating', 'int'):
dt = 'INT'
- elif datatype in ('text', 'comments', 'series'):
+ elif datatype in ('text', 'comments', 'series', 'composite'):
dt = 'TEXT'
elif datatype in ('float',):
dt = 'REAL'
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index 9e9e75a26e..d06d217b76 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -538,6 +538,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
mi.pubdate = self.pubdate(idx, index_is_id=index_is_id)
mi.uuid = self.uuid(idx, index_is_id=index_is_id)
mi.title_sort = self.title_sort(idx, index_is_id=index_is_id)
+ mi.formats = self.formats(idx, index_is_id=index_is_id).split(',')
tags = self.tags(idx, index_is_id=index_is_id)
if tags:
mi.tags = [i.strip() for i in tags.split(',')]
diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py
index 2773f573b2..dcdfcfd9d6 100644
--- a/src/calibre/library/field_metadata.py
+++ b/src/calibre/library/field_metadata.py
@@ -68,7 +68,7 @@ class FieldMetadata(dict):
'''
VALID_DATA_TYPES = frozenset([None, 'rating', 'text', 'comments', 'datetime',
- 'int', 'float', 'bool', 'series'])
+ 'int', 'float', 'bool', 'series', 'composite'])
# Builtin metadata {{{
diff --git a/src/calibre/library/server/mobile.py b/src/calibre/library/server/mobile.py
index ab5b39eed8..8e7c75b0ac 100644
--- a/src/calibre/library/server/mobile.py
+++ b/src/calibre/library/server/mobile.py
@@ -228,29 +228,19 @@ class MobileServer(object):
for key in CKEYS:
def concat(name, val):
return '%s:#:%s'%(name, unicode(val))
- val = record[CFM[key]['rec_index']]
- if val:
- datatype = CFM[key]['datatype']
- if datatype in ['comments']:
- continue
- name = CFM[key]['name']
- if datatype == 'text' and CFM[key]['is_multiple']:
- book[key] = concat(name,
- format_tag_string(val, '|',
- no_tag_count=True))
- elif datatype == 'series':
- book[key] = concat(name, '%s [%s]'%(val,
- fmt_sidx(record[CFM.cc_series_index_column_for(key)])))
- elif datatype == 'datetime':
- book[key] = concat(name,
- format_date(val, CFM[key]['display'].get('date_format','dd MMM yyyy')))
- elif datatype == 'bool':
- if val:
- book[key] = concat(name, __builtin__._('Yes'))
- else:
- book[key] = concat(name, __builtin__._('No'))
- else:
- book[key] = concat(name, val)
+ mi = self.db.get_metadata(record[CFM['id']['rec_index']], index_is_id=True)
+ name, val = mi.format_field(key)
+ if val is None:
+ continue
+ datatype = CFM[key]['datatype']
+ if datatype in ['comments']:
+ continue
+ if datatype == 'text' and CFM[key]['is_multiple']:
+ book[key] = concat(name,
+ format_tag_string(val, ',',
+ no_tag_count=True))
+ else:
+ book[key] = concat(name, val)
updated = self.db.last_modified()
diff --git a/src/calibre/library/server/opds.py b/src/calibre/library/server/opds.py
index e495598a2f..0eb7379ac5 100644
--- a/src/calibre/library/server/opds.py
+++ b/src/calibre/library/server/opds.py
@@ -132,7 +132,8 @@ def CATALOG_GROUP_ENTRY(item, category, base_href, version, updated):
link
)
-def ACQUISITION_ENTRY(item, version, FM, updated, CFM, CKEYS):
+def ACQUISITION_ENTRY(item, version, db, updated, CFM, CKEYS):
+ FM = db.FIELD_MAP
title = item[FM['title']]
if not title:
title = _('Unknown')
@@ -157,22 +158,16 @@ def ACQUISITION_ENTRY(item, version, FM, updated, CFM, CKEYS):
(series,
fmt_sidx(float(item[FM['series_index']]))))
for key in CKEYS:
- val = item[CFM[key]['rec_index']]
+ mi = db.get_metadata(item[CFM['id']['rec_index']], index_is_id=True)
+ name, val = mi.format_field(key)
if val is not None:
- name = CFM[key]['name']
datatype = CFM[key]['datatype']
if datatype == 'text' and CFM[key]['is_multiple']:
- extra.append('%s: %s '%(name, format_tag_string(val, '|',
+ extra.append('%s: %s '%(name, format_tag_string(val, ',',
ignore_max=True,
no_tag_count=True)))
- elif datatype == 'series':
- extra.append('%s: %s [%s] '%(name, val,
- fmt_sidx(item[CFM.cc_series_index_column_for(key)])))
- elif datatype == 'datetime':
- extra.append('%s: %s '%(name,
- format_date(val, CFM[key]['display'].get('date_format','dd MMM yyyy'))))
else:
- extra.append('%s: %s ' % (CFM[key]['name'], val))
+ extra.append('%s: %s '%(name, val))
comments = item[FM['comments']]
if comments:
comments = comments_to_html(comments)
@@ -280,13 +275,14 @@ class NavFeed(Feed):
class AcquisitionFeed(NavFeed):
def __init__(self, updated, id_, items, offsets, page_url, up_url, version,
- FM, CFM):
+ db):
NavFeed.__init__(self, id_, updated, version, offsets, page_url, up_url)
+ CFM = db.field_metadata
CKEYS = [key for key in sorted(CFM.get_custom_fields(),
cmp=lambda x,y: cmp(CFM[x]['name'].lower(),
CFM[y]['name'].lower()))]
for item in items:
- self.root.append(ACQUISITION_ENTRY(item, version, FM, updated,
+ self.root.append(ACQUISITION_ENTRY(item, version, db, updated,
CFM, CKEYS))
class CategoryFeed(NavFeed):
@@ -384,7 +380,7 @@ class OPDSServer(object):
cherrypy.response.headers['Last-Modified'] = self.last_modified(updated)
cherrypy.response.headers['Content-Type'] = 'application/atom+xml;profile=opds-catalog'
return str(AcquisitionFeed(updated, id_, items, offsets,
- page_url, up_url, version, self.db.FIELD_MAP, self.db.field_metadata))
+ page_url, up_url, version, self.db))
def opds_search(self, query=None, version=0, offset=0):
try:
diff --git a/src/calibre/library/server/xml.py b/src/calibre/library/server/xml.py
index 8715dda7d0..7f5bc31e70 100644
--- a/src/calibre/library/server/xml.py
+++ b/src/calibre/library/server/xml.py
@@ -102,31 +102,21 @@ class XMLServer(object):
for key in CKEYS:
def concat(name, val):
return '%s:#:%s'%(name, unicode(val))
- val = record[CFM[key]['rec_index']]
- if val:
- datatype = CFM[key]['datatype']
- if datatype in ['comments']:
- continue
- k = str('CF_'+key[1:])
- name = CFM[key]['name']
- custcols.append(k)
- if datatype == 'text' and CFM[key]['is_multiple']:
- kwargs[k] = concat('#T#'+name,
- format_tag_string(val,'|',
- ignore_max=True))
- elif datatype == 'series':
- kwargs[k] = concat(name, '%s [%s]'%(val,
- fmt_sidx(record[CFM.cc_series_index_column_for(key)])))
- elif datatype == 'datetime':
- kwargs[k] = concat(name,
- format_date(val, CFM[key]['display'].get('date_format','dd MMM yyyy')))
- elif datatype == 'bool':
- if val:
- kwargs[k] = concat(name, __builtin__._('Yes'))
- else:
- kwargs[k] = concat(name, __builtin__._('No'))
- else:
- kwargs[k] = concat(name, val)
+ mi = self.db.get_metadata(record[CFM['id']['rec_index']], index_is_id=True)
+ name, val = mi.format_field(key)
+ if not val:
+ continue
+ datatype = CFM[key]['datatype']
+ if datatype in ['comments']:
+ continue
+ k = str('CF_'+key[1:])
+ name = CFM[key]['name']
+ custcols.append(k)
+ if datatype == 'text' and CFM[key]['is_multiple']:
+ kwargs[k] = concat('#T#'+name, format_tag_string(val,',',
+ ignore_max=True))
+ else:
+ kwargs[k] = concat(name, val)
kwargs['custcols'] = ','.join(custcols)
books.append(E.book(c, **kwargs))
From 83fc5b2cc0452533cdcdc342d8ce21e3ab5501a4 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sat, 18 Sep 2010 11:38:51 +0100
Subject: [PATCH 042/207] Small cleanup of composite field code.
---
src/calibre/ebooks/metadata/book/base.py | 12 ++++++++----
src/calibre/gui2/library/models.py | 2 +-
2 files changed, 9 insertions(+), 5 deletions(-)
diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index 31485dfe1b..ce6e2ee78d 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -46,7 +46,10 @@ composite_formatter = SafeFormat()
compress_spaces = re.compile(r'\s+')
def format_composite(x, mi):
- ans = composite_formatter.vformat(x, [], mi).strip()
+ try:
+ ans = composite_formatter.vformat(x, [], mi).strip()
+ except:
+ ans = x
return compress_spaces.sub(' ', ans)
class Metadata(object):
@@ -86,7 +89,10 @@ class Metadata(object):
except AttributeError:
pass
if field in _data['user_metadata'].iterkeys():
- return _data['user_metadata'][field]['#value#']
+ d = _data['user_metadata'][field]
+ if d['datatype'] != 'composite':
+ return d['#value#']
+ return format_composite(d['display']['composite_template'], self)
raise AttributeError(
'Metadata object has no attribute named: '+ repr(field))
@@ -386,8 +392,6 @@ class Metadata(object):
res = format_date(res, cmeta['display'].get('date_format','dd MMM yyyy'))
elif datatype == 'bool':
res = _('Yes') if res else _('No')
- elif datatype == 'composite':
- res = format_composite(cmeta['display']['composite_template'], self)
return (name, res, orig_res, cmeta)
if key in field_metadata and field_metadata[key]['kind'] == 'field':
diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index 7839b89d7e..2a116f6f3d 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -605,7 +605,7 @@ class BooksModel(QAbstractTableModel): # {{{
def composite_type(r, key=None):
mi = self.get_cached_metadata(r)
- return QVariant(mi.format_field(key)[1])
+ return QVariant(mi.get(key, ''))
self.dc = {
'title' : functools.partial(text_type,
From c59545a96a968488221f494fcee0baccab642a63 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sat, 18 Sep 2010 13:45:01 +0100
Subject: [PATCH 043/207] Change composites to use the cache correctly, so that
searches & sorts used. In the process, remove the metadata cache from
models.py.
Fix some bugs introduced by composite columns:
1) no edit widget in bulk_metadata edit
2) explicitly do not make a delegate in views.py
---
src/calibre/gui2/custom_column_widgets.py | 2 ++
src/calibre/gui2/library/models.py | 28 ++---------------------
src/calibre/gui2/library/views.py | 3 +++
src/calibre/library/caches.py | 20 +++++++++++++++-
src/calibre/library/database2.py | 11 ++++-----
5 files changed, 31 insertions(+), 33 deletions(-)
diff --git a/src/calibre/gui2/custom_column_widgets.py b/src/calibre/gui2/custom_column_widgets.py
index 67ab94d29a..d16233be1a 100644
--- a/src/calibre/gui2/custom_column_widgets.py
+++ b/src/calibre/gui2/custom_column_widgets.py
@@ -348,6 +348,8 @@ def populate_metadata_page(layout, db, book_id, bulk=False, two_column=False, pa
ans = []
column = row = comments_row = 0
for col in cols:
+ if not x[col]['editable']:
+ continue
dt = x[col]['datatype']
if dt == 'comments':
continue
diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index 2a116f6f3d..be1bf9bc2d 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -86,7 +86,6 @@ class BooksModel(QAbstractTableModel): # {{{
self.last_search = '' # The last search performed on this model
self.column_map = []
self.headers = {}
- self.metadata_cache = {}
self.alignment_map = {}
self.buffer_size = buffer
self.cover_cache = None
@@ -115,16 +114,6 @@ class BooksModel(QAbstractTableModel): # {{{
def clear_caches(self):
if self.cover_cache:
self.cover_cache.clear_cache()
- self.metadata_cache = {}
-
- def get_cached_metadata(self, idx):
- if idx not in self.metadata_cache:
- self.metadata_cache[idx] = self.db.get_metadata(idx)
- return self.metadata_cache[idx]
-
- def remove_cached_metadata(self, idx):
- if idx in self.metadata_cache:
- del self.metadata_cache[idx]
def read_config(self):
self.use_roman_numbers = config['use_roman_numerals_for_series_number']
@@ -157,7 +146,6 @@ class BooksModel(QAbstractTableModel): # {{{
elif col in self.custom_columns:
self.headers[col] = self.custom_columns[col]['name']
- self.metadata_cache = {}
self.build_data_convertors()
self.reset()
self.database_changed.emit(db)
@@ -171,13 +159,11 @@ class BooksModel(QAbstractTableModel): # {{{
db.add_listener(refresh_cover)
def refresh_ids(self, ids, current_row=-1):
- self.metadata_cache = {}
rows = self.db.refresh_ids(ids)
if rows:
self.refresh_rows(rows, current_row=current_row)
def refresh_rows(self, rows, current_row=-1):
- self.metadata_cache = {}
for row in rows:
if row == current_row:
self.new_bookdisplay_data.emit(
@@ -207,7 +193,6 @@ class BooksModel(QAbstractTableModel): # {{{
return ret
def count_changed(self, *args):
- self.metadata_cache = {}
self.count_changed_signal.emit(self.db.count())
def row_indices(self, index):
@@ -277,7 +262,6 @@ class BooksModel(QAbstractTableModel): # {{{
self.sorting_done.emit(self.db.index)
def refresh(self, reset=True):
- self.metadata_cache = {}
self.db.refresh(field=None)
self.resort(reset=reset)
@@ -334,7 +318,7 @@ class BooksModel(QAbstractTableModel): # {{{
data[_('Series')] = \
_('Book %s of %s.')%\
(sidx, prepare_string_for_xml(series))
- mi = self.get_cached_metadata(idx)
+ mi = self.db.get_metadata(idx)
for key in mi.user_metadata_keys:
name, val = mi.format_field(key)
if val is not None:
@@ -343,7 +327,6 @@ class BooksModel(QAbstractTableModel): # {{{
def set_cache(self, idx):
l, r = 0, self.count()-1
- self.remove_cached_metadata(idx)
if self.cover_cache is not None:
l = max(l, idx-self.buffer_size)
r = min(r, idx+self.buffer_size)
@@ -603,10 +586,6 @@ class BooksModel(QAbstractTableModel): # {{{
def number_type(r, idx=-1):
return QVariant(self.db.data[r][idx])
- def composite_type(r, key=None):
- mi = self.get_cached_metadata(r)
- return QVariant(mi.get(key, ''))
-
self.dc = {
'title' : functools.partial(text_type,
idx=self.db.field_metadata['title']['rec_index'], mult=False),
@@ -640,7 +619,7 @@ class BooksModel(QAbstractTableModel): # {{{
for col in self.custom_columns:
idx = self.custom_columns[col]['rec_index']
datatype = self.custom_columns[col]['datatype']
- if datatype in ('text', 'comments'):
+ if datatype in ('text', 'comments', 'composite'):
self.dc[col] = functools.partial(text_type, idx=idx,
mult=self.custom_columns[col]['is_multiple'])
elif datatype in ('int', 'float'):
@@ -657,8 +636,6 @@ class BooksModel(QAbstractTableModel): # {{{
elif datatype == 'series':
self.dc[col] = functools.partial(series_type, idx=idx,
siix=self.db.field_metadata.cc_series_index_column_for(col))
- elif datatype == 'composite':
- self.dc[col] = functools.partial(composite_type, key=col)
else:
print 'What type is this?', col, datatype
# build a index column to data converter map, to remove the string lookup in the data loop
@@ -753,7 +730,6 @@ class BooksModel(QAbstractTableModel): # {{{
if role == Qt.EditRole:
row, col = index.row(), index.column()
column = self.column_map[col]
- self.remove_cached_metadata(row)
if self.is_custom_column(column):
if not self.set_custom_column_data(row, column, value):
return False
diff --git a/src/calibre/gui2/library/views.py b/src/calibre/gui2/library/views.py
index d67d286aeb..9951edf21b 100644
--- a/src/calibre/gui2/library/views.py
+++ b/src/calibre/gui2/library/views.py
@@ -391,6 +391,9 @@ class BooksView(QTableView): # {{{
self.setItemDelegateForColumn(cm.index(colhead), self.cc_bool_delegate)
elif cc['datatype'] == 'rating':
self.setItemDelegateForColumn(cm.index(colhead), self.rating_delegate)
+ elif cc['datatype'] == 'composite':
+ pass
+ # no delegate for composite columns, as they are not editable
else:
dattr = colhead+'_delegate'
delegate = colhead if hasattr(self, dattr) else 'text'
diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index 4f795ab733..a013d23cb9 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -121,6 +121,11 @@ class ResultCache(SearchQueryParser):
self.build_date_relop_dict()
self.build_numeric_relop_dict()
+ self.composites = []
+ for key in field_metadata:
+ if field_metadata[key]['datatype'] == 'composite':
+ self.composites.append((key, field_metadata[key]['rec_index']))
+
def __getitem__(self, row):
return self._data[self._map_filtered[row]]
@@ -372,7 +377,7 @@ class ResultCache(SearchQueryParser):
if len(self.field_metadata[x]['search_terms']):
db_col[x] = self.field_metadata[x]['rec_index']
if self.field_metadata[x]['datatype'] not in \
- ['text', 'comments', 'series']:
+ ['composite', 'text', 'comments', 'series']:
exclude_fields.append(db_col[x])
col_datatype[db_col[x]] = self.field_metadata[x]['datatype']
is_multiple_cols[db_col[x]] = self.field_metadata[x]['is_multiple']
@@ -534,6 +539,10 @@ class ResultCache(SearchQueryParser):
self._data[id] = db.conn.get('SELECT * from meta2 WHERE id=?', (id,))[0]
self._data[id].append(db.has_cover(id, index_is_id=True))
self._data[id].append(db.book_on_device_string(id))
+ if len(self.composites) > 0:
+ mi = db.get_metadata(id, index_is_id=True)
+ for k,c in self.composites:
+ self._data[id][c] = mi.format_field(k)[1]
except IndexError:
return None
try:
@@ -550,6 +559,10 @@ class ResultCache(SearchQueryParser):
self._data[id] = db.conn.get('SELECT * from meta2 WHERE id=?', (id,))[0]
self._data[id].append(db.has_cover(id, index_is_id=True))
self._data[id].append(db.book_on_device_string(id))
+ if len(self.composites) > 0:
+ mi = db.get_metadata(id, index_is_id=True)
+ for k,c in self.composites:
+ self._data[id][c] = mi.format_field(k)[1]
self._map[0:0] = ids
self._map_filtered[0:0] = ids
@@ -575,6 +588,11 @@ class ResultCache(SearchQueryParser):
if item is not None:
item.append(db.has_cover(item[0], index_is_id=True))
item.append(db.book_on_device_string(item[0]))
+ if len(self.composites) > 0:
+ mi = db.get_metadata(item[0], index_is_id=True)
+ for k,c in self.composites:
+ item[c] = mi.format_field(k)[1]
+
self._map = [i[0] for i in self._data if i is not None]
if field is not None:
self.sort(field, ascending)
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index d06d217b76..d51a8a62c0 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -323,12 +323,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
self.has_id = self.data.has_id
self.count = self.data.count
- self.refresh_ondevice = functools.partial(self.data.refresh_ondevice, self)
-
- self.refresh()
- self.last_update_check = self.last_modified()
-
-
for prop in ('author_sort', 'authors', 'comment', 'comments', 'isbn',
'publisher', 'rating', 'series', 'series_index', 'tags',
'title', 'timestamp', 'uuid', 'pubdate', 'ondevice'):
@@ -337,6 +331,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
setattr(self, 'title_sort', functools.partial(self.get_property,
loc=self.FIELD_MAP['sort']))
+ self.refresh_ondevice = functools.partial(self.data.refresh_ondevice, self)
+ self.refresh()
+ self.last_update_check = self.last_modified()
+
+
def initialize_database(self):
metadata_sqlite = open(P('metadata_sqlite.sql'), 'rb').read()
self.conn.executescript(metadata_sqlite)
From ed7597ae5f142998c3444f1ad941725fa4d21b0d Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sat, 18 Sep 2010 19:40:44 +0100
Subject: [PATCH 044/207] Playing with search & replace. Added 'global'
template values to the replace expression. Also fixed some problems with
exceptions, and problems with case-insensitive matching in the history boxes.
---
src/calibre/ebooks/metadata/book/base.py | 9 +++
src/calibre/gui2/dialogs/metadata_bulk.py | 68 +++++++++++++++++++----
2 files changed, 66 insertions(+), 11 deletions(-)
diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index ce6e2ee78d..1eae2e5326 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -12,6 +12,7 @@ from calibre.ebooks.metadata.book import SC_COPYABLE_FIELDS
from calibre.ebooks.metadata.book import SC_FIELDS_COPY_NOT_NULL
from calibre.ebooks.metadata.book import STANDARD_METADATA_FIELDS
from calibre.ebooks.metadata.book import TOP_LEVEL_CLASSIFIERS
+from calibre.ebooks.metadata.book import ALL_METADATA_FIELDS
from calibre.library.field_metadata import FieldMetadata
from calibre.utils.date import isoformat, format_date
@@ -131,6 +132,14 @@ class Metadata(object):
def set(self, field, val, extra=None):
self.__setattr__(field, val, extra)
+ @property
+ def all_keys(self):
+ '''
+ All attribute keys known by this instance, even if their value is None
+ '''
+ _data = object.__getattribute__(self, '_data')
+ return frozenset(ALL_METADATA_FIELDS.union(_data['user_metadata'].iterkeys()))
+
@property
def user_metadata_keys(self):
'The set of user metadata names this object knows about'
diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py
index b7d1d0c54b..1fb889757f 100644
--- a/src/calibre/gui2/dialogs/metadata_bulk.py
+++ b/src/calibre/gui2/dialogs/metadata_bulk.py
@@ -4,15 +4,15 @@ __copyright__ = '2008, Kovid Goyal '
'''Dialog to edit metadata in bulk'''
from threading import Thread
-import re
+import re, string
-from PyQt4.Qt import QDialog, QGridLayout
+from PyQt4.Qt import Qt, QDialog, QGridLayout
from PyQt4 import QtGui
from calibre.gui2.dialogs.metadata_bulk_ui import Ui_MetadataBulkDialog
from calibre.gui2.dialogs.tag_editor import TagEditor
from calibre.ebooks.metadata import string_to_authors, \
- authors_to_string
+ authors_to_string, MetaInformation
from calibre.gui2.custom_column_widgets import populate_metadata_page
from calibre.gui2.dialogs.progress import BlockingBusy
from calibre.gui2 import error_dialog, Dispatcher
@@ -99,6 +99,26 @@ class Worker(Thread):
self.callback()
+class SafeFormat(string.Formatter):
+ '''
+ Provides a format function that substitutes '' for any missing value
+ '''
+ def get_value(self, key, args, vals):
+ v = vals.get(key, None)
+ if v is None:
+ return ''
+ if isinstance(v, (tuple, list)):
+ v = ','.join(v)
+ return v
+
+composite_formatter = SafeFormat()
+
+def format_composite(x, mi):
+ try:
+ ans = composite_formatter.vformat(x, [], mi).strip()
+ except:
+ ans = x
+ return ans
class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
@@ -163,7 +183,7 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
self.s_r_number_of_books = min(7, len(self.ids))
for i in range(1,self.s_r_number_of_books+1):
w = QtGui.QLabel(self.tabWidgetPage3)
- w.setText(_('Book %d:'%i))
+ w.setText(_('Book %d:')%i)
self.gridLayout1.addWidget(w, i+offset, 0, 1, 1)
w = QtGui.QLineEdit(self.tabWidgetPage3)
w.setReadOnly(True)
@@ -205,6 +225,10 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
self.test_text.editTextChanged[str].connect(self.s_r_paint_results)
self.central_widget.setCurrentIndex(0)
+ self.search_for.completer().setCaseSensitivity(Qt.CaseSensitive)
+ self.replace_with.completer().setCaseSensitivity(Qt.CaseSensitive)
+
+
def s_r_field_changed(self, txt):
txt = unicode(txt)
for i in range(0, self.s_r_number_of_books):
@@ -220,6 +244,8 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
if val:
val.sort(cmp=lambda x,y: cmp(x.lower(), y.lower()))
val = val[0]
+ if txt == 'authors':
+ val = val.replace('|', ',')
else:
val = ''
else:
@@ -239,37 +265,55 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
for i in range(0,self.s_r_number_of_books):
getattr(self, 'book_%d_result'%(i+1)).setText('')
+ field_match_re = re.compile(r'(^|[^\\])(\\g<)([^>]+)(>)')
+
def s_r_func(self, match):
- rf = self.s_r_functions[unicode(self.replace_func.currentText())]
- rv = unicode(self.replace_with.text())
- val = match.expand(rv)
- return rf(val)
+ rfunc = self.s_r_functions[unicode(self.replace_func.currentText())]
+ rtext = unicode(self.replace_with.text())
+ mi_data = self.mi.get_all_non_none_attributes()
+
+ def fm_func(m):
+ try:
+ if m.group(3) not in self.mi.all_keys: return m.group(0)
+ else: return '%s{%s}'%(m.group(1), m.group(3))
+ except:
+ import traceback
+ traceback.print_exc()
+ return m.group(0)
+
+ rtext = re.sub(self.field_match_re, fm_func, rtext)
+ rtext = match.expand(rtext)
+ rtext = format_composite(rtext, mi_data)
+ return rfunc(rtext)
def s_r_paint_results(self, txt):
self.s_r_error = None
self.s_r_set_colors()
try:
self.s_r_obj = re.compile(unicode(self.search_for.text()))
- except re.error as e:
+ except Exception as e:
self.s_r_obj = None
self.s_r_error = e
self.s_r_set_colors()
return
try:
+ self.mi = MetaInformation(None, None)
self.test_result.setText(self.s_r_obj.sub(self.s_r_func,
unicode(self.test_text.text())))
- except re.error as e:
+ except Exception as e:
self.s_r_error = e
self.s_r_set_colors()
return
for i in range(0,self.s_r_number_of_books):
+ id = self.ids[i]
+ self.mi = self.db.get_metadata(id, index_is_id=True)
wt = getattr(self, 'book_%d_text'%(i+1))
wr = getattr(self, 'book_%d_result'%(i+1))
try:
wr.setText(self.s_r_obj.sub(self.s_r_func, unicode(wt.text())))
- except re.error as e:
+ except Exception as e:
self.s_r_error = e
self.s_r_set_colors()
break
@@ -303,6 +347,8 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
# The standard tags and authors values want to be lists.
# All custom columns are to be strings
val = fm['is_multiple'].join(val)
+ elif field == 'authors':
+ val = [v.replace('|', ',') for v in val]
else:
val = apply_pattern(val)
From 3f763407a02f5c00599bdbe43f053a821fb0a3e3 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Sat, 18 Sep 2010 20:37:59 -0600
Subject: [PATCH 045/207] Refactor to use new field formatting infrastructure
of Metadata class
---
src/calibre/devices/usbms/books.py | 7 ++--
src/calibre/ebooks/metadata/book/__init__.py | 7 ++--
src/calibre/ebooks/metadata/book/base.py | 12 +++---
src/calibre/library/database2.py | 1 +
src/calibre/library/server/mobile.py | 36 +++++++-----------
src/calibre/library/server/opds.py | 25 +++++-------
src/calibre/library/server/xml.py | 40 ++++++++------------
7 files changed, 51 insertions(+), 77 deletions(-)
diff --git a/src/calibre/devices/usbms/books.py b/src/calibre/devices/usbms/books.py
index eab625f7be..13fcb90b49 100644
--- a/src/calibre/devices/usbms/books.py
+++ b/src/calibre/devices/usbms/books.py
@@ -137,7 +137,6 @@ class CollectionsBookList(BookList):
# For existing books, modify the collections only if the user
# specified 'on_connect'
attrs = collection_attributes
- meta_vals = book.get_all_non_none_attributes()
for attr in attrs:
attr = attr.strip()
ign, val, orig_val, fm = book.format_field_extended(attr)
@@ -166,7 +165,7 @@ class CollectionsBookList(BookList):
continue
if attr == 'series' or \
('series' in collection_attributes and
- meta_vals.get('series', None) == category):
+ book.get('series', None) == category):
is_series = True
cat_name = self.compute_category_name(attr, category, fm)
if cat_name not in collections:
@@ -177,10 +176,10 @@ class CollectionsBookList(BookList):
collections_lpaths[cat_name].add(lpath)
if is_series:
collections[cat_name].append(
- (book, meta_vals.get(attr+'_index', sys.maxint)))
+ (book, book.get(attr+'_index', sys.maxint)))
else:
collections[cat_name].append(
- (book, meta_vals.get('title_sort', 'zzzz')))
+ (book, book.get('title_sort', 'zzzz')))
# Sort collections
result = {}
for category, books in collections.items():
diff --git a/src/calibre/ebooks/metadata/book/__init__.py b/src/calibre/ebooks/metadata/book/__init__.py
index e087f8072d..e6dff9110b 100644
--- a/src/calibre/ebooks/metadata/book/__init__.py
+++ b/src/calibre/ebooks/metadata/book/__init__.py
@@ -81,9 +81,8 @@ DEVICE_METADATA_FIELDS = frozenset([
CALIBRE_METADATA_FIELDS = frozenset([
'application_id', # An application id, currently set to the db_id.
- # the calibre primary key of the item.
'db_id', # the calibre primary key of the item.
- # TODO: NEWMETA: May want to remove once Sony's no longer use it
+ 'formats', # list of formats (extensions) for this book
]
)
@@ -124,5 +123,5 @@ SERIALIZABLE_FIELDS = SOCIAL_METADATA_FIELDS.union(
PUBLICATION_METADATA_FIELDS).union(
CALIBRE_METADATA_FIELDS).union(
DEVICE_METADATA_FIELDS) - \
- frozenset(['device_collections'])
- # device_collections is rebuilt when needed
+ frozenset(['device_collections', 'formats'])
+ # these are rebuilt when needed
diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index b252f518da..8868709db2 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -343,26 +343,26 @@ class Metadata(object):
def format_rating(self):
return unicode(self.rating)
- def format_field(self, key):
- name, val, ign, ign = self.format_field_extended(key)
+ def format_field(self, key, series_with_index=True):
+ name, val, ign, ign = self.format_field_extended(key, series_with_index)
return (name, val)
- def format_field_extended(self, key):
+ def format_field_extended(self, key, series_with_index=True):
from calibre.ebooks.metadata import authors_to_string
'''
returns the tuple (field_name, formatted_value)
'''
if key in self.user_metadata_keys:
res = self.get(key, None)
+ cmeta = self.get_user_metadata(key, make_copy=False)
if res is None or res == '':
return (None, None, None, None)
orig_res = res
- cmeta = self.get_user_metadata(key, make_copy=False)
name = unicode(cmeta['name'])
datatype = cmeta['datatype']
if datatype == 'text' and cmeta['is_multiple']:
res = u', '.join(res)
- elif datatype == 'series':
+ elif datatype == 'series' and series_with_index:
res = res + \
' [%s]'%self.format_series_index(val=self.get_extra(key))
elif datatype == 'datetime':
@@ -383,7 +383,7 @@ class Metadata(object):
res = authors_to_string(res)
elif datatype == 'text' and fmeta['is_multiple']:
res = u', '.join(res)
- elif datatype == 'series':
+ elif datatype == 'series' and series_with_index:
res = res + ' [%s]'%self.format_series_index()
elif datatype == 'datetime':
res = format_date(res, fmeta['display'].get('date_format','dd MMM yyyy'))
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index 9e9e75a26e..d06d217b76 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -538,6 +538,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
mi.pubdate = self.pubdate(idx, index_is_id=index_is_id)
mi.uuid = self.uuid(idx, index_is_id=index_is_id)
mi.title_sort = self.title_sort(idx, index_is_id=index_is_id)
+ mi.formats = self.formats(idx, index_is_id=index_is_id).split(',')
tags = self.tags(idx, index_is_id=index_is_id)
if tags:
mi.tags = [i.strip() for i in tags.split(',')]
diff --git a/src/calibre/library/server/mobile.py b/src/calibre/library/server/mobile.py
index ab5b39eed8..8e7c75b0ac 100644
--- a/src/calibre/library/server/mobile.py
+++ b/src/calibre/library/server/mobile.py
@@ -228,29 +228,19 @@ class MobileServer(object):
for key in CKEYS:
def concat(name, val):
return '%s:#:%s'%(name, unicode(val))
- val = record[CFM[key]['rec_index']]
- if val:
- datatype = CFM[key]['datatype']
- if datatype in ['comments']:
- continue
- name = CFM[key]['name']
- if datatype == 'text' and CFM[key]['is_multiple']:
- book[key] = concat(name,
- format_tag_string(val, '|',
- no_tag_count=True))
- elif datatype == 'series':
- book[key] = concat(name, '%s [%s]'%(val,
- fmt_sidx(record[CFM.cc_series_index_column_for(key)])))
- elif datatype == 'datetime':
- book[key] = concat(name,
- format_date(val, CFM[key]['display'].get('date_format','dd MMM yyyy')))
- elif datatype == 'bool':
- if val:
- book[key] = concat(name, __builtin__._('Yes'))
- else:
- book[key] = concat(name, __builtin__._('No'))
- else:
- book[key] = concat(name, val)
+ mi = self.db.get_metadata(record[CFM['id']['rec_index']], index_is_id=True)
+ name, val = mi.format_field(key)
+ if val is None:
+ continue
+ datatype = CFM[key]['datatype']
+ if datatype in ['comments']:
+ continue
+ if datatype == 'text' and CFM[key]['is_multiple']:
+ book[key] = concat(name,
+ format_tag_string(val, ',',
+ no_tag_count=True))
+ else:
+ book[key] = concat(name, val)
updated = self.db.last_modified()
diff --git a/src/calibre/library/server/opds.py b/src/calibre/library/server/opds.py
index e495598a2f..d495f58fa1 100644
--- a/src/calibre/library/server/opds.py
+++ b/src/calibre/library/server/opds.py
@@ -20,7 +20,6 @@ from calibre.library.comments import comments_to_html
from calibre.library.server.utils import format_tag_string
from calibre import guess_type
from calibre.utils.ordered_dict import OrderedDict
-from calibre.utils.date import format_date
BASE_HREFS = {
0 : '/stanza',
@@ -132,7 +131,8 @@ def CATALOG_GROUP_ENTRY(item, category, base_href, version, updated):
link
)
-def ACQUISITION_ENTRY(item, version, FM, updated, CFM, CKEYS):
+def ACQUISITION_ENTRY(item, version, db, updated, CFM, CKEYS):
+ FM = db.FIELD_MAP
title = item[FM['title']]
if not title:
title = _('Unknown')
@@ -157,22 +157,16 @@ def ACQUISITION_ENTRY(item, version, FM, updated, CFM, CKEYS):
(series,
fmt_sidx(float(item[FM['series_index']]))))
for key in CKEYS:
- val = item[CFM[key]['rec_index']]
+ mi = db.get_metadata(item[CFM['id']['rec_index']], index_is_id=True)
+ name, val = mi.format_field(key)
if val is not None:
- name = CFM[key]['name']
datatype = CFM[key]['datatype']
if datatype == 'text' and CFM[key]['is_multiple']:
- extra.append('%s: %s '%(name, format_tag_string(val, '|',
+ extra.append('%s: %s '%(name, format_tag_string(val, ',',
ignore_max=True,
no_tag_count=True)))
- elif datatype == 'series':
- extra.append('%s: %s [%s] '%(name, val,
- fmt_sidx(item[CFM.cc_series_index_column_for(key)])))
- elif datatype == 'datetime':
- extra.append('%s: %s '%(name,
- format_date(val, CFM[key]['display'].get('date_format','dd MMM yyyy'))))
else:
- extra.append('%s: %s ' % (CFM[key]['name'], val))
+ extra.append('%s: %s '%(name, val))
comments = item[FM['comments']]
if comments:
comments = comments_to_html(comments)
@@ -280,13 +274,14 @@ class NavFeed(Feed):
class AcquisitionFeed(NavFeed):
def __init__(self, updated, id_, items, offsets, page_url, up_url, version,
- FM, CFM):
+ db):
NavFeed.__init__(self, id_, updated, version, offsets, page_url, up_url)
+ CFM = db.field_metadata
CKEYS = [key for key in sorted(CFM.get_custom_fields(),
cmp=lambda x,y: cmp(CFM[x]['name'].lower(),
CFM[y]['name'].lower()))]
for item in items:
- self.root.append(ACQUISITION_ENTRY(item, version, FM, updated,
+ self.root.append(ACQUISITION_ENTRY(item, version, db, updated,
CFM, CKEYS))
class CategoryFeed(NavFeed):
@@ -384,7 +379,7 @@ class OPDSServer(object):
cherrypy.response.headers['Last-Modified'] = self.last_modified(updated)
cherrypy.response.headers['Content-Type'] = 'application/atom+xml;profile=opds-catalog'
return str(AcquisitionFeed(updated, id_, items, offsets,
- page_url, up_url, version, self.db.FIELD_MAP, self.db.field_metadata))
+ page_url, up_url, version, self.db))
def opds_search(self, query=None, version=0, offset=0):
try:
diff --git a/src/calibre/library/server/xml.py b/src/calibre/library/server/xml.py
index 8715dda7d0..7f5bc31e70 100644
--- a/src/calibre/library/server/xml.py
+++ b/src/calibre/library/server/xml.py
@@ -102,31 +102,21 @@ class XMLServer(object):
for key in CKEYS:
def concat(name, val):
return '%s:#:%s'%(name, unicode(val))
- val = record[CFM[key]['rec_index']]
- if val:
- datatype = CFM[key]['datatype']
- if datatype in ['comments']:
- continue
- k = str('CF_'+key[1:])
- name = CFM[key]['name']
- custcols.append(k)
- if datatype == 'text' and CFM[key]['is_multiple']:
- kwargs[k] = concat('#T#'+name,
- format_tag_string(val,'|',
- ignore_max=True))
- elif datatype == 'series':
- kwargs[k] = concat(name, '%s [%s]'%(val,
- fmt_sidx(record[CFM.cc_series_index_column_for(key)])))
- elif datatype == 'datetime':
- kwargs[k] = concat(name,
- format_date(val, CFM[key]['display'].get('date_format','dd MMM yyyy')))
- elif datatype == 'bool':
- if val:
- kwargs[k] = concat(name, __builtin__._('Yes'))
- else:
- kwargs[k] = concat(name, __builtin__._('No'))
- else:
- kwargs[k] = concat(name, val)
+ mi = self.db.get_metadata(record[CFM['id']['rec_index']], index_is_id=True)
+ name, val = mi.format_field(key)
+ if not val:
+ continue
+ datatype = CFM[key]['datatype']
+ if datatype in ['comments']:
+ continue
+ k = str('CF_'+key[1:])
+ name = CFM[key]['name']
+ custcols.append(k)
+ if datatype == 'text' and CFM[key]['is_multiple']:
+ kwargs[k] = concat('#T#'+name, format_tag_string(val,',',
+ ignore_max=True))
+ else:
+ kwargs[k] = concat(name, val)
kwargs['custcols'] = ','.join(custcols)
books.append(E.book(c, **kwargs))
From 7eaf417bb10e9d87038b47941c524ea9aa121ad2 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sun, 19 Sep 2010 07:47:03 +0100
Subject: [PATCH 046/207] Fix content server tags display problem
---
resources/content_server/gui.js | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/resources/content_server/gui.js b/resources/content_server/gui.js
index afc21137e1..bd0743a854 100644
--- a/resources/content_server/gui.js
+++ b/resources/content_server/gui.js
@@ -84,7 +84,10 @@ function render_book(book) {
}
title += ''
title += ''
- if (tags) title += 'Tags=[{0}] '.format(tags);
+ if (tags) {
+ t = tags.split(':&:', 2);
+ title += 'Tags=[{0}] '.format(t[1]);
+ }
custcols = book.attr("custcols").split(',')
for ( i = 0; i < custcols.length; i++) {
if (custcols[i].length > 0) {
From db446dc4ee28f3280f4b1a139e8243c7dbf817d7 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sun, 19 Sep 2010 09:24:57 +0100
Subject: [PATCH 047/207] Add an attribute to the main item record for caching
a Metadata instance. LibraryDatabase2.get_metadata it if None, and returns it
if not None.
---
src/calibre/ebooks/metadata/book/base.py | 6 ++++++
src/calibre/library/caches.py | 3 +++
src/calibre/library/database2.py | 25 ++++++++++++++++++------
src/calibre/library/field_metadata.py | 9 +++++++++
4 files changed, 37 insertions(+), 6 deletions(-)
diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index cd2f2a3165..7b8eb07908 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -114,6 +114,12 @@ class Metadata(object):
# Don't abuse this privilege
self.__dict__[field] = val
+ def deepcopy(self):
+ m = Metadata(None)
+ m.__dict__ = copy.deepcopy(self.__dict__)
+ object.__setattr__(m, '_data', copy.deepcopy(object.__getattribute__(self, '_data')))
+ return m
+
def get(self, field, default=None):
if default is not None:
try:
diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index ead7a2b46d..b3f901ecd3 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -539,6 +539,7 @@ class ResultCache(SearchQueryParser):
self._data[id] = db.conn.get('SELECT * from meta2 WHERE id=?', (id,))[0]
self._data[id].append(db.has_cover(id, index_is_id=True))
self._data[id].append(db.book_on_device_string(id))
+ self._data[id].append(None)
if len(self.composites) > 0:
mi = db.get_metadata(id, index_is_id=True)
for k,c in self.composites:
@@ -559,6 +560,7 @@ class ResultCache(SearchQueryParser):
self._data[id] = db.conn.get('SELECT * from meta2 WHERE id=?', (id,))[0]
self._data[id].append(db.has_cover(id, index_is_id=True))
self._data[id].append(db.book_on_device_string(id))
+ self._data[id].append(None)
if len(self.composites) > 0:
mi = db.get_metadata(id, index_is_id=True)
for k,c in self.composites:
@@ -588,6 +590,7 @@ class ResultCache(SearchQueryParser):
if item is not None:
item.append(db.has_cover(item[0], index_is_id=True))
item.append(db.book_on_device_string(item[0]))
+ item.append(None)
if len(self.composites) > 0:
mi = db.get_metadata(item[0], index_is_id=True)
for k,c in self.composites:
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index d51a8a62c0..489504dbb5 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -20,8 +20,8 @@ from calibre.library.caches import ResultCache
from calibre.library.custom_columns import CustomColumns
from calibre.library.sqlite import connect, IntegrityError, DBThread
from calibre.library.prefs import DBPrefs
-from calibre.ebooks.metadata import string_to_authors, authors_to_string, \
- MetaInformation
+from calibre.ebooks.metadata import string_to_authors, authors_to_string
+from calibre.ebooks.metadata.book.base import Metadata
from calibre.ebooks.metadata.meta import get_metadata, metadata_from_formats
from calibre.constants import preferred_encoding, iswindows, isosx, filesystem_encoding
from calibre.ptempfile import PersistentTemporaryFile
@@ -282,6 +282,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
self.field_metadata.set_field_record_index('cover', base+1, prefer_custom=False)
self.FIELD_MAP['ondevice'] = base+2
self.field_metadata.set_field_record_index('ondevice', base+2, prefer_custom=False)
+ self.FIELD_MAP['all_metadata'] = base+3
+ self.field_metadata.set_field_record_index('all_metadata', base+3, prefer_custom=False)
script = '''
DROP VIEW IF EXISTS meta2;
@@ -520,15 +522,26 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
def get_metadata(self, idx, index_is_id=False, get_cover=False):
'''
- Convenience method to return metadata as a L{MetaInformation} object.
+ Convenience method to return metadata as a L{Metadata} object.
'''
+ mi = self.data.get(idx, self.FIELD_MAP['all_metadata'],
+ row_is_id = index_is_id)
+ if mi is not None:
+ return mi
+
+ mi = self.field_metadata.get_empty_metadata_instance()
+ self.data.set(idx, self.FIELD_MAP['all_metadata'], mi,
+ row_is_id = index_is_id)
+
aut_list = self.authors_with_sort_strings(idx, index_is_id=index_is_id)
aum = []
aus = {}
for (author, author_sort) in aut_list:
aum.append(author)
aus[author] = author_sort
- mi = MetaInformation(self.title(idx, index_is_id=index_is_id), aum)
+ mi = self.field_metadata.get_empty_metadata_instance()
+ mi.title = self.title(idx, index_is_id=index_is_id)
+ mi.authors = aum
mi.author_sort = self.author_sort(idx, index_is_id=index_is_id)
mi.author_sort_map = aus
mi.comments = self.comments(idx, index_is_id=index_is_id)
@@ -1056,7 +1069,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
def set_metadata(self, id, mi, ignore_errors=False):
'''
- Set metadata for the book `id` from the `MetaInformation` object `mi`
+ Set metadata for the book `id` from the `Metadata` object `mi`
'''
def doit(func, *args, **kwargs):
try:
@@ -1710,7 +1723,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
try:
mi = get_metadata(stream, format)
except:
- mi = MetaInformation(title, ['calibre'])
+ mi = Metadata(title, ['calibre'])
stream.seek(0)
mi.title, mi.authors = title, ['calibre']
mi.tags = [_('Catalog')]
diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py
index dcdfcfd9d6..258c739e1c 100644
--- a/src/calibre/library/field_metadata.py
+++ b/src/calibre/library/field_metadata.py
@@ -209,6 +209,15 @@ class FieldMetadata(dict):
'search_terms':[],
'is_custom':False,
'is_category':False}),
+ ('all_metadata',{'table':None,
+ 'column':None,
+ 'datatype':None,
+ 'is_multiple':None,
+ 'kind':'field',
+ 'name':None,
+ 'search_terms':[],
+ 'is_custom':False,
+ 'is_category':False}),
('ondevice', {'table':None,
'column':None,
'datatype':'text',
From 354db58545add6acf9ab35c4305d233a6071f7bd Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sun, 19 Sep 2010 10:27:18 +0100
Subject: [PATCH 048/207] 1) fix the 'new' Metadata caching to really work. :)
2) remove a useless comment in FieldMetadata
---
src/calibre/library/caches.py | 1 +
src/calibre/library/database2.py | 3 +--
src/calibre/library/field_metadata.py | 1 -
3 files changed, 2 insertions(+), 3 deletions(-)
diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index b3f901ecd3..073f98583c 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -509,6 +509,7 @@ class ResultCache(SearchQueryParser):
def set(self, row, col, val, row_is_id=False):
id = row if row_is_id else self._map_filtered[row]
+ self._data[id][self.FIELD_MAP['all_metadata']] = None
self._data[id][col] = val
def get(self, row, col, row_is_id=False):
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index 489504dbb5..3158cbf94f 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -529,7 +529,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
if mi is not None:
return mi
- mi = self.field_metadata.get_empty_metadata_instance()
+ mi = Metadata(None)
self.data.set(idx, self.FIELD_MAP['all_metadata'], mi,
row_is_id = index_is_id)
@@ -539,7 +539,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
for (author, author_sort) in aut_list:
aum.append(author)
aus[author] = author_sort
- mi = self.field_metadata.get_empty_metadata_instance()
mi.title = self.title(idx, index_is_id=index_is_id)
mi.authors = aum
mi.author_sort = self.author_sort(idx, index_is_id=index_is_id)
diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py
index 258c739e1c..971d91b248 100644
--- a/src/calibre/library/field_metadata.py
+++ b/src/calibre/library/field_metadata.py
@@ -304,7 +304,6 @@ class FieldMetadata(dict):
# search labels that are not db columns
search_items = [ 'all',
-# 'date',
'search',
]
From ef3fd4df536811ca7b91be06ab10595ae1dc6a4c Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Sun, 19 Sep 2010 10:39:45 -0600
Subject: [PATCH 049/207] ...
---
resources/content_server/gui.js | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/resources/content_server/gui.js b/resources/content_server/gui.js
index afc21137e1..bd0743a854 100644
--- a/resources/content_server/gui.js
+++ b/resources/content_server/gui.js
@@ -84,7 +84,10 @@ function render_book(book) {
}
title += ''
title += ''
- if (tags) title += 'Tags=[{0}] '.format(tags);
+ if (tags) {
+ t = tags.split(':&:', 2);
+ title += 'Tags=[{0}] '.format(t[1]);
+ }
custcols = book.attr("custcols").split(',')
for ( i = 0; i < custcols.length; i++) {
if (custcols[i].length > 0) {
From e22efca956253cd52294dde0cb6e59dfb87ac945 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Sun, 19 Sep 2010 10:42:23 -0600
Subject: [PATCH 050/207] ...
---
src/calibre/library/server/mobile.py | 2 +-
src/calibre/library/server/xml.py | 1 -
2 files changed, 1 insertion(+), 2 deletions(-)
diff --git a/src/calibre/library/server/mobile.py b/src/calibre/library/server/mobile.py
index 8e7c75b0ac..c0a3c122cd 100644
--- a/src/calibre/library/server/mobile.py
+++ b/src/calibre/library/server/mobile.py
@@ -17,7 +17,7 @@ from calibre.library.server.utils import strftime, format_tag_string
from calibre.ebooks.metadata import fmt_sidx
from calibre.constants import __appname__
from calibre import human_readable
-from calibre.utils.date import utcfromtimestamp, format_date
+from calibre.utils.date import utcfromtimestamp
def CLASS(*args, **kwargs): # class is a reserved word in Python
kwargs['class'] = ' '.join(args)
diff --git a/src/calibre/library/server/xml.py b/src/calibre/library/server/xml.py
index 7f5bc31e70..45ffdc2737 100644
--- a/src/calibre/library/server/xml.py
+++ b/src/calibre/library/server/xml.py
@@ -15,7 +15,6 @@ from calibre.library.server.utils import strftime, format_tag_string
from calibre.ebooks.metadata import fmt_sidx
from calibre.constants import preferred_encoding
from calibre import isbytestring
-from calibre.utils.date import format_date
E = ElementMaker()
From 16266c6e700513fd60abafc88de205951cbea90e Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sun, 19 Sep 2010 20:11:39 +0100
Subject: [PATCH 051/207] Change field_keys to sortable_keys
---
src/calibre/library/caches.py | 2 +-
src/calibre/library/field_metadata.py | 6 ++++--
src/calibre/library/server/content.py | 2 +-
3 files changed, 6 insertions(+), 4 deletions(-)
diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index 073f98583c..770a362a1d 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -621,7 +621,7 @@ class ResultCache(SearchQueryParser):
def multisort(self, fields=[], subsort=False):
fields = [(self.sanitize_sort_field_name(x), bool(y)) for x, y in fields]
- keys = self.field_metadata.field_keys()
+ keys = self.field_metadata.sortable_keys()
fields = [x for x in fields if x[0] in keys]
if subsort and 'sort' not in [x[0] for x in fields]:
fields += [('sort', True)]
diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py
index 971d91b248..e4a4f5270d 100644
--- a/src/calibre/library/field_metadata.py
+++ b/src/calibre/library/field_metadata.py
@@ -348,8 +348,10 @@ class FieldMetadata(dict):
def keys(self):
return self._tb_cats.keys()
- def field_keys(self):
- return [k for k in self._tb_cats.keys() if self._tb_cats[k]['kind']=='field']
+ def sortable_keys(self):
+ return [k for k in self._tb_cats.keys()
+ if self._tb_cats[k]['kind']=='field' and
+ self._tb_cats[k]['datatype'] is not None]
def iterkeys(self):
for key in self._tb_cats:
diff --git a/src/calibre/library/server/content.py b/src/calibre/library/server/content.py
index 95794a8c1d..acfc1f9ab1 100644
--- a/src/calibre/library/server/content.py
+++ b/src/calibre/library/server/content.py
@@ -56,7 +56,7 @@ class ContentServer(object):
def sort(self, items, field, order):
field = self.db.data.sanitize_sort_field_name(field)
- if field not in self.db.field_metadata.field_keys():
+ if field not in self.db.field_metadata.sortable_keys():
raise cherrypy.HTTPError(400, '%s is not a valid sort field'%field)
keyg = CSSortKeyGenerator([(field, order)], self.db.field_metadata)
items.sort(key=keyg, reverse=not order)
From 3a45aa84b07b19a91e66684bdaa178d8fc5cd71a Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sun, 19 Sep 2010 20:42:22 +0100
Subject: [PATCH 052/207] Fix exception instead of error when composite column
template is blank
---
.../gui2/preferences/create_custom_column.py | 17 +++++++++++------
1 file changed, 11 insertions(+), 6 deletions(-)
diff --git a/src/calibre/gui2/preferences/create_custom_column.py b/src/calibre/gui2/preferences/create_custom_column.py
index 4b21301ccd..ab1e736223 100644
--- a/src/calibre/gui2/preferences/create_custom_column.py
+++ b/src/calibre/gui2/preferences/create_custom_column.py
@@ -82,7 +82,8 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
ct = c['datatype'] if not c['is_multiple'] else '*text'
self.orig_column_number = c['colnum']
self.orig_column_name = col
- column_numbers = dict(map(lambda x:(self.column_types[x]['datatype'], x), self.column_types))
+ column_numbers = dict(map(lambda x:(self.column_types[x]['datatype'], x),
+ self.column_types))
self.column_type_box.setCurrentIndex(column_numbers[ct])
self.column_type_box.setEnabled(False)
if ct == 'datetime':
@@ -109,9 +110,11 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
if not col:
return self.simple_error('', _('No lookup name was provided'))
if re.match('^\w*$', col) is None or not col[0].isalpha() or col.lower() != col:
- return self.simple_error('', _('The lookup name must contain only lower case letters, digits and underscores, and start with a letter'))
+ return self.simple_error('', _('The lookup name must contain only '
+ 'lower case letters, digits and underscores, and start with a letter'))
if col.endswith('_index'):
- return self.simple_error('', _('Lookup names cannot end with _index, because these names are reserved for the index of a series column.'))
+ return self.simple_error('', _('Lookup names cannot end with _index, '
+ 'because these names are reserved for the index of a series column.'))
col_heading = unicode(self.column_heading_box.text())
col_type = self.column_types[self.column_type_box.currentIndex()]['datatype']
if col_type == '*text':
@@ -123,7 +126,8 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
return self.simple_error('', _('No column heading was provided'))
bad_col = False
if col in self.parent.custcols:
- if not self.editing_col or self.parent.custcols[col]['colnum'] != self.orig_column_number:
+ if not self.editing_col or \
+ self.parent.custcols[col]['colnum'] != self.orig_column_number:
bad_col = True
if bad_col:
return self.simple_error('', _('The lookup name %s is already used')%col)
@@ -131,7 +135,8 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
bad_head = False
for t in self.parent.custcols:
if self.parent.custcols[t]['name'] == col_heading:
- if not self.editing_col or self.parent.custcols[t]['colnum'] != self.orig_column_number:
+ if not self.editing_col or \
+ self.parent.custcols[t]['colnum'] != self.orig_column_number:
bad_head = True
for t in self.standard_colheads:
if self.standard_colheads[t] == col_heading:
@@ -148,7 +153,7 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
if col_type == 'composite':
if not self.composite_box.text():
- return self.simple_error('', _('You must enter a template for composite fields')%col_heading)
+ return self.simple_error('', _('You must enter a template for composite fields'))
display_dict = {'composite_template':unicode(self.composite_box.text())}
is_editable = False
else:
From ce865b1eee32aacded18b57bff861c8ea4d4e267 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Sun, 19 Sep 2010 13:54:03 -0600
Subject: [PATCH 053/207] Beta version numbering
---
src/calibre/constants.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/calibre/constants.py b/src/calibre/constants.py
index 4f1dfc25c2..334406e01b 100644
--- a/src/calibre/constants.py
+++ b/src/calibre/constants.py
@@ -2,7 +2,7 @@ __license__ = 'GPL v3'
__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
__docformat__ = 'restructuredtext en'
__appname__ = 'calibre'
-__version__ = '0.7.19'
+__version__ = '0.7.900'
__author__ = "Kovid Goyal "
import re
From 9f42077e11027c13cbecc1a3c1260c38b22197ff Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Sun, 19 Sep 2010 14:46:48 -0600
Subject: [PATCH 054/207] Document the calibre template language
---
setup/installer/__init__.py | 2 +-
src/calibre/manual/faq.rst | 2 +-
src/calibre/manual/gui.rst | 7 ++++
src/calibre/manual/index.rst | 6 +++
src/calibre/manual/template_lang.rst | 61 ++++++++++++++++++++++++++++
5 files changed, 76 insertions(+), 2 deletions(-)
create mode 100644 src/calibre/manual/template_lang.rst
diff --git a/setup/installer/__init__.py b/setup/installer/__init__.py
index 959d8d14e1..39ca671f09 100644
--- a/setup/installer/__init__.py
+++ b/setup/installer/__init__.py
@@ -123,7 +123,7 @@ class VMInstaller(Command):
subprocess.check_call(['scp',
self.VM_NAME+':build/calibre/'+installer, 'dist'])
if not os.path.exists(installer):
- self.warn('Failed to download installer')
+ self.warn('Failed to download installer: '+installer)
raise SystemExit(1)
def clean(self):
diff --git a/src/calibre/manual/faq.rst b/src/calibre/manual/faq.rst
index 02881881c0..10c523f030 100644
--- a/src/calibre/manual/faq.rst
+++ b/src/calibre/manual/faq.rst
@@ -280,7 +280,7 @@ Why doesn't |app| have a column for foo?
|app| is designed to have columns for the most frequently and widely used fields. In addition, you can add any columns you like. Columns can be added via :guilabel:`Preferences->Interface->Add your own columns`.
Watch the tutorial `UI Power tips `_ to learn how to create your own columns.
-You can also create "virtual columns" that contain combinations of the metadata from other columns. In the add column dialog choose the option "Column from other columns" and in the template enter the other column names. For example to create a virtual column containing formats or ISBN, enter ``{formats}`` for formats or ``{isbn}`` for ISBN.
+You can also create "virtual columns" that contain combinations of the metadata from other columns. In the add column dialog choose the option "Column from other columns" and in the template enter the other column names. For example to create a virtual column containing formats or ISBN, enter ``{formats}`` for formats or ``{isbn}`` for ISBN. For more details, see :ref:`templatelangcalibre`.
Can I have a column showing the formats or the ISBN?
diff --git a/src/calibre/manual/gui.rst b/src/calibre/manual/gui.rst
index aa49c51b76..377c409bd0 100644
--- a/src/calibre/manual/gui.rst
+++ b/src/calibre/manual/gui.rst
@@ -84,6 +84,9 @@ Send to device
1. **Send to main memory**: The selected books are transferred to the main memory of the ebook reader.
2. **Send to card**: The selected books are transferred to the storage card on the ebook reader.
+You can control the file name and folder structure of files sent to the device by setting up a template in
+:guilabel:`Preferences->Import/Export->Sending books to devices`. Also see :ref:`templatelangcalibre`.
+
.. _save_to_disk:
Save to disk
@@ -108,6 +111,10 @@ All available formats as well as metadata is stored to disk for each selected bo
Saved books can be re-imported to the library without any loss of information by using the :ref:`Add books ` action.
+You can control the file name and folder structure of files saved to disk by setting up a template in
+:guilabel:`Preferences->Import/Export->Saving books to disk`. Also see :ref:`templatelangcalibre`.
+
+
.. _fetch_news:
Fetch news
diff --git a/src/calibre/manual/index.rst b/src/calibre/manual/index.rst
index 827d848eb1..40c260b8b5 100644
--- a/src/calibre/manual/index.rst
+++ b/src/calibre/manual/index.rst
@@ -39,4 +39,10 @@ Sections
develop
glossary
+.. toctree::
+ :hidden:
+ :maxdepth: 2
+
+ template_lang
+ portable
diff --git a/src/calibre/manual/template_lang.rst b/src/calibre/manual/template_lang.rst
new file mode 100644
index 0000000000..541b5da138
--- /dev/null
+++ b/src/calibre/manual/template_lang.rst
@@ -0,0 +1,61 @@
+
+.. include:: global.rst
+
+.. _templatelangcalibre:
+
+The |app| template language
+=======================================================
+
+The |app| template language is used in various places. It is used to control the folder structure and file name when saving files from the |app| library to the disk or eBook reader.
+It is used to define "virtual" columns that contain data from other columns and so on.
+
+In essence, the template language is very simple. The basic idea is that a template consists of names in curly brackets that are then replaced by the corresponding metadata from the book being processed. So, for example, the default template used for saving books to device in |app| is::
+
+ {author_sort}/{title}/{title} - {authors}
+
+For the book "The Foundation" by "Isaac Asimov" it will become::
+
+ Asimov, Isaac/The Foundation/The Foundation - Isaac Asimov
+
+You can use all the various metadata fields available in calibre in a template, including the custom columns you have created yourself. To find out the template name for a column sinply hover your mouse over the column header. Names for custom fields (columns you have created yourself) are always prefixed by an #. For series type fields, there is always an additional field named ``series_index`` that becomes the series index for that series. So if you have a custom series field named #myseries, there will also be a field named #myseries_index. In addition to the column based fields, you also can use::
+
+ {formats} - A list of formats available in the |app| library for a book
+ {isbn} - The ISBN number of the book
+
+If a particular book does not have a particular piece of metadata, the field in the template is automatically removed for that book. So for example::
+
+ {author_sort}/{series}/{title} {series_index}
+
+will become::
+
+ {Asimov, Isaac}/Foundation/Second Foundation - 3
+
+and if a book does not have a series::
+
+ {Asimov, Isaac}/Second Foundation
+
+(|app| automatically removes multiple slashes and leading or trailing spaces).
+
+
+Advanced formatting
+----------------------
+
+You can do more than just simple substitution with the templates. You can also control how the substituted data is formatted. For instance, suppose you wanted to ensure that the series_index is always formatted as three digits with leading zeros. This would do the trick::
+
+ {series_index:0>3s} - Three digits with leading zeros
+
+If instead of leading zeros you want leading spaces, use::
+
+ {series_index:>3s} - Thre digits with leading spaces
+
+For trailing zeros, use::
+
+ {series_index:0<3s} - Three digits with trailing zeros
+
+
+If you want only the first two letters of the data to be rendered, use::
+
+ {author_sort:.2} - Only the first two letter of the author sort name
+
+The |app| template language comes from python and for more details on the syntax of these advanced formatting operations, look at the `Python documentation `_.
+
From 792fccbcad06c995b93ce9b97539d359e17740b5 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Mon, 20 Sep 2010 08:32:16 +0100
Subject: [PATCH 055/207] Fix validation exception and exception when empty
fields are subscripted in templates
---
src/calibre/gui2/preferences/save_template.py | 18 +++++++++++++++---
src/calibre/library/save_to_disk.py | 5 ++++-
2 files changed, 19 insertions(+), 4 deletions(-)
diff --git a/src/calibre/gui2/preferences/save_template.py b/src/calibre/gui2/preferences/save_template.py
index 0dbee5bf21..0f48893b69 100644
--- a/src/calibre/gui2/preferences/save_template.py
+++ b/src/calibre/gui2/preferences/save_template.py
@@ -6,12 +6,24 @@ __license__ = 'GPL v3'
__copyright__ = '2009, Kovid Goyal '
__docformat__ = 'restructuredtext en'
+import string
+
from PyQt4.Qt import QWidget, pyqtSignal
from calibre.gui2 import error_dialog
from calibre.gui2.preferences.save_template_ui import Ui_Form
-from calibre.library.save_to_disk import FORMAT_ARG_DESCS, preprocess_template,\
- safe_format
+from calibre.library.save_to_disk import FORMAT_ARG_DESCS, preprocess_template
+
+class ValidateFormat(string.Formatter):
+ '''
+ Provides a format function that substitutes '' for any missing value
+ '''
+ def get_value(self, key, args, kwargs):
+ return 'this is some text that should be long enough'
+
+validate_formatter = ValidateFormat()
+def validate_format(x, format_args):
+ return validate_formatter.vformat(x, [], format_args).strip()
class SaveTemplate(QWidget, Ui_Form):
@@ -52,7 +64,7 @@ class SaveTemplate(QWidget, Ui_Form):
tmpl = preprocess_template(self.opt_template.text())
fa = {}
try:
- safe_format(tmpl, fa)
+ validate_format(tmpl, fa)
except Exception, err:
error_dialog(self, _('Invalid template'),
'
'+_('The template %s is invalid:')%tmpl + \
diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py
index 71850abcd5..d5300d93e9 100644
--- a/src/calibre/library/save_to_disk.py
+++ b/src/calibre/library/save_to_disk.py
@@ -113,7 +113,10 @@ class SafeFormat(string.Formatter):
safe_formatter = SafeFormat()
def safe_format(x, format_args):
- ans = safe_formatter.vformat(x, [], format_args).strip()
+ try:
+ ans = safe_formatter.vformat(x, [], format_args).strip()
+ except:
+ ans = ''
return re.sub(r'\s+', ' ', ans)
def get_components(template, mi, id, timefmt='%b %Y', length=250,
From 89f64db891cfbc3c1ab276c87dc2eb8e826edb56 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Mon, 20 Sep 2010 09:51:04 +0100
Subject: [PATCH 056/207] Field interface, including refactoring (renaming)
some existing methods.
---
src/calibre/ebooks/metadata/book/base.py | 88 +++++++++++++++--------
src/calibre/gui2/dialogs/metadata_bulk.py | 4 +-
src/calibre/gui2/library/models.py | 2 +-
src/calibre/gui2/ui.py | 3 +
src/calibre/library/caches.py | 2 +-
src/calibre/library/database2.py | 36 ++++++++++
src/calibre/library/field_metadata.py | 46 +++++-------
src/calibre/library/save_to_disk.py | 2 +-
src/calibre/library/server/content.py | 2 +-
9 files changed, 123 insertions(+), 62 deletions(-)
diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index 7b8eb07908..3d6d6b1bb8 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -138,20 +138,66 @@ class Metadata(object):
def set(self, field, val, extra=None):
self.__setattr__(field, val, extra)
- @property
- def all_keys(self):
+ # field-oriented interface. Intended to be the same as in LibraryDatabase
+
+ def standard_field_keys(self):
'''
- All attribute keys known by this instance, even if their value is None
+ return a list of all possible keys, even if this book doesn't have them
+ '''
+ return STANDARD_METADATA_FIELDS
+
+ def custom_field_keys(self):
+ '''
+ return a list of the custom fields in this book
+ '''
+ return object.__getattribute__(self, '_data')['user_metadata'].iterkeys()
+
+ def all_field_keys(self):
+ '''
+ All field keys known by this instance, even if their value is None
'''
_data = object.__getattribute__(self, '_data')
return frozenset(ALL_METADATA_FIELDS.union(_data['user_metadata'].iterkeys()))
- @property
+ def metadata_for_field(self, key):
+ '''
+ return metadata describing a standard or custom field.
+ '''
+ if key in self.user_metadata_keys():
+ return self.get_standard_metadata(self, key, make_copy=False)
+ return self.get_user_metadata(key, make_copy=False)
+
def user_metadata_keys(self):
- 'The set of user metadata names this object knows about'
+ '''
+ Return the standard keys actually in this book.
+ '''
_data = object.__getattribute__(self, '_data')
return frozenset(_data['user_metadata'].iterkeys())
+ def all_non_none_fields(self):
+ '''
+ Return a dictionary containing all non-None metadata fields, including
+ the custom ones.
+ '''
+ result = {}
+ _data = object.__getattribute__(self, '_data')
+ for attr in STANDARD_METADATA_FIELDS:
+ v = _data.get(attr, None)
+ if v is not None:
+ result[attr] = v
+ for attr in _data['user_metadata'].iterkeys():
+ v = _data['user_metadata'][attr]['#value#']
+ if v is not None:
+ result[attr] = v
+ if _data['user_metadata'][attr]['datatype'] == 'series':
+ result[attr+'_index'] = _data['user_metadata'][attr]['#extra#']
+ return result
+
+ # End of field-oriented interface
+
+ # Extended interfaces. These permit one to get copies of metadata dictionaries, and to
+ # get and set custom field metadata
+
def get_standard_metadata(self, field, make_copy):
'''
return field metadata from the field if it is there. Otherwise return
@@ -237,30 +283,11 @@ class Metadata(object):
_data = object.__getattribute__(self, '_data')
_data['user_metadata'][field] = metadata
- def get_all_non_none_attributes(self):
- '''
- Return a dictionary containing all non-None metadata fields, including
- the custom ones.
- '''
- result = {}
- _data = object.__getattribute__(self, '_data')
- for attr in STANDARD_METADATA_FIELDS:
- v = _data.get(attr, None)
- if v is not None:
- result[attr] = v
- for attr in _data['user_metadata'].iterkeys():
- v = _data['user_metadata'][attr]['#value#']
- if v is not None:
- result[attr] = v
- if _data['user_metadata'][attr]['datatype'] == 'series':
- result[attr+'_index'] = _data['user_metadata'][attr]['#extra#']
- return result
-
# Old Metadata API {{{
def print_all_attributes(self):
for x in STANDARD_METADATA_FIELDS:
prints('%s:'%x, getattr(self, x, 'None'))
- for x in self.user_metadata_keys:
+ for x in self.user_metadata_keys():
meta = self.get_user_metadata(x, make_copy=False)
if meta is not None:
prints(x, meta)
@@ -326,7 +353,7 @@ class Metadata(object):
self.cover_data = other.cover_data
if getattr(other, 'user_metadata_keys', None):
- for x in other.user_metadata_keys:
+ for x in other.user_metadata_keys():
meta = other.get_user_metadata(x, make_copy=True)
if meta is not None:
self_tags = self.get(x, [])
@@ -389,7 +416,7 @@ class Metadata(object):
'''
returns the tuple (field_name, formatted_value)
'''
- if key in self.user_metadata_keys:
+ if key in self.user_metadata_keys():
res = self.get(key, None)
cmeta = self.get_user_metadata(key, make_copy=False)
if cmeta['datatype'] != 'composite' and (res is None or res == ''):
@@ -432,6 +459,9 @@ class Metadata(object):
return (None, None, None, None)
+ def expand_template(self, template):
+ return format_composite(template, self)
+
def __unicode__(self):
from calibre.ebooks.metadata import authors_to_string
ans = []
@@ -466,7 +496,7 @@ class Metadata(object):
fmt('Published', isoformat(self.pubdate))
if self.rights is not None:
fmt('Rights', unicode(self.rights))
- for key in self.user_metadata_keys:
+ for key in self.user_metadata_keys():
val = self.get(key, None)
if val is not None:
(name, val) = self.format_field(key)
@@ -491,7 +521,7 @@ class Metadata(object):
ans += [(_('Published'), unicode(self.pubdate.isoformat(' ')))]
if self.rights is not None:
ans += [(_('Rights'), unicode(self.rights))]
- for key in self.user_metadata_keys:
+ for key in self.user_metadata_keys():
val = self.get(key, None)
if val is not None:
(name, val) = self.format_field(key)
diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py
index 1fb889757f..83cf6278e5 100644
--- a/src/calibre/gui2/dialogs/metadata_bulk.py
+++ b/src/calibre/gui2/dialogs/metadata_bulk.py
@@ -270,11 +270,11 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
def s_r_func(self, match):
rfunc = self.s_r_functions[unicode(self.replace_func.currentText())]
rtext = unicode(self.replace_with.text())
- mi_data = self.mi.get_all_non_none_attributes()
+ mi_data = self.mi.all_non_none_fields()
def fm_func(m):
try:
- if m.group(3) not in self.mi.all_keys: return m.group(0)
+ if m.group(3) not in self.mi.all_field_keys(): return m.group(0)
else: return '%s{%s}'%(m.group(1), m.group(3))
except:
import traceback
diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index be1bf9bc2d..6941869e44 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -319,7 +319,7 @@ class BooksModel(QAbstractTableModel): # {{{
_('Book %s of %s.')%\
(sidx, prepare_string_for_xml(series))
mi = self.db.get_metadata(idx)
- for key in mi.user_metadata_keys:
+ for key in mi.user_metadata_keys():
name, val = mi.format_field(key)
if val is not None:
data[name] = val
diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py
index f8d50d1cd2..647e31ff51 100644
--- a/src/calibre/gui2/ui.py
+++ b/src/calibre/gui2/ui.py
@@ -533,6 +533,9 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, # {{{
# Save the current field_metadata for applications like calibre2opds
# Goes here, because if cf is valid, db is valid.
db.prefs['field_metadata'] = db.field_metadata.all_metadata()
+ if db.gm_count > 0:
+ print 'get_metadata cache: {0:d} calls, {1:4.2f}% misses'.format(
+ db.gm_count, (db.gm_missed*100.0)/db.gm_count)
for action in self.iactions.values():
if not action.shutting_down():
return
diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index 770a362a1d..5f7fbdccc9 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -621,7 +621,7 @@ class ResultCache(SearchQueryParser):
def multisort(self, fields=[], subsort=False):
fields = [(self.sanitize_sort_field_name(x), bool(y)) for x, y in fields]
- keys = self.field_metadata.sortable_keys()
+ keys = self.field_metadata.sortable_field_keys()
fields = [x for x in fields if x[0] in keys]
if subsort and 'sort' not in [x[0] for x in fields]:
fields += [('sort', True)]
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index 106b498ee8..f5a474edbc 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -325,6 +325,10 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
self.has_id = self.data.has_id
self.count = self.data.count
+ # Count times get_metadata is called, and how many times in the cache
+ self.gm_count = 0
+ self.gm_missed = 0
+
for prop in ('author_sort', 'authors', 'comment', 'comments', 'isbn',
'publisher', 'rating', 'series', 'series_index', 'tags',
'title', 'timestamp', 'uuid', 'pubdate', 'ondevice'):
@@ -520,15 +524,47 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
f.close()
return ans
+ ### The field-style interface. These use field keys.
+
+ def get_field(self, idx, key, default=None, index_is_id=False):
+ mi = self.get_metadata(idx, index_is_id=index_is_id, get_cover=True)
+ try:
+ return mi[key]
+ except:
+ return default
+
+ def standard_field_keys(self):
+ return self.field_metadata.standard_field_keys()
+
+ def custom_field_keys(self):
+ return self.field_metadata.custom_field_keys()
+
+ def all_field_keys(self):
+ return self.field_metadata.all_field_keys()
+
+ def sortable_field_keys(self):
+ return self.field_metadata.sortable_field_keys()
+
+ def searchable_fields(self):
+ return self.field_metadata.searchable_field_keys()
+
+ def search_term_to_field_key(self, term):
+ return self.field_metadata.search_term_to_key(term)
+
+ def metadata_for_field(self, key):
+ return self.field_metadata[key]
+
def get_metadata(self, idx, index_is_id=False, get_cover=False):
'''
Convenience method to return metadata as a :class:`Metadata` object.
'''
+ self.gm_count += 1
mi = self.data.get(idx, self.FIELD_MAP['all_metadata'],
row_is_id = index_is_id)
if mi is not None:
return mi
+ self.gm_missed += 1
mi = Metadata(None)
self.data.set(idx, self.FIELD_MAP['all_metadata'], mi,
row_is_id = index_is_id)
diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py
index e4a4f5270d..a8031e5172 100644
--- a/src/calibre/library/field_metadata.py
+++ b/src/calibre/library/field_metadata.py
@@ -348,11 +348,24 @@ class FieldMetadata(dict):
def keys(self):
return self._tb_cats.keys()
- def sortable_keys(self):
+ def sortable_field_keys(self):
return [k for k in self._tb_cats.keys()
if self._tb_cats[k]['kind']=='field' and
self._tb_cats[k]['datatype'] is not None]
+ def standard_field_keys(self):
+ return [k for k in self._tb_cats.keys()
+ if self._tb_cats[k]['kind']=='field' and
+ not self._tb_cats[k]['is_custom']]
+
+ def custom_field_keys(self):
+ return [k for k in self._tb_cats.keys()
+ if self._tb_cats[k]['kind']=='field' and
+ self._tb_cats[k]['is_custom']]
+
+ def all_field_keys(self):
+ return [k for k in self._tb_cats.keys() if self._tb_cats[k]['kind']=='field']
+
def iterkeys(self):
for key in self._tb_cats:
yield key
@@ -474,36 +487,10 @@ class FieldMetadata(dict):
key = self.custom_field_prefix+label
self._tb_cats[key]['rec_index'] = index # let the exception fly ...
-
-# DEFAULT_LOCATIONS = frozenset([
-# 'all',
-# 'author', # compatibility
-# 'authors',
-# 'comment', # compatibility
-# 'comments',
-# 'cover',
-# 'date',
-# 'format', # compatibility
-# 'formats',
-# 'isbn',
-# 'ondevice',
-# 'pubdate',
-# 'publisher',
-# 'search',
-# 'series',
-# 'rating',
-# 'tag', # compatibility
-# 'tags',
-# 'title',
-# ])
-
def get_search_terms(self):
s_keys = sorted(self._search_term_map.keys())
for v in self.search_items:
s_keys.append(v)
-# if set(s_keys) != self.DEFAULT_LOCATIONS:
-# print 'search labels and default_locations do not match:'
-# print set(s_keys) ^ self.DEFAULT_LOCATIONS
return s_keys
def _add_search_terms_to_map(self, key, terms):
@@ -518,3 +505,8 @@ class FieldMetadata(dict):
if term in self._search_term_map:
return self._search_term_map[term]
return term
+
+ def searchable_field_keys(self):
+ return [k for k in self._tb_cats.keys()
+ if self._tb_cats[k]['kind']=='field' and
+ len(self._tb_cats[k]['search_terms']) > 0]
diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py
index d5300d93e9..fe62dcb7fd 100644
--- a/src/calibre/library/save_to_disk.py
+++ b/src/calibre/library/save_to_disk.py
@@ -125,7 +125,7 @@ def get_components(template, mi, id, timefmt='%b %Y', length=250,
library_order = tweaks['save_template_title_series_sorting'] == 'library_order'
tsfmt = title_sort if library_order else lambda x: x
format_args = FORMAT_ARGS.copy()
- format_args.update(mi.get_all_non_none_attributes())
+ format_args.update(mi.all_non_none_fields())
if mi.title:
format_args['title'] = tsfmt(mi.title)
if mi.authors:
diff --git a/src/calibre/library/server/content.py b/src/calibre/library/server/content.py
index c3a662c0fd..041ea78051 100644
--- a/src/calibre/library/server/content.py
+++ b/src/calibre/library/server/content.py
@@ -56,7 +56,7 @@ class ContentServer(object):
def sort(self, items, field, order):
field = self.db.data.sanitize_sort_field_name(field)
- if field not in self.db.field_metadata.sortable_keys():
+ if field not in self.db.field_metadata.sortable_field_keys():
raise cherrypy.HTTPError(400, '%s is not a valid sort field'%field)
keyg = CSSortKeyGenerator([(field, order)], self.db.field_metadata)
items.sort(key=keyg, reverse=not order)
From e721bd44eeb674b89346baf0ab13c053bd26e149 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Mon, 20 Sep 2010 14:52:53 +0100
Subject: [PATCH 057/207] Interim release
---
src/calibre/gui2/dialogs/metadata_bulk.py | 248 ++++++++++++++--------
src/calibre/gui2/dialogs/metadata_bulk.ui | 152 +++++++++++--
src/calibre/library/database2.py | 5 +-
3 files changed, 294 insertions(+), 111 deletions(-)
diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py
index 83cf6278e5..3659547b13 100644
--- a/src/calibre/gui2/dialogs/metadata_bulk.py
+++ b/src/calibre/gui2/dialogs/metadata_bulk.py
@@ -122,12 +122,20 @@ def format_composite(x, mi):
class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
- s_r_functions = {
- '' : lambda x: x,
- _('Lower Case') : lambda x: x.lower(),
- _('Upper Case') : lambda x: x.upper(),
- _('Title Case') : lambda x: x.title(),
- }
+ s_r_functions = { '' : lambda x: x,
+ _('Lower Case') : lambda x: x.lower(),
+ _('Upper Case') : lambda x: x.upper(),
+ _('Title Case') : lambda x: x.title(),
+ }
+
+ s_r_match_modes = [ _('Character match'),
+ _('Regular Expression'),
+ ]
+
+ s_r_replace_modes = [ _('Replace field'),
+ _('Prepend to field'),
+ _('Append to field'),
+ ]
def __init__(self, window, rows, db):
QDialog.__init__(self, window)
@@ -179,27 +187,34 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
fields.sort()
self.search_field.addItems(fields)
self.search_field.setMaxVisibleItems(min(len(fields), 20))
+ self.destination_field.addItems(fields)
+ self.destination_field.setMaxVisibleItems(min(len(fields), 20))
offset = 10
self.s_r_number_of_books = min(7, len(self.ids))
for i in range(1,self.s_r_number_of_books+1):
w = QtGui.QLabel(self.tabWidgetPage3)
w.setText(_('Book %d:')%i)
- self.gridLayout1.addWidget(w, i+offset, 0, 1, 1)
+ self.testgrid.addWidget(w, i+offset, 0, 1, 1)
w = QtGui.QLineEdit(self.tabWidgetPage3)
w.setReadOnly(True)
name = 'book_%d_text'%i
setattr(self, name, w)
self.book_1_text.setObjectName(name)
- self.gridLayout1.addWidget(w, i+offset, 1, 1, 1)
+ self.testgrid.addWidget(w, i+offset, 1, 1, 1)
w = QtGui.QLineEdit(self.tabWidgetPage3)
w.setReadOnly(True)
name = 'book_%d_result'%i
setattr(self, name, w)
self.book_1_text.setObjectName(name)
- self.gridLayout1.addWidget(w, i+offset, 2, 1, 1)
+ self.testgrid.addWidget(w, i+offset, 2, 1, 1)
self.s_r_heading.setText('
'+
- _('Search and replace in text fields using '
+ _('You can destroy your library '
+ 'using this feature. Changes are permanent. There '
+ 'is no undo function. You are strongly encouraged '
+ 'to back up your library before proceeding.'
+ ) + '
' + _(
+ 'Search and replace in text fields using '
'regular expressions. The search text is an '
'arbitrary python-compatible regular expression. '
'The replacement text can contain backreferences '
@@ -209,51 +224,86 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
' '
'this reference '
'for more information, and in particular the \'sub\' '
- 'function.') + '
' + _(
- 'Note: you can destroy your library '
- 'using this feature. Changes are permanent. There '
- 'is no undo function. You are strongly encouraged '
- 'to back up your library before proceeding.'))
+ 'function.'
+ ))
+ self.search_mode.addItems(self.s_r_match_modes)
+ self.search_mode.setCurrentIndex(0)
+ self.replace_mode.addItems(self.s_r_replace_modes)
+ self.replace_mode.setCurrentIndex(0)
+
+ self.s_r_search_mode = 0
self.s_r_error = None
self.s_r_obj = None
self.replace_func.addItems(sorted(self.s_r_functions.keys()))
- self.search_field.currentIndexChanged[str].connect(self.s_r_field_changed)
+ self.search_mode.currentIndexChanged[int].connect(self.s_r_search_mode_changed)
+ self.search_field.currentIndexChanged[str].connect(self.s_r_search_field_changed)
+ self.destination_field.currentIndexChanged[str].connect(self.s_r_destination_field_changed)
+
+ self.replace_mode.currentIndexChanged[int].connect(self.s_r_paint_results)
self.replace_func.currentIndexChanged[str].connect(self.s_r_paint_results)
self.search_for.editTextChanged[str].connect(self.s_r_paint_results)
self.replace_with.editTextChanged[str].connect(self.s_r_paint_results)
self.test_text.editTextChanged[str].connect(self.s_r_paint_results)
+ self.comma_separated.stateChanged.connect(self.s_r_paint_results)
+ self.case_sensitive.stateChanged.connect(self.s_r_paint_results)
self.central_widget.setCurrentIndex(0)
self.search_for.completer().setCaseSensitivity(Qt.CaseSensitive)
self.replace_with.completer().setCaseSensitivity(Qt.CaseSensitive)
+ self.s_r_search_mode_changed(0)
- def s_r_field_changed(self, txt):
+ def s_r_get_field(self, mi, field):
+ if field:
+ fm = self.db.metadata_for_field(field)
+ val = mi.get(field, None)
+ if val is None:
+ val = []
+ elif not fm['is_multiple']:
+ val = [val]
+ elif field == 'authors':
+ val = [v.replace(',', '|') for v in val]
+ else:
+ val = []
+ return val
+
+ def s_r_search_field_changed(self, txt):
txt = unicode(txt)
for i in range(0, self.s_r_number_of_books):
- if txt:
- fm = self.db.field_metadata[txt]
- id = self.ids[i]
- val = self.db.get_property(id, index_is_id=True,
- loc=fm['rec_index'])
- if val is None:
- val = ''
- if fm['is_multiple']:
- val = [t.strip() for t in val.split(fm['is_multiple']) if t.strip()]
- if val:
- val.sort(cmp=lambda x,y: cmp(x.lower(), y.lower()))
- val = val[0]
- if txt == 'authors':
- val = val.replace('|', ',')
- else:
- val = ''
- else:
- val = ''
w = getattr(self, 'book_%d_text'%(i+1))
- w.setText(val)
+ mi = self.db.get_metadata(self.ids[i], index_is_id=True)
+ src = unicode(self.search_field.currentText())
+ t = self.s_r_get_field(mi, src)
+ w.setText(''.join(t[0:1]))
self.s_r_paint_results(None)
+ def s_r_destination_field_changed(self, txt):
+ txt = unicode(txt)
+ self.comma_separated.setEnabled(True)
+ if txt:
+ fm = self.db.metadata_for_field(txt)
+ if fm['is_multiple']:
+ self.comma_separated.setEnabled(False)
+ self.comma_separated.setChecked(True)
+ self.s_r_paint_results(None)
+
+ def s_r_search_mode_changed(self, val):
+ if val == 0:
+ self.destination_field.setCurrentIndex(0)
+ self.destination_field.setVisible(False)
+ self.destination_field_label.setVisible(False)
+ self.replace_mode.setCurrentIndex(0)
+ self.replace_mode.setVisible(False)
+ self.replace_mode_label.setVisible(False)
+ self.comma_separated.setVisible(False)
+ else:
+ self.destination_field.setVisible(True)
+ self.destination_field_label.setVisible(True)
+ self.replace_mode.setVisible(True)
+ self.replace_mode_label.setVisible(True)
+ self.comma_separated.setVisible(True)
+
def s_r_set_colors(self):
if self.s_r_error is not None:
col = 'rgb(255, 0, 0, 20%)'
@@ -265,32 +315,66 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
for i in range(0,self.s_r_number_of_books):
getattr(self, 'book_%d_result'%(i+1)).setText('')
- field_match_re = re.compile(r'(^|[^\\])(\\g<)([^>]+)(>)')
-
def s_r_func(self, match):
rfunc = self.s_r_functions[unicode(self.replace_func.currentText())]
rtext = unicode(self.replace_with.text())
- mi_data = self.mi.all_non_none_fields()
-
- def fm_func(m):
- try:
- if m.group(3) not in self.mi.all_field_keys(): return m.group(0)
- else: return '%s{%s}'%(m.group(1), m.group(3))
- except:
- import traceback
- traceback.print_exc()
- return m.group(0)
-
- rtext = re.sub(self.field_match_re, fm_func, rtext)
rtext = match.expand(rtext)
- rtext = format_composite(rtext, mi_data)
return rfunc(rtext)
+ def s_r_do_regexp(self, mi):
+ src_field = unicode(self.search_field.currentText())
+ src = self.s_r_get_field(mi, src_field)
+ result = []
+ for s in src:
+ result.append(self.s_r_obj.sub(self.s_r_func, s))
+ return result
+
+ def s_r_do_destination(self, mi, val):
+ src = unicode(self.search_field.currentText())
+ if src == '':
+ return ''
+ dest = unicode(self.destination_field.currentText())
+ if dest == '':
+ dest = src
+ dest_mode = self.replace_mode.currentIndex()
+
+ if dest_mode != 0:
+ dest_val = mi.get(dest, '')
+ if dest_val is None:
+ dest_val = []
+ elif isinstance(dest_val, list):
+ if dest == 'authors':
+ dest_val = [v.replace(',', '|') for v in dest_val]
+ else:
+ dest_val = [dest_val]
+ else:
+ dest_val = []
+
+ if len(val) > 0:
+ if src == 'authors':
+ val = [v.replace(',', '|') for v in val]
+ if dest_mode == 1:
+ val.extend(dest_val)
+ elif dest_mode == 2:
+ val[0:0] = dest_val
+ return val
+
+ def s_r_replace_mode_separator(self):
+ if self.comma_separated.isChecked():
+ return ','
+ return ''
+
def s_r_paint_results(self, txt):
self.s_r_error = None
self.s_r_set_colors()
+
+ if self.case_sensitive.isChecked():
+ flags = 0
+ else:
+ flags = re.I
+
try:
- self.s_r_obj = re.compile(unicode(self.search_for.text()))
+ self.s_r_obj = re.compile(unicode(self.search_for.text()), flags)
except Exception as e:
self.s_r_obj = None
self.s_r_error = e
@@ -298,7 +382,6 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
return
try:
- self.mi = MetaInformation(None, None)
self.test_result.setText(self.s_r_obj.sub(self.s_r_func,
unicode(self.test_text.text())))
except Exception as e:
@@ -307,60 +390,53 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
return
for i in range(0,self.s_r_number_of_books):
- id = self.ids[i]
- self.mi = self.db.get_metadata(id, index_is_id=True)
- wt = getattr(self, 'book_%d_text'%(i+1))
+ mi = self.db.get_metadata(self.ids[i], index_is_id=True)
wr = getattr(self, 'book_%d_result'%(i+1))
try:
- wr.setText(self.s_r_obj.sub(self.s_r_func, unicode(wt.text())))
+ result = self.s_r_do_regexp(mi)
+ t = self.s_r_do_destination(mi, result[0:1])
+ t = self.s_r_replace_mode_separator().join(t)
+ wr.setText(t)
except Exception as e:
+ import traceback
+ traceback.print_exc()
self.s_r_error = e
self.s_r_set_colors()
break
def do_search_replace(self):
- field = unicode(self.search_field.currentText())
- if not field or not self.s_r_obj:
+ source = unicode(self.search_field.currentText())
+ if not source or not self.s_r_obj:
return
-
- fm = self.db.field_metadata[field]
-
- def apply_pattern(val):
- try:
- return self.s_r_obj.sub(self.s_r_func, val)
- except:
- return val
+ dest = unicode(self.destination_field.currentText())
+ if not dest:
+ dest = source
+ dfm = self.db.field_metadata[source]
for id in self.ids:
- val = self.db.get_property(id, index_is_id=True,
- loc=fm['rec_index'])
+ mi = self.db.get_metadata(id, index_is_id=True,)
+ val = mi.get(source)
if val is None:
continue
- if fm['is_multiple']:
- res = []
- for val in [t.strip() for t in val.split(fm['is_multiple'])]:
- v = apply_pattern(val).strip()
- if v:
- res.append(v)
- val = res
- if fm['is_custom']:
+ val = self.s_r_do_regexp(mi)
+ val = self.s_r_do_destination(mi, val)
+ if dfm['is_multiple']:
+ if dfm['is_custom']:
# The standard tags and authors values want to be lists.
# All custom columns are to be strings
- val = fm['is_multiple'].join(val)
- elif field == 'authors':
- val = [v.replace('|', ',') for v in val]
+ val = dfm['is_multiple'].join(val)
else:
- val = apply_pattern(val)
+ val = self.s_r_replace_mode_separator().join(val)
- if fm['is_custom']:
- extra = self.db.get_custom_extra(id, label=fm['label'], index_is_id=True)
- self.db.set_custom(id, val, label=fm['label'], extra=extra,
+ if dfm['is_custom']:
+ extra = self.db.get_custom_extra(id, label=dfm['label'], index_is_id=True)
+ self.db.set_custom(id, val, label=dfm['label'], extra=extra,
commit=False)
else:
- if field == 'comments':
+ if dest == 'comments':
setter = self.db.set_comment
else:
- setter = getattr(self.db, 'set_'+field)
+ setter = getattr(self.db, 'set_'+dest)
setter(id, val, notify=False, commit=False)
self.db.commit()
diff --git a/src/calibre/gui2/dialogs/metadata_bulk.ui b/src/calibre/gui2/dialogs/metadata_bulk.ui
index aca7b0cb75..e433aaf327 100644
--- a/src/calibre/gui2/dialogs/metadata_bulk.ui
+++ b/src/calibre/gui2/dialogs/metadata_bulk.ui
@@ -319,7 +319,7 @@ Future conversion of these books will use the default settings.
&Search and replace (experimental)
-
+ QLayout::SetMinimumSize
@@ -351,6 +351,39 @@ Future conversion of these books will use the default settings.
+
+
+
+
+
+
+
+ Search mode:
+
+
+ search_field
+
+
+
+
+
+
+
+
+
+ Qt::Horizontal
+
+
+
+ 20
+ 10
+
+
+
+
+
+
+ &Search for:
@@ -360,7 +393,20 @@ Future conversion of these books will use the default settings.
-
+
+
+
+
+
+
+ Case sensitive
+
+
+ true
+
+
+
+ &Replace with:
@@ -370,29 +416,93 @@ Future conversion of these books will use the default settings.
-
-
-
-
-
-
-
+
-
-
+
+
+
+
+
+ Apply function after replace:
+
+
+ replace_func
+
+
+
+
+
+
+
+
+
+ Qt::Horizontal
+
+
+
+ 20
+ 10
+
+
+
+
+
+
+
+
- Apply function &after replace:
+ &Destination field:
- replace_func
+ destination_field
-
-
-
+
+
+
+
+
+
+
+ Mode:
+
+
+ replace_mode
+
+
+
+
+
+
+
+
+
+ use comma
+
+
+ true
+
+
+
+
+
+
+ Qt::Horizontal
+
+
+
+ 20
+ 10
+
+
+
+
+
+
+ Test &text
@@ -402,8 +512,8 @@ Future conversion of these books will use the default settings.
-
-
+
+ Test re&sult
@@ -412,17 +522,17 @@ Future conversion of these books will use the default settings.
-
+ Your test:
-
+
-
+
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index f5a474edbc..2f9f9b6f89 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -528,10 +528,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
def get_field(self, idx, key, default=None, index_is_id=False):
mi = self.get_metadata(idx, index_is_id=index_is_id, get_cover=True)
- try:
- return mi[key]
- except:
- return default
+ return mi.get(key, default)
def standard_field_keys(self):
return self.field_metadata.standard_field_keys()
From ea44e9053faf49c85df6d7e6abf8392ef37ffd12 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Mon, 20 Sep 2010 17:03:53 +0100
Subject: [PATCH 058/207] Finish search and replace.
Fix a bug in database2 that seems to be triggered by interactions with the cover cache.
---
src/calibre/gui2/dialogs/metadata_bulk.py | 60 +++++++++++++----------
src/calibre/library/database2.py | 23 ++++++---
2 files changed, 51 insertions(+), 32 deletions(-)
diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py
index 3659547b13..b01869deaa 100644
--- a/src/calibre/gui2/dialogs/metadata_bulk.py
+++ b/src/calibre/gui2/dialogs/metadata_bulk.py
@@ -11,11 +11,11 @@ from PyQt4 import QtGui
from calibre.gui2.dialogs.metadata_bulk_ui import Ui_MetadataBulkDialog
from calibre.gui2.dialogs.tag_editor import TagEditor
-from calibre.ebooks.metadata import string_to_authors, \
- authors_to_string, MetaInformation
+from calibre.ebooks.metadata import string_to_authors, authors_to_string
from calibre.gui2.custom_column_widgets import populate_metadata_page
from calibre.gui2.dialogs.progress import BlockingBusy
from calibre.gui2 import error_dialog, Dispatcher
+from calibre.utils.config import dynamic
class Worker(Thread):
@@ -208,26 +208,27 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
self.book_1_text.setObjectName(name)
self.testgrid.addWidget(w, i+offset, 2, 1, 1)
- self.s_r_heading.setText('
'+
- _('You can destroy your library '
- 'using this feature. Changes are permanent. There '
- 'is no undo function. You are strongly encouraged '
- 'to back up your library before proceeding.'
- ) + '
' + _(
- 'Search and replace in text fields using '
- 'regular expressions. The search text is an '
- 'arbitrary python-compatible regular expression. '
- 'The replacement text can contain backreferences '
- 'to parenthesized expressions in the pattern. '
- 'The search is not anchored, and can match and '
- 'replace multiple times on the same string. See '
- ' '
- 'this reference '
- 'for more information, and in particular the \'sub\' '
- 'function.'
- ))
+ self.s_r_heading.setText('
'+ _(
+ 'You can destroy your library using this feature. '
+ 'Changes are permanent. There is no undo function. '
+ ' This feature is experimental, and there may be bugs. '
+ 'You are strongly encouraged to back up your library '
+ 'before proceeding.'
+ ) + '
' + _(
+ 'Search and replace in text fields using character matching '
+ 'or regular expressions. In character mode, search text '
+ 'found in the specified field is replaced with replace '
+ 'text. In regular expression mode, the search text is an '
+ 'arbitrary python-compatible regular expression. The '
+ 'replacement text can contain backreferences to parenthesized '
+ 'expressions in the pattern. The search is not anchored, '
+ 'and can match and replace multiple times on the same string. '
+ 'See '
+ 'this reference for more information, and in particular '
+ 'the \'sub\' function.'
+ ))
self.search_mode.addItems(self.s_r_match_modes)
- self.search_mode.setCurrentIndex(0)
+ self.search_mode.setCurrentIndex(dynamic.get('s_r_search_mode', 0))
self.replace_mode.addItems(self.s_r_replace_modes)
self.replace_mode.setCurrentIndex(0)
@@ -252,7 +253,7 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
self.search_for.completer().setCaseSensitivity(Qt.CaseSensitive)
self.replace_with.completer().setCaseSensitivity(Qt.CaseSensitive)
- self.s_r_search_mode_changed(0)
+ self.s_r_search_mode_changed(self.search_mode.currentIndex())
def s_r_get_field(self, mi, field):
if field:
@@ -303,6 +304,7 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
self.replace_mode.setVisible(True)
self.replace_mode_label.setVisible(True)
self.comma_separated.setVisible(True)
+ self.s_r_paint_results(None)
def s_r_set_colors(self):
if self.s_r_error is not None:
@@ -325,8 +327,12 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
src_field = unicode(self.search_field.currentText())
src = self.s_r_get_field(mi, src_field)
result = []
+ rfunc = self.s_r_functions[unicode(self.replace_func.currentText())]
for s in src:
- result.append(self.s_r_obj.sub(self.s_r_func, s))
+ t = self.s_r_obj.sub(self.s_r_func, s)
+ if self.search_mode.currentIndex() == 0:
+ t = rfunc(t)
+ result.append(t)
return result
def s_r_do_destination(self, mi, val):
@@ -374,7 +380,10 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
flags = re.I
try:
- self.s_r_obj = re.compile(unicode(self.search_for.text()), flags)
+ if self.search_mode.currentIndex() == 0:
+ self.s_r_obj = re.compile(re.escape(unicode(self.search_for.text())), flags)
+ else:
+ self.s_r_obj = re.compile(unicode(self.search_for.text()), flags)
except Exception as e:
self.s_r_obj = None
self.s_r_error = e
@@ -411,7 +420,7 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
dest = unicode(self.destination_field.currentText())
if not dest:
dest = source
- dfm = self.db.field_metadata[source]
+ dfm = self.db.field_metadata[dest]
for id in self.ids:
mi = self.db.get_metadata(id, index_is_id=True,)
@@ -439,6 +448,7 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
setter = getattr(self.db, 'set_'+dest)
setter(id, val, notify=False, commit=False)
self.db.commit()
+ dynamic['s_r_search_mode'] = self.search_mode.currentIndex()
def create_custom_column_editors(self):
w = self.central_widget.widget(1)
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index 2f9f9b6f89..c1ada94a84 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -464,11 +464,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
# change case don't cause any changes to the directories in the file
# system. This can lead to having the directory names not match the
# title/author, which leads to trouble when libraries are copied to
- # a case-sensitive system. The following code fixes this by checking
- # each segment. If they are different because of case, then rename
- # the segment to some temp file name, then rename it back to the
- # correct name. Note that the code above correctly handles files in
- # the directories, so no need to do them here.
+ # a case-sensitive system. The following code attempts to fix this
+ # by checking each segment. If they are different because of case,
+ # then rename the segment to some temp file name, then rename it
+ # back to the correct name. Note that the code above correctly
+ # handles files in the directories, so no need to do them here.
for oldseg, newseg in zip(c1, c2):
if oldseg.lower() == newseg.lower() and oldseg != newseg:
while True:
@@ -476,8 +476,17 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
tempname = os.path.join(curpath, 'TEMP.%f'%time.time())
if not os.path.exists(tempname):
break
- os.rename(os.path.join(curpath, oldseg), tempname)
- os.rename(tempname, os.path.join(curpath, newseg))
+ try:
+ os.rename(os.path.join(curpath, oldseg), tempname)
+ except (IOError, OSError):
+ # Windows (at least) sometimes refuses to do the rename
+ # probably because a file such a cover is open in the
+ # hierarchy. Just go on -- nothing is hurt beyond the
+ # case of the filesystem not matching the case in
+ # name stored by calibre
+ print 'rename of library component failed'
+ else:
+ os.rename(tempname, os.path.join(curpath, newseg))
curpath = os.path.join(curpath, newseg)
def add_listener(self, listener):
From e2f4b969bc6d36fbb285818cb50184cf83efed6e Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Mon, 20 Sep 2010 19:49:03 +0100
Subject: [PATCH 059/207] 1) add tooltips 2) change main heading text depending
on the mode 3) add an error if attempting to assign '' to authors or title
---
src/calibre/gui2/dialogs/metadata_bulk.py | 48 +++++++++++++++++----
src/calibre/gui2/dialogs/metadata_bulk.ui | 52 +++++++++++++++++++----
2 files changed, 84 insertions(+), 16 deletions(-)
diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py
index b01869deaa..681f65b19e 100644
--- a/src/calibre/gui2/dialogs/metadata_bulk.py
+++ b/src/calibre/gui2/dialogs/metadata_bulk.py
@@ -208,25 +208,43 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
self.book_1_text.setObjectName(name)
self.testgrid.addWidget(w, i+offset, 2, 1, 1)
- self.s_r_heading.setText('
'+ _(
+ self.main_heading = _(
'You can destroy your library using this feature. '
'Changes are permanent. There is no undo function. '
' This feature is experimental, and there may be bugs. '
'You are strongly encouraged to back up your library '
'before proceeding.'
- ) + '
' + _(
+ + '
' +
'Search and replace in text fields using character matching '
- 'or regular expressions. In character mode, search text '
- 'found in the specified field is replaced with replace '
- 'text. In regular expression mode, the search text is an '
+ 'or regular expressions. ')
+
+ self.character_heading = _(
+ 'In character mode, the field is searched for the entered '
+ 'search text. The text is replaced by the specified replacement '
+ 'text everywhere it is found in the specified field. After '
+ 'replacement is finished, the text can be changed to '
+ 'upper-case, lower-case, or title-case. If the case-sensitive '
+ 'check box is checked, the search text must match exactly. If '
+ 'it is unchecked, the search text will match both upper- and '
+ 'lower-case letters'
+ )
+
+ self.regexp_heading = _(
+ 'In regular expression mode, the search text is an '
'arbitrary python-compatible regular expression. The '
'replacement text can contain backreferences to parenthesized '
'expressions in the pattern. The search is not anchored, '
'and can match and replace multiple times on the same string. '
+ 'The modification functions (lower-case etc) are applied to the '
+ 'matched text, not to the field as a whole. '
+ 'The destination box specifies the field where the result after '
+ 'matching and replacement is to be assigned. You can replace '
+ 'the text in the field, or prepend or append the matched text. '
'See '
- 'this reference for more information, and in particular '
- 'the \'sub\' function.'
- ))
+ 'this reference for more information on python\'s regular '
+ 'expressions, and in particular the \'sub\' function.'
+ )
+
self.search_mode.addItems(self.s_r_match_modes)
self.search_mode.setCurrentIndex(dynamic.get('s_r_search_mode', 0))
self.replace_mode.addItems(self.s_r_replace_modes)
@@ -298,12 +316,14 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
self.replace_mode.setVisible(False)
self.replace_mode_label.setVisible(False)
self.comma_separated.setVisible(False)
+ self.s_r_heading.setText('
'+self.main_heading + self.regexp_heading)
self.s_r_paint_results(None)
def s_r_set_colors(self):
@@ -434,8 +454,20 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
# The standard tags and authors values want to be lists.
# All custom columns are to be strings
val = dfm['is_multiple'].join(val)
+ if dest == 'authors' and len(val) == 0:
+ error_dialog(self, _('Search/replace invalid'),
+ _('Authors cannot be set to the empty string. '
+ 'Book title %s not processed')%mi.title,
+ show=True)
+ continue
else:
val = self.s_r_replace_mode_separator().join(val)
+ if dest == 'title' and len(val) == 0:
+ error_dialog(self, _('Search/replace invalid'),
+ _('Title cannot be set to the empty string. '
+ 'Book title %s not processed')%mi.title,
+ show=True)
+ continue
if dfm['is_custom']:
extra = self.db.get_custom_extra(id, label=dfm['label'], index_is_id=True)
diff --git a/src/calibre/gui2/dialogs/metadata_bulk.ui b/src/calibre/gui2/dialogs/metadata_bulk.ui
index e433aaf327..b2a3e11b4a 100644
--- a/src/calibre/gui2/dialogs/metadata_bulk.ui
+++ b/src/calibre/gui2/dialogs/metadata_bulk.ui
@@ -351,7 +351,11 @@ Future conversion of these books will use the default settings.
-
+
+
+ The name of the field that you want to search
+
+
@@ -361,12 +365,16 @@ Future conversion of these books will use the default settings.
Search mode:
- search_field
+ search_mode
-
+
+
+ Choose whether to use basic text matching or advanced regular expression matching
+
+
@@ -394,7 +402,11 @@ Future conversion of these books will use the default settings.
-
+
+
+ Enter the what you are looking for, either plain text or a regular expression, depending on the mode
+
+
@@ -404,6 +416,9 @@ Future conversion of these books will use the default settings.
true
+
+ Check this box if the search string must match exactly upper and lower case. Uncheck it if case is to be ignored
+
@@ -417,7 +432,11 @@ Future conversion of these books will use the default settings.
-
+
+
+ The replacement text. The matched search text will be replaced with this string
+
+
@@ -432,7 +451,12 @@ Future conversion of these books will use the default settings.
-
+
+
+ Specify how the text is to be processed after matching and replacement. In character mode, the entire
+field is processed. In regular expression mode, only the matched text is processed
+
+
@@ -460,7 +484,11 @@ Future conversion of these books will use the default settings.
-
+
+
+ The field that the text will be put into after all replacements. If blank, the source field is used.
+
+
@@ -475,7 +503,11 @@ Future conversion of these books will use the default settings.
-
+
+
+ Specify how the text should be copied into the destination.
+
+
@@ -485,6 +517,10 @@ Future conversion of these books will use the default settings.
true
+
+ If the replace mode is prepend or append, then this box indicates whether a comma or
+nothing should be put between the original text and the inserted text
+
From 57a8705ec1b869d97d04ceab75db9caf1426faad Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Mon, 20 Sep 2010 13:27:55 -0600
Subject: [PATCH 060/207] Don't maintain a separate prompt buffer
---
src/calibre/utils/pyconsole/editor.py | 49 +++++++++++++++------------
1 file changed, 28 insertions(+), 21 deletions(-)
diff --git a/src/calibre/utils/pyconsole/editor.py b/src/calibre/utils/pyconsole/editor.py
index 68b83539f2..431a10eda5 100644
--- a/src/calibre/utils/pyconsole/editor.py
+++ b/src/calibre/utils/pyconsole/editor.py
@@ -58,7 +58,6 @@ class Editor(QTextEdit):
QTextEdit.__init__(self, parent)
self.buf = ''
self.prompt_frame = None
- self.current_prompt = ['']
self.allow_output = False
self.prompt_frame_format = QTextFrameFormat()
self.prompt_frame_format.setBorder(1)
@@ -84,11 +83,21 @@ class Editor(QTextEdit):
self.interpreter = Interpreter(parent=self)
self.interpreter.show_error.connect(self.show_error)
- #it = self.prompt_frame.begin()
- #while not it.atEnd():
- # bl = it.currentBlock()
- # prints(repr(bl.text()))
- # it += 1
+ print list(self.prompt())
+
+
+ def prompt(self, strip_prompt_strings=True):
+ if not self.prompt_frame:
+ yield u'' if strip_prompt_strings else self.formatter.prompt
+ else:
+ it = self.prompt_frame.begin()
+ while not it.atEnd():
+ bl = it.currentBlock()
+ t = unicode(bl.text())
+ if strip_prompt_strings:
+ t = t[self.prompt_len:]
+ yield t
+ it += 1
# Rendering {{{
@@ -113,15 +122,16 @@ class Editor(QTextEdit):
c.setPosition(self.prompt_frame.firstPosition())
def render_current_prompt(self):
+ cp = list(self.prompt())
self.clear_current_prompt()
- for i, line in enumerate(self.current_prompt):
+ for i, line in enumerate(cp):
start = i == 0
- end = i == len(self.current_prompt) - 1
+ end = i == len(cp) - 1
self.formatter.render_prompt(not start, self.cursor)
self.formatter.render(self.lexer.get_tokens(line), self.cursor)
if not end:
- self.cursor.insertText('\n')
+ self.cursor.insertBlock()
def show_error(self, is_syntax_err, tb):
if self.prompt_frame is not None:
@@ -194,32 +204,29 @@ class Editor(QTextEdit):
def enter_pressed(self):
if self.prompt_frame is None:
return
- if self.current_prompt[0]:
+ cp = list(self.prompt())
+ if cp[0]:
c = self.root_frame.lastCursorPosition()
self.setTextCursor(c)
old_pf = self.prompt_frame
self.prompt_frame = None
oldbuf = self.buf
self.buf = ''
- ret = self.interpreter.runsource('\n'.join(self.current_prompt))
+ ret = self.interpreter.runsource('\n'.join(cp))
if ret: # Incomplete command
self.buf = oldbuf
self.prompt_frame = old_pf
- self.current_prompt.append('')
+ c = old_pf.lastCursorPosition()
+ c.insertBlock()
+ self.setTextCursor(c)
else: # Command completed
- self.current_prompt = ['']
old_pf.setFrameFormat(QTextFrameFormat())
self.render_current_prompt()
def text_typed(self, text):
- if not self.current_prompt[0]:
- self.cursor.beginEditBlock()
- else:
- self.cursor.joinPreviousEditBlock()
- self.current_prompt[-1] += text
- self.render_current_prompt()
- self.cursor.endEditBlock()
-
+ if self.prompt_frame is not None:
+ self.cursor.insertText(text)
+ self.render_current_prompt()
# }}}
From 111c73ab80549913cc405d86d09ca69ef648e583 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Mon, 20 Sep 2010 13:46:11 -0600
Subject: [PATCH 061/207] ...
---
.../utils/pyconsole/{editor.py => console.py} | 31 ++++++++++++-------
src/calibre/utils/pyconsole/main.py | 4 +--
2 files changed, 21 insertions(+), 14 deletions(-)
rename src/calibre/utils/pyconsole/{editor.py => console.py} (96%)
diff --git a/src/calibre/utils/pyconsole/editor.py b/src/calibre/utils/pyconsole/console.py
similarity index 96%
rename from src/calibre/utils/pyconsole/editor.py
rename to src/calibre/utils/pyconsole/console.py
index 431a10eda5..d95e86c7ef 100644
--- a/src/calibre/utils/pyconsole/editor.py
+++ b/src/calibre/utils/pyconsole/console.py
@@ -29,7 +29,7 @@ class EditBlock(object): # {{{
self.cursor.endEditBlock()
# }}}
-class Editor(QTextEdit):
+class Console(QTextEdit):
@property
def doc(self):
@@ -86,6 +86,8 @@ class Editor(QTextEdit):
print list(self.prompt())
+ # Prompt management {{{
+
def prompt(self, strip_prompt_strings=True):
if not self.prompt_frame:
yield u'' if strip_prompt_strings else self.formatter.prompt
@@ -99,15 +101,8 @@ class Editor(QTextEdit):
yield t
it += 1
-
- # Rendering {{{
-
- def render_block(self, text, restore_prompt=True):
- self.formatter.render(self.lexer.get_tokens(text), self.cursor)
- self.cursor.insertBlock()
- self.cursor.movePosition(self.cursor.End)
- if restore_prompt:
- self.render_current_prompt()
+ def set_prompt(self, lines):
+ self.render_current_prompt(lines)
def clear_current_prompt(self):
if self.prompt_frame is None:
@@ -121,8 +116,8 @@ class Editor(QTextEdit):
c.removeSelectedText()
c.setPosition(self.prompt_frame.firstPosition())
- def render_current_prompt(self):
- cp = list(self.prompt())
+ def render_current_prompt(self, lines=None):
+ cp = list(self.prompt()) if lines is None else lines
self.clear_current_prompt()
for i, line in enumerate(cp):
@@ -133,6 +128,18 @@ class Editor(QTextEdit):
if not end:
self.cursor.insertBlock()
+ # }}}
+
+
+ # Non-prompt Rendering {{{
+
+ def render_block(self, text, restore_prompt=True):
+ self.formatter.render(self.lexer.get_tokens(text), self.cursor)
+ self.cursor.insertBlock()
+ self.cursor.movePosition(self.cursor.End)
+ if restore_prompt:
+ self.render_current_prompt()
+
def show_error(self, is_syntax_err, tb):
if self.prompt_frame is not None:
# At a prompt, so redirect output
diff --git a/src/calibre/utils/pyconsole/main.py b/src/calibre/utils/pyconsole/main.py
index c2694aae5f..af99ec66bb 100644
--- a/src/calibre/utils/pyconsole/main.py
+++ b/src/calibre/utils/pyconsole/main.py
@@ -10,7 +10,7 @@ from PyQt4.Qt import QMainWindow, QToolBar, QStatusBar, QLabel, QFont, Qt, \
QApplication
from calibre.constants import __appname__, __version__
-from calibre.utils.pyconsole.editor import Editor
+from calibre.utils.pyconsole.console import Console
class MainWindow(QMainWindow):
@@ -37,7 +37,7 @@ class MainWindow(QMainWindow):
self.tool_bar.setToolButtonStyle(Qt.ToolButtonTextOnly)
# }}}
- self.editor = Editor(parent=self)
+ self.editor = Console(parent=self)
self.setCentralWidget(self.editor)
From f770aa43bbf4a175cb614e437694732361209637 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Mon, 20 Sep 2010 14:24:42 -0600
Subject: [PATCH 062/207] Left and right arrow keys work
---
src/calibre/utils/pyconsole/console.py | 64 ++++++++++++++++++--------
1 file changed, 45 insertions(+), 19 deletions(-)
diff --git a/src/calibre/utils/pyconsole/console.py b/src/calibre/utils/pyconsole/console.py
index d95e86c7ef..73a19e7958 100644
--- a/src/calibre/utils/pyconsole/console.py
+++ b/src/calibre/utils/pyconsole/console.py
@@ -45,18 +45,30 @@ class Console(QTextEdit):
@property
def cursor_pos(self):
- pass
- #pos = self.cursor.position() - self.prompt_frame.firstPosition()
- #i = 0
- #for line in self.current_prompt:
- # i += self.prompt_len
+ '''
+ Return cursor position in prompt frame as (row, col).
+ row starts at 0 for the first line
+ col is 0 if the cursor is at the start of the line, 1 if it is after
+ the first character, n if it is after the nth char.
+ '''
+ if self.prompt_frame is not None:
+ pos = self.cursor.position()
+ it = self.prompt_frame.begin()
+ lineno = 0
+ while not it.atEnd():
+ bl = it.currentBlock()
+ if bl.contains(pos):
+ return (lineno, pos - bl.position())
+ it += 1
+ lineno += 1
+ return (-1, -1)
def __init__(self,
prompt='>>> ',
continuation='... ',
parent=None):
QTextEdit.__init__(self, parent)
- self.buf = ''
+ self.buf = []
self.prompt_frame = None
self.allow_output = False
self.prompt_frame_format = QTextFrameFormat()
@@ -130,7 +142,6 @@ class Console(QTextEdit):
# }}}
-
# Non-prompt Rendering {{{
def render_block(self, text, restore_prompt=True):
@@ -143,26 +154,25 @@ class Console(QTextEdit):
def show_error(self, is_syntax_err, tb):
if self.prompt_frame is not None:
# At a prompt, so redirect output
- return prints(tb)
+ return prints(tb, end='')
try:
- self.buf += tb
+ self.buf.append(tb)
if is_syntax_err:
self.formatter.render_syntax_error(tb, self.cursor)
else:
self.formatter.render(self.tb_lexer.get_tokens(tb), self.cursor)
except:
- prints(tb)
+ prints(tb, end='')
def show_output(self, raw):
if self.prompt_frame is not None:
# At a prompt, so redirect output
- return prints(raw)
+ return prints(raw, end='')
try:
- self.current_prompt_range = None
- self.buf += raw
+ self.buf.append(raw)
self.formatter.render_raw(raw, self.cursor)
except:
- prints(raw)
+ prints(raw, end='')
# }}}
@@ -187,13 +197,29 @@ class Console(QTextEdit):
QTextEdit.keyPressEvent(self, ev)
def left_pressed(self):
- pass
+ lineno, pos = self.cursor_pos
+ if lineno < 0: return
+ if pos > self.prompt_len:
+ c = self.cursor
+ c.movePosition(c.PreviousCharacter)
+ self.setTextCursor(c)
+ elif lineno > 0:
+ c = self.cursor
+ c.movePosition(c.Up)
+ c.movePosition(c.EndOfLine)
+ self.setTextCursor(c)
def right_pressed(self):
- if self.prompt_frame is not None:
- c = self.cursor
+ lineno, pos = self.cursor_pos
+ if lineno < 0: return
+ c = self.cursor
+ lineno, pos = self.cursor_pos
+ cp = list(self.prompt(False))
+ if pos < len(cp[lineno]):
c.movePosition(c.NextCharacter)
- self.setTextCursor(c)
+ elif lineno < len(cp)-1:
+ c.movePosition(c.NextCharacter, n=1+self.prompt_len)
+ self.setTextCursor(c)
def home_pressed(self):
if self.prompt_frame is not None:
@@ -218,7 +244,7 @@ class Console(QTextEdit):
old_pf = self.prompt_frame
self.prompt_frame = None
oldbuf = self.buf
- self.buf = ''
+ self.buf = []
ret = self.interpreter.runsource('\n'.join(cp))
if ret: # Incomplete command
self.buf = oldbuf
From acec240ef8a8ad5be4a02deb52e7eb72f01bfa0f Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Mon, 20 Sep 2010 21:48:52 +0100
Subject: [PATCH 063/207] Add scroll bar. Increase number of books to 10
---
src/calibre/gui2/dialogs/metadata_bulk.py | 2 +-
src/calibre/gui2/dialogs/metadata_bulk.ui | 42 +++++++++++++++++------
2 files changed, 32 insertions(+), 12 deletions(-)
diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py
index 681f65b19e..7122fe14fa 100644
--- a/src/calibre/gui2/dialogs/metadata_bulk.py
+++ b/src/calibre/gui2/dialogs/metadata_bulk.py
@@ -190,7 +190,7 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
self.destination_field.addItems(fields)
self.destination_field.setMaxVisibleItems(min(len(fields), 20))
offset = 10
- self.s_r_number_of_books = min(7, len(self.ids))
+ self.s_r_number_of_books = min(10, len(self.ids))
for i in range(1,self.s_r_number_of_books+1):
w = QtGui.QLabel(self.tabWidgetPage3)
w.setText(_('Book %d:')%i)
diff --git a/src/calibre/gui2/dialogs/metadata_bulk.ui b/src/calibre/gui2/dialogs/metadata_bulk.ui
index ec5a952346..f28f3fb57c 100644
--- a/src/calibre/gui2/dialogs/metadata_bulk.ui
+++ b/src/calibre/gui2/dialogs/metadata_bulk.ui
@@ -319,7 +319,7 @@ Future conversion of these books will use the default settings.
&Search and replace (experimental)
-
+ QLayout::SetMinimumSize
@@ -406,6 +406,12 @@ Future conversion of these books will use the default settings.
Enter the what you are looking for, either plain text or a regular expression, depending on the mode
+
+
+ 100
+ 0
+
+
@@ -558,19 +564,33 @@ nothing should be put between the original text and the inserted text
-
-
-
- Your test:
+
+
+
+ QFrame::NoFrame
+
+ true
+
+
+
+
+
+
+ Your test:
+
+
+
+
+
+
+
+
+
+
+
-
-
-
-
-
-
From 4b981a5257be82a9f4cfaeb8fca32f48f54dc7d0 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Mon, 20 Sep 2010 14:58:24 -0600
Subject: [PATCH 064/207] Inserting, not just appending text now works
---
src/calibre/utils/pyconsole/console.py | 68 +++++++++++++++++---------
1 file changed, 45 insertions(+), 23 deletions(-)
diff --git a/src/calibre/utils/pyconsole/console.py b/src/calibre/utils/pyconsole/console.py
index 73a19e7958..19a24dfdd7 100644
--- a/src/calibre/utils/pyconsole/console.py
+++ b/src/calibre/utils/pyconsole/console.py
@@ -43,25 +43,6 @@ class Console(QTextEdit):
def root_frame(self):
return self.doc.rootFrame()
- @property
- def cursor_pos(self):
- '''
- Return cursor position in prompt frame as (row, col).
- row starts at 0 for the first line
- col is 0 if the cursor is at the start of the line, 1 if it is after
- the first character, n if it is after the nth char.
- '''
- if self.prompt_frame is not None:
- pos = self.cursor.position()
- it = self.prompt_frame.begin()
- lineno = 0
- while not it.atEnd():
- bl = it.currentBlock()
- if bl.contains(pos):
- return (lineno, pos - bl.position())
- it += 1
- lineno += 1
- return (-1, -1)
def __init__(self,
prompt='>>> ',
@@ -95,11 +76,48 @@ class Console(QTextEdit):
self.interpreter = Interpreter(parent=self)
self.interpreter.show_error.connect(self.show_error)
- print list(self.prompt())
-
# Prompt management {{{
+ @dynamic_property
+ def cursor_pos(self):
+ doc = '''
+ The cursor position in the prompt has the form (row, col).
+ row starts at 0 for the first line
+ col is 0 if the cursor is at the start of the line, 1 if it is after
+ the first character, n if it is after the nth char.
+ '''
+
+ def fget(self):
+ if self.prompt_frame is not None:
+ pos = self.cursor.position()
+ it = self.prompt_frame.begin()
+ lineno = 0
+ while not it.atEnd():
+ bl = it.currentBlock()
+ if bl.contains(pos):
+ return (lineno, pos - bl.position())
+ it += 1
+ lineno += 1
+ return (-1, -1)
+
+ def fset(self, val):
+ row, col = val
+ if self.prompt_frame is not None:
+ it = self.prompt_frame.begin()
+ lineno = 0
+ while not it.atEnd():
+ if lineno == row:
+ c = self.cursor
+ c.setPosition(it.currentBlock().position())
+ c.movePosition(c.NextCharacter, n=col)
+ self.setTextCursor(c)
+ break
+ it += 1
+ lineno += 1
+
+ return property(fget=fget, fset=fset, doc=doc)
+
def prompt(self, strip_prompt_strings=True):
if not self.prompt_frame:
yield u'' if strip_prompt_strings else self.formatter.prompt
@@ -128,7 +146,8 @@ class Console(QTextEdit):
c.removeSelectedText()
c.setPosition(self.prompt_frame.firstPosition())
- def render_current_prompt(self, lines=None):
+ def render_current_prompt(self, lines=None, restore_cursor=False):
+ row, col = self.cursor_pos
cp = list(self.prompt()) if lines is None else lines
self.clear_current_prompt()
@@ -140,6 +159,9 @@ class Console(QTextEdit):
if not end:
self.cursor.insertBlock()
+ if row > -1 and restore_cursor:
+ self.cursor_pos = (row, col)
+
# }}}
# Non-prompt Rendering {{{
@@ -259,7 +281,7 @@ class Console(QTextEdit):
def text_typed(self, text):
if self.prompt_frame is not None:
self.cursor.insertText(text)
- self.render_current_prompt()
+ self.render_current_prompt(restore_cursor=True)
# }}}
From fe6d962bee771a34dd4937b6f332e0d0d4114676 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Mon, 20 Sep 2010 19:33:17 -0600
Subject: [PATCH 065/207] Use dialog instead of main window. Set console
stylesheet based on pygments style. Don't block if there is a lot of output
---
imgsrc/console.svg | 4339 ++++++++++++++++++++++
resources/images/console.png | Bin 0 -> 5110 bytes
src/calibre/utils/pyconsole/__init__.py | 9 +
src/calibre/utils/pyconsole/console.py | 117 +-
src/calibre/utils/pyconsole/formatter.py | 14 +-
src/calibre/utils/pyconsole/main.py | 44 +-
6 files changed, 4482 insertions(+), 41 deletions(-)
create mode 100644 imgsrc/console.svg
create mode 100644 resources/images/console.png
diff --git a/imgsrc/console.svg b/imgsrc/console.svg
new file mode 100644
index 0000000000..0d502bb1da
--- /dev/null
+++ b/imgsrc/console.svg
@@ -0,0 +1,4339 @@
+
+
+
diff --git a/resources/images/console.png b/resources/images/console.png
new file mode 100644
index 0000000000000000000000000000000000000000..168f0ccb2a177087d2ee6157bb67166688c84c05
GIT binary patch
literal 5110
zcmVxU{8
zPX+SQ2HuQqEW9l2vVZVDFa={E2%3ks4|OUPwOKoMg@^=5$wNZbR$+}383{v5y#|{)
zyEePt#rEF4GjsYdf9{zx-#K$;?wz}{*YlIcXJ*ddIp6pFzVDowy#{hz*Dy5h%gW@)
zE|}yz1W~vKcNDmDs%(zzfpIAVU~}iBB8Zk=aE}szB6s9UkbUsNx46TEqX;$#B`CS6
zC;TFSb^smj(e9S<*Z^Gl9NB#QUXx{+5vnGzzy!P6hNC>E?0nT
zLR0_~Ko!6>09V`ta1U2e
zwR!XArvXd=xCG!ToPaRE0NxXNOF(}4SG?z*d+xY&=@Q1r$H7BqB2^Qdq+UJ!)Z0HL
z5M2Q^Gp|yqV0LyEj4`ZTyLJ<5mhZXnfD7RLx|dnt4r9rZC2bcjTtFisT3OV2Bl^+l
zU#A@v6=x}La&i*w?d=^1FX02=wL%J>_IWf~U^3|R(}=M&>v_@wuMm|=g;J2vLq$gR
zwr$4>R)B@K@J5#rpI`L5S;-Uc3wh*2QZ68H#1(!LWXZy1WwJ!?x_P#31Htz>A(Uhp
zRD-8Zox;@AR8*B53y2jfR$#-14FL3(j*$w$V|scT!^6YLMwX)~ICSWcFL;*$l!(Ou
z9=f`^uxj;6tXjPaot-zpFbsMP%K(O2>SVboW6oNHO1W};{aPOUGVscRtD{&?i(?9m
zO*N*-v4!W1TB|n_j4{m5&*RdiOZf83FEKki8_JJDECuj<;;PjvvF>|!DJlgLUp&0P
z#1d|2H9?B`if8dlsh&49CN7Pbr+}_Sj!h_)Vhm*%20A)AaP!SK0{~`cX9IDI4g-im
zUUs;nIL#s+hKLBS-rFc)vfen=Kz*3Uk;!qBVrXm
zB9Q`gb#+P0)Naz3Iu_>@3{JhM2{fC8X_sIhp&lvX2sV+bMUw^Zp`)V%0Mcau56Wej
zglfYVlB!G;I6(nKlZoj+)R1CK6jLpbz(r6pNsR$iVQqE!8Alt~-9n^ztZtpenQ1Bpi9
z70#ox#SB`o$_Lh@c%6C+<^TCNf8tv}&)TUaxeAH)0c2&u;grncgp?6Wt3W+TzkdI!
z!AVD?)=*+e8X6;^lu78HF&22|;+=a!)uD2#&E_*eprVlSmoX{6;r?I3Cekq}>kGFc
zToW%`QP7P*Q(Ztqmw!@$iy1VV)V=)E(Jm(TDj7dxuq?|LvM1Dw&C>8wJjf;Ol25s%Z6=R
zFii`VZGqVg{yTvMSoW#VPz8{E9~Uf4U+P$#m!}sS#GrpMb?fVljWNKoE!dVLaND-R
zFN30CUJr?(=uUOwrlBbu;z|U9h(R&1pP@*6HSQ$l5Dsn$xC8!a{uor
zm2F$FOv@L1=vx3~WEdhH06wObfP0>x@N@
zjq6&fgBcTi^J4%h)iOc)qBBl!`R56o3f!Uu7KllF!G~QM|7=;Ie;~dBXuS-g{%lIN
z{OjG-i*K#|RH7dq
zq1tKB?YCp+qdRf#+Nv8ydrP*-yN(y>v+
zG9-8ksqOHq#7$B!Sw+0Q;#
zP%V>MoMl@uP1E0FmyTEYFp@rR2XA{vU|H574kLf;{~0cfSP#L!nUck7B8a
zHEX_u!Ka3?ZR;agvEnAK1G&X9+eWoIk89Vip;D>9G)?J4DR%(_0t|GWh=l@K*^s-B
zxF3ba`oj(0AS;B=-~DBviJ
zA+`v!YQnNiSeEmfMvSmPrIt&mnhXmf>Md6PuZzUXK$XQ!OibV({_*e8b9)aSc;G>-
z`Su#{-gpC-XD-uCD&{!(#~-b=`v=4i0_%$bD2hec;y(QF6rMe>AOH5#f5ptqj2|0#
zD~pbf4*dC_ZNZQK?#HzjNWPZQzuHoTa8(bIU^HGM%||L5LE)Q$gf=pwonQR&7wGNV
zg`fQMVa&|Tpja#hj?T_5b>Nkh{zDBBRTaS~9UEki{yn7rHZbnTAGU2d{@pZT6bdL6
zi^|_5X&C(n>eFM3ej_Q#&z{@?q___d+6Yq7zYEK>P&G~1mIW9FzW@Ed#QlGDKb9?9
z9?>b)hX(zNso;xrm`awX#X=cC)0TlMpK6&F=BrgO+Xe%$e*Jp<&0jx+TW`Hp+TQu|
z=kc>+M~Rj_fhQjnX~+@vV#f>_M9Oai!-8@wzUseczvGUzc=(}*aeGgXbSBS?pTX$p
zPx1cyzafPjNARTTC~3TPmVX|a`#RXoax8w{|3I>*XC)qf=wW>KyMH3}_2c7bFgiMl
z_s4z{{M<@N)nikXl?9{jKauUL*dTjy_tBiUfu7(k$Kw4RI{=UG>chs3e~MzUK+e+_
zU!2F!jvd3V-}@COi0Gh_A4}}zUpO34Jg`CuGc0%t4O0MdBgk_1=sm$Z1)XHmrcFd6
z!1&p5jE;_C?8N&-JFc~$mkykz6{bqA_k~b=9HGopN)@?cJJO3Ze6wfBVF3!@c&5=
zYa_^KfZ9Crm%)n$S)B8O*OEFhpk^Vu{l)12Q+XXMHXa>#$@4^;Qdu0pP%4$`pZG*$
z)2yE{r21Jncej?R1gQGKy{|A8K;h{hP
z7z`;mAu`m28u+Up+43*m*cJ;5r1Hn1)DlpTLo`AGpW5O8hEG2EgoysRl(}LRQgJm9
z=uT^F#DgwEG{gn)i25QZGywOXkiZr7{Zbl!Tgo8Pn;J}&w*N^P;-~vjNTvXSrHZ~Z
zNVT+-b?84-O~aUFF$p~;y;x>3`TvH06BxPsr@UFaSC7Rd`S>Hyyat;nhm?J}Cq{iP
z0FX|ORN&w<)Pha&I|O7n5mB#|{C~q4fNnfOfSNB_MAYs7Rg~0%AAe9tMFEsQ67BvI
zhvFOV|4G3|u@GIq73e>XvLL-eeUO
zAf}7bs+VyzWgh^5bduC;Ao2I!l{S~xQc~blJ1S2U$CtDi5-$Tq($u5cAYxIi#Q)bW
zBz<3JAAdAyAHYRIi&O0qhnkW4e^G!!KRPjqSch+Iamn3BbM^sZ`c@r_)AIik$uc23
z5=1acjsJQ;9SjgnwZU>UwEVw>{&fT{Do1E+g!U{}`Ujfi0_aIqfUr0ns!i|PfBJN>
z(?8JEeEcS2i4f9VvAVtNm2jXKo>L{&=x1&O6%hf@1B&Y^t{xfLaHvH
zhImw)g#NXPsDer+i$!SmX$AkE1hF=PLhk}nw7C3-ACsZE`v5#tv#zpKssAV1X2fl|
zJ}6_qR`vf){2m}owRFHoqi+ANtap8U{Lu*1zYOG|VR1TCn|S%hm{KpI#?}ffurMw_
zI;m=llhc26y%^y{uqmoe3kcprbN2xj`r{AH*g%h|u0Q$*>V6xj0AX=;(|@48$Og&2
z{r8`wMdY69_W;UI!eUyS7X6E<5dD#CKtaJmts;02sm1{9?d^eUsldT!&;m{J@rT$i
zl_#s$iUA*mVzC&jd@Kbhm&*V=0Yq6UjW~iQdr&ATj90vjY%-bC*Lu6%XH?UfrqAXQtOmj4$+dA>yWwj4w-8X9y9
zpi-$Ms+ULsKK=AlImIG;D&ER}__6VroSaNlPy0urcsFqF^Ybta11ned;D*l5U_rv-
zbf`A};m2f{o14Sr^p@(0SCw0ES^;cz75G4<7XI4Gs?Cg%@7%
zuLlMO@Z59H`PWZA`6Qlw_F4bBudfgL_wVR`mx6z!|vU?{p+1OcjD=%
zpZ5FTv112@hK78_*tTsO1_uZI@{c_72nGfQ{OfJowqa;!$iLpPV+Te?Mgr~g{q5Sd
zD=?nNAAdYB-Y1@TBJe!@{r!RGed?*F@ci@7``1H5LwNDU7yavzkrBM~(o6pJo;`c;
z$}6w<*ZcPE!>g~p>R&(e%rkiHwb#%nBw7aY=x|?(S}Bf3ot8f&qsG+5vO|
zSOMUh?y&+u=La8ra4<>G$DcPk0F<>Y3jl%a1#XSXi^As3o3{YC0N^5kuK>&g@Rta6
zxd4&=hlYlHx-$%er2hj44*1tP^uK4%9{<{-e^0O;{d?f)-!Kep-MSS60|WjOrh!pa_qSbm?40XP$eAL$`6<4`a;F_XH%;eH5Lm#^bfm_a|$g
zzZToqoiECrBtfJLIC=8qL7=g;o#r`z&oGQk&~%!4QHbjT7;$JVy#@yd19T_Se}8|!
zPydEt`1J3!-P_w+``(wd;Qfx|Gpe4l!Stjkgdrnx{njhs_2|wJ?NCz&5xpc
z72QUy7Iem2&uSq7{gd+bTtFKnE?{hI>~Bk@QVB?ps?$0LuJ0{i+qSFs-h1y~Am{?B
zh;RY6doTcOVq)S#WKRjutldQss)=ibJRX-XUp|AHEJ+l&Oi%EZdzb*M!-o&QV_BA&
z_BL%)8*%%gSb#z+j1g{EV=K#
z`)+&i!3Q^Xc6N4(6ziQA3WdPEq-FSfUU{*cPTL;%iL-4xa4kmDjIqFLRFQ5e`i*Ln
z%BNDX->5bz7>2FlAPz109z>NTw
z!x8>+xC#I#STwjgIkE-Dr6zzm0J8wT1aQesXWaW1R~{wQmI)Ss3Vr$Q2;V;1$4zBkXxNGOVDs`xjCGLpU$EM+r_`;2pW}vkzVgECg3`{L(%E
zQJ@BQ6u5J$Y>w=KR|1a>0^9l`WI`30A;;nZlbq+skt0Wr96562$dMyQjvP61 -1 and restore_cursor:
self.cursor_pos = (row, col)
+ self.ensureCursorVisible()
+
# }}}
# Non-prompt Rendering {{{
@@ -185,16 +237,26 @@ class Console(QTextEdit):
self.formatter.render(self.tb_lexer.get_tokens(tb), self.cursor)
except:
prints(tb, end='')
+ self.ensureCursorVisible()
+ QCoreApplication.processEvents()
def show_output(self, raw):
+ def do_show():
+ try:
+ self.buf.append(raw)
+ self.formatter.render_raw(raw, self.cursor)
+ except:
+ import traceback
+ prints(traceback.format_exc())
+ prints(raw, end='')
+
if self.prompt_frame is not None:
- # At a prompt, so redirect output
- return prints(raw, end='')
- try:
- self.buf.append(raw)
- self.formatter.render_raw(raw, self.cursor)
- except:
- prints(raw, end='')
+ with Prepender(self):
+ do_show()
+ else:
+ do_show()
+ self.ensureCursorVisible()
+ QCoreApplication.processEvents()
# }}}
@@ -203,16 +265,11 @@ class Console(QTextEdit):
def keyPressEvent(self, ev):
text = unicode(ev.text())
key = ev.key()
- if key in (Qt.Key_Enter, Qt.Key_Return):
- self.enter_pressed()
- elif key == Qt.Key_Home:
- self.home_pressed()
- elif key == Qt.Key_End:
- self.end_pressed()
- elif key == Qt.Key_Left:
- self.left_pressed()
- elif key == Qt.Key_Right:
- self.right_pressed()
+ action = self.key_dispatcher.get(key, None)
+ if callable(action):
+ action()
+ elif key in (Qt.Key_Escape,):
+ QTextEdit.keyPressEvent(self, ev)
elif text:
self.text_typed(text)
else:
@@ -230,6 +287,7 @@ class Console(QTextEdit):
c.movePosition(c.Up)
c.movePosition(c.EndOfLine)
self.setTextCursor(c)
+ self.ensureCursorVisible()
def right_pressed(self):
lineno, pos = self.cursor_pos
@@ -242,6 +300,7 @@ class Console(QTextEdit):
elif lineno < len(cp)-1:
c.movePosition(c.NextCharacter, n=1+self.prompt_len)
self.setTextCursor(c)
+ self.ensureCursorVisible()
def home_pressed(self):
if self.prompt_frame is not None:
@@ -249,12 +308,14 @@ class Console(QTextEdit):
c.movePosition(c.StartOfLine)
c.movePosition(c.NextCharacter, n=self.prompt_len)
self.setTextCursor(c)
+ self.ensureCursorVisible()
def end_pressed(self):
if self.prompt_frame is not None:
c = self.cursor
c.movePosition(c.EndOfLine)
self.setTextCursor(c)
+ self.ensureCursorVisible()
def enter_pressed(self):
if self.prompt_frame is None:
@@ -267,7 +328,13 @@ class Console(QTextEdit):
self.prompt_frame = None
oldbuf = self.buf
self.buf = []
- ret = self.interpreter.runsource('\n'.join(cp))
+ self.running.emit()
+ try:
+ ret = self.interpreter.runsource('\n'.join(cp))
+ except SystemExit:
+ ret = False
+ self.show_output('Raising SystemExit not allowed\n')
+ self.running_done.emit()
if ret: # Incomplete command
self.buf = oldbuf
self.prompt_frame = old_pf
@@ -275,7 +342,13 @@ class Console(QTextEdit):
c.insertBlock()
self.setTextCursor(c)
else: # Command completed
- old_pf.setFrameFormat(QTextFrameFormat())
+ try:
+ old_pf.setFrameFormat(QTextFrameFormat())
+ except RuntimeError:
+ # Happens if enough lines of output that the old
+ # frame was deleted
+ pass
+
self.render_current_prompt()
def text_typed(self, text):
diff --git a/src/calibre/utils/pyconsole/formatter.py b/src/calibre/utils/pyconsole/formatter.py
index 7f99983ef6..9409007ec6 100644
--- a/src/calibre/utils/pyconsole/formatter.py
+++ b/src/calibre/utils/pyconsole/formatter.py
@@ -8,7 +8,7 @@ __docformat__ = 'restructuredtext en'
from PyQt4.Qt import QTextCharFormat, QFont, QBrush, QColor
from pygments.formatter import Formatter as PF
-from pygments.token import Token
+from pygments.token import Token, Generic
class Formatter(object):
@@ -22,11 +22,16 @@ class Formatter(object):
pf = PF(**options)
self.styles = {}
self.normal = self.base_fmt()
+ self.background_color = pf.style.background_color
+ self.color = 'black'
+
for ttype, ndef in pf.style:
fmt = self.base_fmt()
if ndef['color']:
fmt.setForeground(QBrush(QColor('#%s'%ndef['color'])))
fmt.setUnderlineColor(QColor('#%s'%ndef['color']))
+ if ttype == Generic.Output:
+ self.color = '#%s'%ndef['color']
if ndef['bold']:
fmt.setFontWeight(QFont.Bold)
if ndef['italic']:
@@ -40,6 +45,11 @@ class Formatter(object):
self.styles[ttype] = fmt
+ self.stylesheet = '''
+ QTextEdit { color: %s; background-color: %s }
+ '''%(self.color, self.background_color)
+
+
def base_fmt(self):
fmt = QTextCharFormat()
fmt.setFontFamily('monospace')
@@ -74,7 +84,7 @@ class Formatter(object):
def render_prompt(self, is_continuation, cursor):
pr = self.continuation if is_continuation else self.prompt
- fmt = self.styles[Token.Generic.Subheading]
+ fmt = self.styles[Generic.Prompt]
cursor.insertText(pr, fmt)
diff --git a/src/calibre/utils/pyconsole/main.py b/src/calibre/utils/pyconsole/main.py
index af99ec66bb..f098ce2ee2 100644
--- a/src/calibre/utils/pyconsole/main.py
+++ b/src/calibre/utils/pyconsole/main.py
@@ -6,19 +6,31 @@ __copyright__ = '2010, Kovid Goyal '
__docformat__ = 'restructuredtext en'
__version__ = '0.1.0'
-from PyQt4.Qt import QMainWindow, QToolBar, QStatusBar, QLabel, QFont, Qt, \
- QApplication
+from functools import partial
+
+from PyQt4.Qt import QDialog, QToolBar, QStatusBar, QLabel, QFont, Qt, \
+ QApplication, QIcon, QVBoxLayout
from calibre.constants import __appname__, __version__
from calibre.utils.pyconsole.console import Console
-class MainWindow(QMainWindow):
+class MainWindow(QDialog):
- def __init__(self, default_status_msg):
+ def __init__(self,
+ default_status_msg=_('Welcome to') + ' ' + __appname__+' console',
+ parent=None):
- QMainWindow.__init__(self)
+ QDialog.__init__(self, parent)
+ self.l = QVBoxLayout()
+ self.setLayout(self.l)
- self.resize(600, 700)
+ self.resize(800, 600)
+
+ # Setup tool bar {{{
+ self.tool_bar = QToolBar(self)
+ self.tool_bar.setToolButtonStyle(Qt.ToolButtonTextOnly)
+ self.l.addWidget(self.tool_bar)
+ # }}}
# Setup status bar {{{
self.status_bar = QStatusBar(self)
@@ -28,25 +40,23 @@ class MainWindow(QMainWindow):
self.status_bar._font.setBold(True)
self.status_bar.defmsg.setFont(self.status_bar._font)
self.status_bar.addWidget(self.status_bar.defmsg)
- self.setStatusBar(self.status_bar)
# }}}
- # Setup tool bar {{{
- self.tool_bar = QToolBar(self)
- self.addToolBar(Qt.BottomToolBarArea, self.tool_bar)
- self.tool_bar.setToolButtonStyle(Qt.ToolButtonTextOnly)
- # }}}
-
- self.editor = Console(parent=self)
- self.setCentralWidget(self.editor)
-
+ self.console = Console(parent=self)
+ self.console.running.connect(partial(self.status_bar.showMessage,
+ _('Code is running')))
+ self.console.running_done.connect(self.status_bar.clearMessage)
+ self.l.addWidget(self.console)
+ self.l.addWidget(self.status_bar)
+ self.setWindowTitle(__appname__ + ' console')
+ self.setWindowIcon(QIcon(I('console.png')))
def main():
QApplication.setApplicationName(__appname__+' console')
QApplication.setOrganizationName('Kovid Goyal')
app = QApplication([])
- m = MainWindow(_('Welcome to') + ' ' + __appname__+' console')
+ m = MainWindow()
m.show()
app.exec_()
From 3fff4da652dd242fbdeb52c8d5676043c02df7c4 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Mon, 20 Sep 2010 21:32:29 -0600
Subject: [PATCH 066/207] Infrastructure changes to launch pyconsole
interpreter process
---
src/calibre/utils/ipc/launch.py | 21 ++++++++++++---------
src/calibre/utils/ipc/worker.py | 4 ++++
src/calibre/utils/pyconsole/console.py | 1 +
3 files changed, 17 insertions(+), 9 deletions(-)
diff --git a/src/calibre/utils/ipc/launch.py b/src/calibre/utils/ipc/launch.py
index 0de81ed644..aa93469119 100644
--- a/src/calibre/utils/ipc/launch.py
+++ b/src/calibre/utils/ipc/launch.py
@@ -22,13 +22,15 @@ class Worker(object):
have the environment variable :envvar:`CALIBRE_WORKER` set.
Useful attributes: ``is_alive``, ``returncode``
- usefule methods: ``kill``
+ Useful methods: ``kill``
To launch child simply call the Worker object. By default, the child's
output is redirected to an on disk file, the path to which is returned by
the call.
'''
+ exe_name = 'calibre-parallel'
+
@property
def osx_interpreter(self):
exe = os.path.basename(sys.executable)
@@ -41,32 +43,33 @@ class Worker(object):
@property
def executable(self):
+ e = self.exe_name
if iswindows:
return os.path.join(os.path.dirname(sys.executable),
- 'calibre-parallel.exe' if isfrozen else \
- 'Scripts\\calibre-parallel.exe')
+ e+'.exe' if isfrozen else \
+ 'Scripts\\%s.exe'%e)
if isnewosx:
- return os.path.join(sys.console_binaries_path, 'calibre-parallel')
+ return os.path.join(sys.console_binaries_path, e)
if isosx:
- if not isfrozen: return 'calibre-parallel'
+ if not isfrozen: return e
contents = os.path.join(self.osx_contents_dir,
'console.app', 'Contents')
return os.path.join(contents, 'MacOS', self.osx_interpreter)
if isfrozen:
- return os.path.join(getattr(sys, 'frozen_path'), 'calibre-parallel')
+ return os.path.join(getattr(sys, 'frozen_path'), e)
- c = os.path.join(sys.executables_location, 'calibre-parallel')
+ c = os.path.join(sys.executables_location, e)
if os.access(c, os.X_OK):
return c
- return 'calibre-parallel'
+ return e
@property
def gui_executable(self):
if isnewosx:
- return os.path.join(sys.binaries_path, 'calibre-parallel')
+ return os.path.join(sys.binaries_path, self.exe_name)
if isfrozen and isosx:
return os.path.join(self.osx_contents_dir,
diff --git a/src/calibre/utils/ipc/worker.py b/src/calibre/utils/ipc/worker.py
index 73233840fe..b7510426aa 100644
--- a/src/calibre/utils/ipc/worker.py
+++ b/src/calibre/utils/ipc/worker.py
@@ -80,8 +80,12 @@ def main():
if isosx and 'CALIBRE_WORKER_ADDRESS' not in os.environ:
# On some OS X computers launchd apparently tries to
# launch the last run process from the bundle
+ # so launch the gui as usual
from calibre.gui2.main import main as gui_main
return gui_main(['calibre'])
+ if 'CALIBRE_LAUNCH_INTERPRETER' in os.environ:
+ from calibre.utils.pyconsole.interpreter import main
+ return main()
address = cPickle.loads(unhexlify(os.environ['CALIBRE_WORKER_ADDRESS']))
key = unhexlify(os.environ['CALIBRE_WORKER_KEY'])
resultf = unhexlify(os.environ['CALIBRE_WORKER_RESULT'])
diff --git a/src/calibre/utils/pyconsole/console.py b/src/calibre/utils/pyconsole/console.py
index 251e8424a0..f741562f03 100644
--- a/src/calibre/utils/pyconsole/console.py
+++ b/src/calibre/utils/pyconsole/console.py
@@ -47,6 +47,7 @@ class Prepender(object): # {{{
self.console.cursor_pos = self.opos
# }}}
+
class Console(QTextEdit):
running = pyqtSignal()
From ff73865d9e75d5d4e44eb584961209654d7239e9 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Tue, 21 Sep 2010 14:13:03 +0100
Subject: [PATCH 067/207] Prevent cross-thread lock errors by having the cover
cache get the image on the GUI thread.
---
src/calibre/gui2/__init__.py | 30 +++++++++++++++++++++++++++++-
src/calibre/gui2/library/models.py | 4 ++--
src/calibre/library/caches.py | 7 +++++--
3 files changed, 36 insertions(+), 5 deletions(-)
diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py
index e58dce5559..ba32c09e06 100644
--- a/src/calibre/gui2/__init__.py
+++ b/src/calibre/gui2/__init__.py
@@ -1,7 +1,7 @@
__license__ = 'GPL v3'
__copyright__ = '2008, Kovid Goyal '
""" The GUI """
-import os, sys
+import os, sys, Queue
from threading import RLock
from PyQt4.Qt import QVariant, QFileInfo, QObject, SIGNAL, QBuffer, Qt, \
@@ -296,6 +296,34 @@ class Dispatcher(QObject):
def dispatch(self, args, kwargs):
self.func(*args, **kwargs)
+class FunctionDispatcher(QObject):
+ '''
+ Convenience class to use Qt signals with arbitrary python functions.
+ By default, ensures that a function call always happens in the
+ thread this Dispatcher was created in.
+ '''
+ dispatch_signal = pyqtSignal(object, object, object)
+
+ def __init__(self, func, queued=True, parent=None):
+ QObject.__init__(self, parent)
+ self.func = func
+ typ = Qt.QueuedConnection
+ if not queued:
+ typ = Qt.AutoConnection if queued is None else Qt.DirectConnection
+ self.dispatch_signal.connect(self.dispatch, type=typ)
+
+ def __call__(self, *args, **kwargs):
+ q = Queue.Queue()
+ self.dispatch_signal.emit(q, args, kwargs)
+ return q.get()
+
+ def dispatch(self, q, args, kwargs):
+ try:
+ res = self.func(*args, **kwargs)
+ except:
+ res = None
+ q.put(res)
+
class GetMetadata(QObject):
'''
Convenience class to ensure that metadata readers are used only in the
diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index 6941869e44..4b1e974b12 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -12,7 +12,7 @@ from operator import attrgetter
from PyQt4.Qt import QAbstractTableModel, Qt, pyqtSignal, QIcon, QImage, \
QModelIndex, QVariant, QDate
-from calibre.gui2 import NONE, config, UNDEFINED_QDATE
+from calibre.gui2 import NONE, config, UNDEFINED_QDATE, FunctionDispatcher
from calibre.utils.pyparsing import ParseException
from calibre.ebooks.metadata import fmt_sidx, authors_to_string, string_to_authors
from calibre.ptempfile import PersistentTemporaryFile
@@ -151,7 +151,7 @@ class BooksModel(QAbstractTableModel): # {{{
self.database_changed.emit(db)
if self.cover_cache is not None:
self.cover_cache.stop()
- self.cover_cache = CoverCache(db)
+ self.cover_cache = CoverCache(db, FunctionDispatcher(self.db.cover))
self.cover_cache.start()
def refresh_cover(event, ids):
if event == 'cover' and self.cover_cache is not None:
diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index 5f7fbdccc9..573c1f5797 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -23,10 +23,11 @@ from calibre import fit_image
class CoverCache(Thread):
- def __init__(self, db):
+ def __init__(self, db, cover_func):
Thread.__init__(self)
self.daemon = True
self.db = db
+ self.cover_func = cover_func
self.load_queue = Queue()
self.keep_running = True
self.cache = {}
@@ -37,7 +38,9 @@ class CoverCache(Thread):
self.keep_running = False
def _image_for_id(self, id_):
- img = self.db.cover(id_, index_is_id=True, as_image=True)
+ import time
+ time.sleep(0.050) # Limit 20/second to not overwhelm the GUI
+ img = self.cover_func(id_, index_is_id=True, as_image=True)
if img is None:
img = QImage()
if not img.isNull():
From be2210f928d0023ae1f752aeea3fb51f84132e8d Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Tue, 21 Sep 2010 15:26:02 +0100
Subject: [PATCH 068/207] Add 'start series renumbering from N' to bulk edit.
---
src/calibre/gui2/dialogs/metadata_bulk.py | 25 +++++++-
src/calibre/gui2/dialogs/metadata_bulk.ui | 69 +++++++++++++++++++----
2 files changed, 79 insertions(+), 15 deletions(-)
diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py
index 7122fe14fa..8a692d94d5 100644
--- a/src/calibre/gui2/dialogs/metadata_bulk.py
+++ b/src/calibre/gui2/dialogs/metadata_bulk.py
@@ -31,7 +31,8 @@ class Worker(Thread):
def doit(self):
remove, add, au, aus, do_aus, rating, pub, do_series, \
do_autonumber, do_remove_format, remove_format, do_swap_ta, \
- do_remove_conv, do_auto_author, series = self.args
+ do_remove_conv, do_auto_author, series, do_series_restart, \
+ series_start_value = self.args
# first loop: do author and title. These will commit at the end of each
# operation, because each operation modifies the file system. We want to
@@ -69,7 +70,11 @@ class Worker(Thread):
self.db.set_publisher(id, pub, notify=False, commit=False)
if do_series:
- next = self.db.get_next_series_num_for(series)
+ if do_series_restart:
+ next = series_start_value
+ series_start_value += 1
+ else:
+ next = self.db.get_next_series_num_for(series)
self.db.set_series(id, series, notify=False, commit=False)
num = next if do_autonumber and series else 1.0
self.db.set_series_index(id, num, notify=False, commit=False)
@@ -163,6 +168,7 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
self.series.currentIndexChanged[int].connect(self.series_changed)
self.series.editTextChanged.connect(self.series_changed)
self.tag_editor_button.clicked.connect(self.tag_editor)
+ self.autonumber_series.stateChanged[int].connect(self.auto_number_changed)
if len(db.custom_column_label_map) == 0:
self.central_widget.removeTab(1)
@@ -538,6 +544,16 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
self.tags.update_tags_cache(self.db.all_tags())
self.remove_tags.update_tags_cache(self.db.all_tags())
+ def auto_number_changed(self, state):
+ if state:
+ self.series_numbering_restarts.setEnabled(True)
+ self.series_start_number.setEnabled(True)
+ else:
+ self.series_numbering_restarts.setEnabled(False)
+ self.series_numbering_restarts.setChecked(False)
+ self.series_start_number.setEnabled(False)
+ self.series_start_number.setValue(1)
+
def accept(self):
if len(self.ids) < 1:
return QDialog.accept(self)
@@ -566,6 +582,8 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
do_series = self.write_series
series = unicode(self.series.currentText()).strip()
do_autonumber = self.autonumber_series.isChecked()
+ do_series_restart = self.series_numbering_restarts.isChecked()
+ series_start_value = self.series_start_number.value()
do_remove_format = self.remove_format.currentIndex() > -1
remove_format = unicode(self.remove_format.currentText())
do_swap_ta = self.swap_title_and_author.isChecked()
@@ -574,7 +592,8 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
args = (remove, add, au, aus, do_aus, rating, pub, do_series,
do_autonumber, do_remove_format, remove_format, do_swap_ta,
- do_remove_conv, do_auto_author, series)
+ do_remove_conv, do_auto_author, series, do_series_restart,
+ series_start_value)
bb = BlockingBusy(_('Applying changes to %d books. This may take a while.')
%len(self.ids), parent=self)
diff --git a/src/calibre/gui2/dialogs/metadata_bulk.ui b/src/calibre/gui2/dialogs/metadata_bulk.ui
index f28f3fb57c..10e22c5df9 100644
--- a/src/calibre/gui2/dialogs/metadata_bulk.ui
+++ b/src/calibre/gui2/dialogs/metadata_bulk.ui
@@ -270,18 +270,63 @@
-
-
-
- Selected books will be automatically numbered,
-in the order you selected them.
-So if you selected Book A and then Book B,
+
+
+
+
+
+ If not checked, the series number for the books will be set to 1.
+If checked, selected books will be automatically numbered, in the order
+you selected them. So if you selected Book A and then Book B,
Book A will have series number 1 and Book B series number 2.
-
-
- Automatically number books in this series
-
-
+
+
+ Automatically number books in this series
+
+
+
+
+
+
+ false
+
+
+ Series will normally be renumbered from the highest number in the database
+for that series. Checking this box will tell calibre to start numbering
+from the value in the box
+
+
+ Force numbers to start with
+
+
+
+
+
+
+ false
+
+
+ 1
+
+
+ 1
+
+
+
+
+
+
+ Qt::Horizontal
+
+
+
+ 20
+ 10
+
+
+
+
+
@@ -599,7 +644,7 @@ nothing should be put between the original text and the inserted text
20
- 40
+ 0
From 8a3aa64776aaf11c9909a6518c4f0f90f55a2946 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Tue, 21 Sep 2010 15:53:54 +0100
Subject: [PATCH 069/207] Changed sort to use
field_metadata.search_term_to_field_key.
In the process, refactored field_metadata and LibraryDatabase2 to use the same method names for the same function (in more cases).
---
src/calibre/library/caches.py | 11 ++++-------
src/calibre/library/database2.py | 4 ++--
src/calibre/library/field_metadata.py | 4 ++--
3 files changed, 8 insertions(+), 11 deletions(-)
diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index 573c1f5797..d310a0e6fe 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -334,7 +334,7 @@ class ResultCache(SearchQueryParser):
if query and query.strip():
# get metadata key associated with the search term. Eliminates
# dealing with plurals and other aliases
- location = self.field_metadata.search_term_to_key(location.lower().strip())
+ location = self.field_metadata.search_term_to_field_key(location.lower().strip())
if isinstance(location, list):
if allow_recursion:
for loc in location:
@@ -610,12 +610,9 @@ class ResultCache(SearchQueryParser):
# Sorting functions {{{
def sanitize_sort_field_name(self, field):
- field = field.lower().strip()
- if field not in self.field_metadata.iterkeys():
- if field in ('author', 'tag', 'comment'):
- field += 's'
- if field == 'date': field = 'timestamp'
- elif field == 'title': field = 'sort'
+ field = self.field_metadata.search_term_to_field_key(field.lower().strip())
+ # translate some fields to their hidden equivalent
+ if field == 'title': field = 'sort'
elif field == 'authors': field = 'author_sort'
return field
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index c1ada94a84..77e3afc8a3 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -552,10 +552,10 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
return self.field_metadata.sortable_field_keys()
def searchable_fields(self):
- return self.field_metadata.searchable_field_keys()
+ return self.field_metadata.searchable_fields()
def search_term_to_field_key(self, term):
- return self.field_metadata.search_term_to_key(term)
+ return self.field_metadata.search_term_to_field_key(term)
def metadata_for_field(self, key):
return self.field_metadata[key]
diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py
index a8031e5172..bac423f46d 100644
--- a/src/calibre/library/field_metadata.py
+++ b/src/calibre/library/field_metadata.py
@@ -501,12 +501,12 @@ class FieldMetadata(dict):
raise ValueError('Attempt to add duplicate search term "%s"'%t)
self._search_term_map[t] = key
- def search_term_to_key(self, term):
+ def search_term_to_field_key(self, term):
if term in self._search_term_map:
return self._search_term_map[term]
return term
- def searchable_field_keys(self):
+ def searchable_fields(self):
return [k for k in self._tb_cats.keys()
if self._tb_cats[k]['kind']=='field' and
len(self._tb_cats[k]['search_terms']) > 0]
From ba6f2f0c5e6cd9943827cda2ccc4a59d32fdbb8b Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Tue, 21 Sep 2010 18:50:21 +0100
Subject: [PATCH 070/207] Take out commit= parameter on set_authors and
set_title. Change other code where necessary
---
src/calibre/gui2/dialogs/metadata_bulk.py | 5 ++++-
src/calibre/library/database2.py | 19 ++++++++-----------
2 files changed, 12 insertions(+), 12 deletions(-)
diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py
index 8a692d94d5..18d00191cc 100644
--- a/src/calibre/gui2/dialogs/metadata_bulk.py
+++ b/src/calibre/gui2/dialogs/metadata_bulk.py
@@ -484,7 +484,10 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
setter = self.db.set_comment
else:
setter = getattr(self.db, 'set_'+dest)
- setter(id, val, notify=False, commit=False)
+ if dest in ['title', 'authors']:
+ setter(id, val, notify=False)
+ else:
+ setter(id, val, notify=False, commit=False)
self.db.commit()
dynamic['s_r_search_mode'] = self.search_mode.currentIndex()
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index 77e3afc8a3..1fdacfc09f 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -407,7 +407,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
path = path.lower()
return path
- def set_path(self, index, index_is_id=False, commit=True):
+ def set_path(self, index, index_is_id=False):
'''
Set the path to the directory containing this books files based on its
current title and author. If there was a previous directory, its contents
@@ -447,8 +447,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
self.add_format(id, format, stream, index_is_id=True,
path=tpath, notify=False)
self.conn.execute('UPDATE books SET path=? WHERE id=?', (path, id))
- if commit:
- self.conn.commit()
+ self.conn.commit()
self.data.set(id, self.FIELD_MAP['path'], path, row_is_id=True)
# Delete not needed directories
if current_path and os.path.exists(spath):
@@ -1212,7 +1211,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
result.append(r)
return ' & '.join(result).replace('|', ',')
- def set_authors(self, id, authors, notify=True, commit=True):
+ def set_authors(self, id, authors, notify=True):
'''
`authors`: A list of authors.
'''
@@ -1240,17 +1239,16 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
ss = self.author_sort_from_book(id, index_is_id=True)
self.conn.execute('UPDATE books SET author_sort=? WHERE id=?',
(ss, id))
- if commit:
- self.conn.commit()
+ self.conn.commit()
self.data.set(id, self.FIELD_MAP['authors'],
','.join([a.replace(',', '|') for a in authors]),
row_is_id=True)
self.data.set(id, self.FIELD_MAP['author_sort'], ss, row_is_id=True)
- self.set_path(id, index_is_id=True, commit=commit)
+ self.set_path(id, index_is_id=True)
if notify:
self.notify('metadata', [id])
- def set_title(self, id, title, notify=True, commit=True):
+ def set_title(self, id, title, notify=True):
if not title:
return
if not isinstance(title, unicode):
@@ -1261,9 +1259,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
self.data.set(id, self.FIELD_MAP['sort'], title_sort(title), row_is_id=True)
else:
self.data.set(id, self.FIELD_MAP['sort'], title, row_is_id=True)
- self.set_path(id, index_is_id=True, commit=commit)
- if commit:
- self.conn.commit()
+ self.set_path(id, index_is_id=True)
+ self.conn.commit()
if notify:
self.notify('metadata', [id])
From a62ad5f70cbae026d64d183117a3ec02de59444c Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Tue, 21 Sep 2010 20:20:34 +0100
Subject: [PATCH 071/207] Use one queue object in FunctionDispatch.
Theory: the producer (Qt GUI cover function) exists only once per instance of FunctionDispatcher. This follows from the fact that the dispatcher instance is created on the recipient thread. The consumer (the cover cache) could in theory be multiple threads (but it isn't). Because the items produced by the producer are not equivalent, we need to ensure that the order of items put in the queue by the producer is equal to the order of the requests. To guarantee this order, regardless of the number of consumer threads, we ensure that only one request to the producer can be outstanding.
---
src/calibre/gui2/__init__.py | 11 +++++++----
1 file changed, 7 insertions(+), 4 deletions(-)
diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py
index 66e199b8a0..8cfcc17eba 100644
--- a/src/calibre/gui2/__init__.py
+++ b/src/calibre/gui2/__init__.py
@@ -1,7 +1,7 @@
__license__ = 'GPL v3'
__copyright__ = '2008, Kovid Goyal '
""" The GUI """
-import os, sys, Queue
+import os, sys, Queue, threading
from threading import RLock
from PyQt4.Qt import QVariant, QFileInfo, QObject, SIGNAL, QBuffer, Qt, \
@@ -311,11 +311,14 @@ class FunctionDispatcher(QObject):
if not queued:
typ = Qt.AutoConnection if queued is None else Qt.DirectConnection
self.dispatch_signal.connect(self.dispatch, type=typ)
+ self.q = Queue.Queue()
+ self.lock = threading.Lock()
def __call__(self, *args, **kwargs):
- q = Queue.Queue()
- self.dispatch_signal.emit(q, args, kwargs)
- return q.get()
+ with self.lock:
+ self.dispatch_signal.emit(self.q, args, kwargs)
+ res = self.q.get()
+ return res
def dispatch(self, q, args, kwargs):
try:
From 06173bccebb0d8b3df497d0d0df030c1710aa80b Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Tue, 21 Sep 2010 13:45:34 -0600
Subject: [PATCH 072/207] Second beta
---
src/calibre/constants.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/calibre/constants.py b/src/calibre/constants.py
index 334406e01b..91c114359c 100644
--- a/src/calibre/constants.py
+++ b/src/calibre/constants.py
@@ -2,7 +2,7 @@ __license__ = 'GPL v3'
__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
__docformat__ = 'restructuredtext en'
__appname__ = 'calibre'
-__version__ = '0.7.900'
+__version__ = '0.7.901'
__author__ = "Kovid Goyal "
import re
From 63f02aa91beaa330a45d8c572cb6e092832b6cb0 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Tue, 21 Sep 2010 15:19:14 -0600
Subject: [PATCH 073/207] Fix regression in get_metadata for books with no
formats
---
src/calibre/library/database2.py | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index 01d46083b2..3e7b932808 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -590,7 +590,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
mi.pubdate = self.pubdate(idx, index_is_id=index_is_id)
mi.uuid = self.uuid(idx, index_is_id=index_is_id)
mi.title_sort = self.title_sort(idx, index_is_id=index_is_id)
- mi.formats = self.formats(idx, index_is_id=index_is_id).split(',')
+ mi.formats = self.formats(idx, index_is_id=index_is_id)
+ if hasattr(mi.formats, 'split'):
+ mi.formats = mi.formats.split(',')
+ else:
+ mi.formats = None
tags = self.tags(idx, index_is_id=index_is_id)
if tags:
mi.tags = [i.strip() for i in tags.split(',')]
From d225fd9ff5c08083d79238f489cd72abbef8b61d Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Tue, 21 Sep 2010 15:22:00 -0600
Subject: [PATCH 074/207] Allow --reinitialize-db to use an SQL dump from
elsewhere
---
src/calibre/debug.py | 47 ++++++++++++++++++++++++++++----------------
1 file changed, 30 insertions(+), 17 deletions(-)
diff --git a/src/calibre/debug.py b/src/calibre/debug.py
index 8a2097ddd1..8cc125b118 100644
--- a/src/calibre/debug.py
+++ b/src/calibre/debug.py
@@ -36,13 +36,17 @@ Run an embedded python interpreter.
'plugin code.')
parser.add_option('--reinitialize-db', default=None,
help='Re-initialize the sqlite calibre database at the '
- 'specified path. Useful to recover from db corruption.')
+ 'specified path. Useful to recover from db corruption.'
+ ' You can also specify the path to an SQL dump which '
+ 'will be used instead of trying to dump the database.'
+ ' This can be useful when dumping fails, but dumping '
+ 'with sqlite3 works.')
parser.add_option('-p', '--py-console', help='Run python console',
default=False, action='store_true')
return parser
-def reinit_db(dbpath, callback=None):
+def reinit_db(dbpath, callback=None, sql_dump=None):
if not os.path.exists(dbpath):
raise ValueError(dbpath + ' does not exist')
from calibre.library.sqlite import connect
@@ -52,26 +56,32 @@ def reinit_db(dbpath, callback=None):
uv = conn.get('PRAGMA user_version;', all=False)
conn.execute('PRAGMA writable_schema=ON')
conn.commit()
- sql_lines = conn.dump()
+ if sql_dump is None:
+ sql_lines = conn.dump()
+ else:
+ sql_lines = open(sql_dump, 'rb').read()
conn.close()
dest = dbpath + '.tmp'
try:
with closing(connect(dest, False)) as nconn:
nconn.execute('create temporary table temp_sequence(id INTEGER PRIMARY KEY AUTOINCREMENT)')
nconn.commit()
- if callable(callback):
- callback(len(sql_lines), True)
- for i, line in enumerate(sql_lines):
- try:
- nconn.execute(line)
- except:
- import traceback
- prints('SQL line %r failed with error:'%line)
- prints(traceback.format_exc())
- continue
- finally:
- if callable(callback):
- callback(i, False)
+ if sql_dump is None:
+ if callable(callback):
+ callback(len(sql_lines), True)
+ for i, line in enumerate(sql_lines):
+ try:
+ nconn.execute(line)
+ except:
+ import traceback
+ prints('SQL line %r failed with error:'%line)
+ prints(traceback.format_exc())
+ continue
+ finally:
+ if callable(callback):
+ callback(i, False)
+ else:
+ nconn.executescript(sql_lines)
nconn.execute('pragma user_version=%d'%int(uv))
nconn.commit()
os.remove(dbpath)
@@ -170,7 +180,10 @@ def main(args=sys.argv):
prints('CALIBRE_EXTENSIONS_PATH='+sys.extensions_location)
prints('CALIBRE_PYTHON_PATH='+os.pathsep.join(sys.path))
elif opts.reinitialize_db is not None:
- reinit_db(opts.reinitialize_db)
+ sql_dump = None
+ if len(args) > 1 and os.access(args[-1], os.R_OK):
+ sql_dump = args[-1]
+ reinit_db(opts.reinitialize_db, sql_dump=sql_dump)
else:
from calibre import ipython
ipython()
From 7f472f742ece9ada0736690c440dc2e9da30cc74 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Tue, 21 Sep 2010 21:05:23 -0600
Subject: [PATCH 075/207] Styling cleanups
---
src/calibre/utils/pyconsole/formatter.py | 23 ++++++++++++++++++-----
1 file changed, 18 insertions(+), 5 deletions(-)
diff --git a/src/calibre/utils/pyconsole/formatter.py b/src/calibre/utils/pyconsole/formatter.py
index 9409007ec6..6e7d982a82 100644
--- a/src/calibre/utils/pyconsole/formatter.py
+++ b/src/calibre/utils/pyconsole/formatter.py
@@ -8,18 +8,20 @@ __docformat__ = 'restructuredtext en'
from PyQt4.Qt import QTextCharFormat, QFont, QBrush, QColor
from pygments.formatter import Formatter as PF
-from pygments.token import Token, Generic
+from pygments.token import Token, Generic, string_to_tokentype
class Formatter(object):
- def __init__(self, prompt, continuation, **options):
+ def __init__(self, prompt, continuation, style='default'):
if len(prompt) != len(continuation):
raise ValueError('%r does not have the same length as %r' %
(prompt, continuation))
self.prompt, self.continuation = prompt, continuation
+ self.set_style(style)
- pf = PF(**options)
+ def set_style(self, style):
+ pf = PF(style=style)
self.styles = {}
self.normal = self.base_fmt()
self.background_color = pf.style.background_color
@@ -27,6 +29,7 @@ class Formatter(object):
for ttype, ndef in pf.style:
fmt = self.base_fmt()
+ fmt.setProperty(fmt.UserProperty, str(ttype))
if ndef['color']:
fmt.setForeground(QBrush(QColor('#%s'%ndef['color'])))
fmt.setUnderlineColor(QColor('#%s'%ndef['color']))
@@ -49,6 +52,14 @@ class Formatter(object):
QTextEdit { color: %s; background-color: %s }
'''%(self.color, self.background_color)
+ def get_fmt(self, token):
+ if type(token) != type(Token.Generic):
+ token = string_to_tokentype(token)
+ fmt = self.styles.get(token, None)
+ if fmt is None:
+ fmt = self.base_fmt()
+ fmt.setProperty(fmt.UserProperty, str(token))
+ return fmt
def base_fmt(self):
fmt = QTextCharFormat()
@@ -59,7 +70,7 @@ class Formatter(object):
cursor.insertText(raw, self.normal)
def render_syntax_error(self, tb, cursor):
- fmt = self.styles[Token.Error]
+ fmt = self.get_fmt(Token.Error)
cursor.insertText(tb, fmt)
def render(self, tokens, cursor):
@@ -84,7 +95,9 @@ class Formatter(object):
def render_prompt(self, is_continuation, cursor):
pr = self.continuation if is_continuation else self.prompt
- fmt = self.styles[Generic.Prompt]
+ fmt = self.get_fmt(Generic.Prompt)
+ if fmt is None:
+ fmt = self.base_fmt()
cursor.insertText(pr, fmt)
From 35dba964c6241d5600d6aeba4bfcf27d104ee8f9 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Tue, 21 Sep 2010 21:46:56 -0600
Subject: [PATCH 076/207] Theming support for console via right click menu
---
src/calibre/utils/pyconsole/__init__.py | 14 +++++---
src/calibre/utils/pyconsole/console.py | 44 +++++++++++++++++++++---
src/calibre/utils/pyconsole/formatter.py | 4 ---
3 files changed, 49 insertions(+), 13 deletions(-)
diff --git a/src/calibre/utils/pyconsole/__init__.py b/src/calibre/utils/pyconsole/__init__.py
index 0dfa9398e1..06a7011132 100644
--- a/src/calibre/utils/pyconsole/__init__.py
+++ b/src/calibre/utils/pyconsole/__init__.py
@@ -8,14 +8,18 @@ __docformat__ = 'restructuredtext en'
import sys
from calibre import prints as prints_
-from calibre.utils.config import Config, StringConfig
+from calibre.utils.config import Config, ConfigProxy
-def console_config(defaults=None):
- desc=_('Settings to control the calibre content server')
- c = Config('console', desc) if defaults is None else StringConfig(defaults, desc)
+def console_config():
+ desc='Settings to control the calibre console'
+ c = Config('console', desc)
- c.add_opt('--theme', default='default', help='The color theme')
+ c.add_opt('theme', default='default', help='The color theme')
+
+ return c
+
+prefs = ConfigProxy(console_config())
def prints(*args, **kwargs):
diff --git a/src/calibre/utils/pyconsole/console.py b/src/calibre/utils/pyconsole/console.py
index f741562f03..b0ecce0cb3 100644
--- a/src/calibre/utils/pyconsole/console.py
+++ b/src/calibre/utils/pyconsole/console.py
@@ -6,16 +6,18 @@ __copyright__ = '2010, Kovid Goyal '
__docformat__ = 'restructuredtext en'
import sys, textwrap, traceback, StringIO
+from functools import partial
from PyQt4.Qt import QTextEdit, Qt, QTextFrameFormat, pyqtSignal, \
- QCoreApplication
+ QCoreApplication, QColor, QPalette, QMenu, QActionGroup
from pygments.lexers import PythonLexer, PythonTracebackLexer
+from pygments.styles import get_all_styles
from calibre.constants import __appname__, __version__
from calibre.utils.pyconsole.formatter import Formatter
from calibre.utils.pyconsole.repl import Interpreter, DummyFile
-from calibre.utils.pyconsole import prints
+from calibre.utils.pyconsole import prints, prefs
from calibre.gui2 import error_dialog
class EditBlock(object): # {{{
@@ -47,6 +49,28 @@ class Prepender(object): # {{{
self.console.cursor_pos = self.opos
# }}}
+class ThemeMenu(QMenu):
+
+ def __init__(self, parent):
+ QMenu.__init__(self, _('Choose theme (needs restart)'))
+ parent.addMenu(self)
+ self.group = QActionGroup(self)
+ current = prefs['theme']
+ alls = list(sorted(get_all_styles()))
+ if current not in alls:
+ current = prefs['theme'] = 'default'
+ self.actions = []
+ for style in alls:
+ ac = self.group.addAction(style)
+ if current == style:
+ ac.setChecked(True)
+ self.actions.append(ac)
+ ac.triggered.connect(partial(self.set_theme, style))
+ self.addAction(ac)
+
+ def set_theme(self, style, *args):
+ prefs['theme'] = style
+
class Console(QTextEdit):
@@ -99,8 +123,16 @@ class Console(QTextEdit):
self.doc.setMaximumBlockCount(10000)
self.lexer = PythonLexer(ensurenl=False)
self.tb_lexer = PythonTracebackLexer()
- self.formatter = Formatter(prompt, continuation, style='default')
- self.setStyleSheet(self.formatter.stylesheet)
+
+ self.context_menu = cm = QMenu(self) # {{{
+ cm.theme = ThemeMenu(cm)
+ # }}}
+
+ self.formatter = Formatter(prompt, continuation, style=prefs['theme'])
+ p = QPalette()
+ p.setColor(p.Base, QColor(self.formatter.background_color))
+ p.setColor(p.Text, QColor(self.formatter.color))
+ self.setPalette(p)
self.key_dispatcher = { # {{{
Qt.Key_Enter : self.enter_pressed,
@@ -127,6 +159,10 @@ class Console(QTextEdit):
sys.excepthook = self.unhandled_exception
+ def contextMenuEvent(self, event):
+ self.context_menu.popup(event.globalPos())
+ event.accept()
+
# Prompt management {{{
diff --git a/src/calibre/utils/pyconsole/formatter.py b/src/calibre/utils/pyconsole/formatter.py
index 6e7d982a82..17360fecb3 100644
--- a/src/calibre/utils/pyconsole/formatter.py
+++ b/src/calibre/utils/pyconsole/formatter.py
@@ -48,10 +48,6 @@ class Formatter(object):
self.styles[ttype] = fmt
- self.stylesheet = '''
- QTextEdit { color: %s; background-color: %s }
- '''%(self.color, self.background_color)
-
def get_fmt(self, token):
if type(token) != type(Token.Generic):
token = string_to_tokentype(token)
From e7adf45c01b21a584158728de903837d79f82298 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Tue, 21 Sep 2010 22:21:36 -0600
Subject: [PATCH 077/207] Restart and context menu added to console
---
src/calibre/utils/pyconsole/__init__.py | 2 +-
src/calibre/utils/pyconsole/console.py | 5 +++--
src/calibre/utils/pyconsole/main.py | 23 ++++++++++++++++++-----
3 files changed, 22 insertions(+), 8 deletions(-)
diff --git a/src/calibre/utils/pyconsole/__init__.py b/src/calibre/utils/pyconsole/__init__.py
index 06a7011132..32eb926143 100644
--- a/src/calibre/utils/pyconsole/__init__.py
+++ b/src/calibre/utils/pyconsole/__init__.py
@@ -15,7 +15,7 @@ def console_config():
desc='Settings to control the calibre console'
c = Config('console', desc)
- c.add_opt('theme', default='default', help='The color theme')
+ c.add_opt('theme', default='native', help='The color theme')
return c
diff --git a/src/calibre/utils/pyconsole/console.py b/src/calibre/utils/pyconsole/console.py
index b0ecce0cb3..164cf4e2ca 100644
--- a/src/calibre/utils/pyconsole/console.py
+++ b/src/calibre/utils/pyconsole/console.py
@@ -49,7 +49,7 @@ class Prepender(object): # {{{
self.console.cursor_pos = self.opos
# }}}
-class ThemeMenu(QMenu):
+class ThemeMenu(QMenu): # {{{
def __init__(self, parent):
QMenu.__init__(self, _('Choose theme (needs restart)'))
@@ -62,6 +62,7 @@ class ThemeMenu(QMenu):
self.actions = []
for style in alls:
ac = self.group.addAction(style)
+ ac.setCheckable(True)
if current == style:
ac.setChecked(True)
self.actions.append(ac)
@@ -71,6 +72,7 @@ class ThemeMenu(QMenu):
def set_theme(self, style, *args):
prefs['theme'] = style
+# }}}
class Console(QTextEdit):
@@ -163,7 +165,6 @@ class Console(QTextEdit):
self.context_menu.popup(event.globalPos())
event.accept()
-
# Prompt management {{{
@dynamic_property
diff --git a/src/calibre/utils/pyconsole/main.py b/src/calibre/utils/pyconsole/main.py
index f098ce2ee2..a5a4b42266 100644
--- a/src/calibre/utils/pyconsole/main.py
+++ b/src/calibre/utils/pyconsole/main.py
@@ -9,7 +9,7 @@ __version__ = '0.1.0'
from functools import partial
from PyQt4.Qt import QDialog, QToolBar, QStatusBar, QLabel, QFont, Qt, \
- QApplication, QIcon, QVBoxLayout
+ QApplication, QIcon, QVBoxLayout, QAction
from calibre.constants import __appname__, __version__
from calibre.utils.pyconsole.console import Console
@@ -19,8 +19,9 @@ class MainWindow(QDialog):
def __init__(self,
default_status_msg=_('Welcome to') + ' ' + __appname__+' console',
parent=None):
-
QDialog.__init__(self, parent)
+
+ self.restart_requested = False
self.l = QVBoxLayout()
self.setLayout(self.l)
@@ -51,14 +52,26 @@ class MainWindow(QDialog):
self.setWindowTitle(__appname__ + ' console')
self.setWindowIcon(QIcon(I('console.png')))
+ self.restart_action = QAction(_('Restart'), self)
+ self.restart_action.setShortcut(_('Ctrl+R'))
+ self.addAction(self.restart_action)
+ self.restart_action.triggered.connect(self.restart)
+ self.console.context_menu.addAction(self.restart_action)
+
+ def restart(self):
+ self.restart_requested = True
+ self.reject()
def main():
QApplication.setApplicationName(__appname__+' console')
QApplication.setOrganizationName('Kovid Goyal')
app = QApplication([])
- m = MainWindow()
- m.show()
- app.exec_()
+ app
+ while True:
+ m = MainWindow()
+ m.exec_()
+ if not m.restart_requested:
+ break
if __name__ == '__main__':
From a41f481ee7d3a72b727c1f6b2d98ae10443cb0d2 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Tue, 21 Sep 2010 22:28:49 -0600
Subject: [PATCH 078/207] Backspace and delete now work in console
---
src/calibre/utils/pyconsole/console.py | 18 ++++++++++++++++++
1 file changed, 18 insertions(+)
diff --git a/src/calibre/utils/pyconsole/console.py b/src/calibre/utils/pyconsole/console.py
index 164cf4e2ca..aa0ff84d77 100644
--- a/src/calibre/utils/pyconsole/console.py
+++ b/src/calibre/utils/pyconsole/console.py
@@ -143,6 +143,8 @@ class Console(QTextEdit):
Qt.Key_End : self.end_pressed,
Qt.Key_Left : self.left_pressed,
Qt.Key_Right : self.right_pressed,
+ Qt.Key_Backspace : self.backspace_pressed,
+ Qt.Key_Delete : self.delete_pressed,
} # }}}
motd = textwrap.dedent('''\
@@ -327,6 +329,22 @@ class Console(QTextEdit):
self.setTextCursor(c)
self.ensureCursorVisible()
+ def backspace_pressed(self):
+ lineno, pos = self.cursor_pos
+ if lineno < 0: return
+ if pos > self.prompt_len:
+ self.cursor.deletePreviousChar()
+ elif lineno > 0:
+ c = self.cursor
+ c.movePosition(c.Up)
+ c.movePosition(c.EndOfLine)
+ self.setTextCursor(c)
+ self.ensureCursorVisible()
+
+ def delete_pressed(self):
+ self.cursor.deleteChar()
+ self.ensureCursorVisible()
+
def right_pressed(self):
lineno, pos = self.cursor_pos
if lineno < 0: return
From fdd839af7cafa964b26cc9481cc55a7d5b65b16a Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Tue, 21 Sep 2010 22:46:54 -0600
Subject: [PATCH 079/207] Ctrl+Home and Ctrl+End now work
---
src/calibre/utils/pyconsole/console.py | 23 ++++++++++++++++-------
src/calibre/utils/pyconsole/main.py | 2 +-
2 files changed, 17 insertions(+), 8 deletions(-)
diff --git a/src/calibre/utils/pyconsole/console.py b/src/calibre/utils/pyconsole/console.py
index aa0ff84d77..81169140cd 100644
--- a/src/calibre/utils/pyconsole/console.py
+++ b/src/calibre/utils/pyconsole/console.py
@@ -9,7 +9,7 @@ import sys, textwrap, traceback, StringIO
from functools import partial
from PyQt4.Qt import QTextEdit, Qt, QTextFrameFormat, pyqtSignal, \
- QCoreApplication, QColor, QPalette, QMenu, QActionGroup
+ QApplication, QColor, QPalette, QMenu, QActionGroup
from pygments.lexers import PythonLexer, PythonTracebackLexer
from pygments.styles import get_all_styles
@@ -278,7 +278,7 @@ class Console(QTextEdit):
except:
prints(tb, end='')
self.ensureCursorVisible()
- QCoreApplication.processEvents()
+ QApplication.processEvents()
def show_output(self, raw):
def do_show():
@@ -296,7 +296,7 @@ class Console(QTextEdit):
else:
do_show()
self.ensureCursorVisible()
- QCoreApplication.processEvents()
+ QApplication.processEvents()
# }}}
@@ -360,14 +360,23 @@ class Console(QTextEdit):
def home_pressed(self):
if self.prompt_frame is not None:
- c = self.cursor
- c.movePosition(c.StartOfLine)
- c.movePosition(c.NextCharacter, n=self.prompt_len)
- self.setTextCursor(c)
+ mods = QApplication.keyboardModifiers()
+ ctrl = bool(int(mods & Qt.CTRL))
+ if ctrl:
+ self.cursor_pos = (0, self.prompt_len)
+ else:
+ c = self.cursor
+ c.movePosition(c.StartOfLine)
+ c.movePosition(c.NextCharacter, n=self.prompt_len)
+ self.setTextCursor(c)
self.ensureCursorVisible()
def end_pressed(self):
if self.prompt_frame is not None:
+ mods = QApplication.keyboardModifiers()
+ ctrl = bool(int(mods & Qt.CTRL))
+ if ctrl:
+ self.cursor_pos = (len(list(self.prompt()))-1, self.prompt_len)
c = self.cursor
c.movePosition(c.EndOfLine)
self.setTextCursor(c)
diff --git a/src/calibre/utils/pyconsole/main.py b/src/calibre/utils/pyconsole/main.py
index a5a4b42266..a708ca1652 100644
--- a/src/calibre/utils/pyconsole/main.py
+++ b/src/calibre/utils/pyconsole/main.py
@@ -52,7 +52,7 @@ class MainWindow(QDialog):
self.setWindowTitle(__appname__ + ' console')
self.setWindowIcon(QIcon(I('console.png')))
- self.restart_action = QAction(_('Restart'), self)
+ self.restart_action = QAction(_('Restart console'), self)
self.restart_action.setShortcut(_('Ctrl+R'))
self.addAction(self.restart_action)
self.restart_action.triggered.connect(self.restart)
From 7893c01807e401ae6014485e6278ced3a633bb9c Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Wed, 22 Sep 2010 13:22:02 +0100
Subject: [PATCH 080/207] Three changes: 1) make get_metadata return an
unverified list of formats. Avoids a file system operation per format 2)
enhancement request #2845 3) permit composite fields as search/replace source
fields.
---
src/calibre/gui2/dialogs/metadata_bulk.py | 53 +++++++++++++++--------
src/calibre/gui2/dialogs/metadata_bulk.ui | 13 +++++-
src/calibre/library/database2.py | 10 +++--
3 files changed, 55 insertions(+), 21 deletions(-)
diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py
index 18d00191cc..fa3b1a9aa7 100644
--- a/src/calibre/gui2/dialogs/metadata_bulk.py
+++ b/src/calibre/gui2/dialogs/metadata_bulk.py
@@ -32,24 +32,30 @@ class Worker(Thread):
remove, add, au, aus, do_aus, rating, pub, do_series, \
do_autonumber, do_remove_format, remove_format, do_swap_ta, \
do_remove_conv, do_auto_author, series, do_series_restart, \
- series_start_value = self.args
+ series_start_value, do_title_case = self.args
# first loop: do author and title. These will commit at the end of each
# operation, because each operation modifies the file system. We want to
# try hard to keep the DB and the file system in sync, even in the face
# of exceptions or forced exits.
for id in self.ids:
+ title_set = False
if do_swap_ta:
title = self.db.title(id, index_is_id=True)
aum = self.db.authors(id, index_is_id=True)
if aum:
aum = [a.strip().replace('|', ',') for a in aum.split(',')]
new_title = authors_to_string(aum)
+ if do_title_case:
+ new_title = new_title.title()
self.db.set_title(id, new_title, notify=False)
+ title_set = True
if title:
new_authors = string_to_authors(title)
self.db.set_authors(id, new_authors, notify=False)
-
+ if do_title_case and not title_set:
+ title = self.db.title(id, index_is_id=True)
+ self.db.set_title(id, title.title(), notify=False)
if au:
self.db.set_authors(id, string_to_authors(au), notify=False)
@@ -182,19 +188,22 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
self.search_for.initialize('bulk_edit_search_for')
self.replace_with.initialize('bulk_edit_replace_with')
self.test_text.initialize('bulk_edit_test_test')
- fields = ['']
+ self.all_fields = ['']
+ self.writable_fields = ['']
fm = self.db.field_metadata
for f in fm:
if (f in ['author_sort'] or (
- fm[f]['datatype'] == 'text' or fm[f]['datatype'] == 'series')
+ fm[f]['datatype'] in ['text', 'series'])
and fm[f].get('search_terms', None)
and f not in ['formats', 'ondevice']):
- fields.append(f)
- fields.sort()
- self.search_field.addItems(fields)
- self.search_field.setMaxVisibleItems(min(len(fields), 20))
- self.destination_field.addItems(fields)
- self.destination_field.setMaxVisibleItems(min(len(fields), 20))
+ self.all_fields.append(f)
+ self.writable_fields.append(f)
+ if fm[f]['datatype'] == 'composite':
+ self.all_fields.append(f)
+ self.all_fields.sort()
+ self.writable_fields.sort()
+ self.search_field.setMaxVisibleItems(20)
+ self.destination_field.setMaxVisibleItems(20)
offset = 10
self.s_r_number_of_books = min(10, len(self.ids))
for i in range(1,self.s_r_number_of_books+1):
@@ -262,7 +271,7 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
self.replace_func.addItems(sorted(self.s_r_functions.keys()))
self.search_mode.currentIndexChanged[int].connect(self.s_r_search_mode_changed)
- self.search_field.currentIndexChanged[str].connect(self.s_r_search_field_changed)
+ self.search_field.currentIndexChanged[int].connect(self.s_r_search_field_changed)
self.destination_field.currentIndexChanged[str].connect(self.s_r_destination_field_changed)
self.replace_mode.currentIndexChanged[int].connect(self.s_r_paint_results)
@@ -293,15 +302,18 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
val = []
return val
- def s_r_search_field_changed(self, txt):
- txt = unicode(txt)
+ def s_r_search_field_changed(self, idx):
for i in range(0, self.s_r_number_of_books):
w = getattr(self, 'book_%d_text'%(i+1))
mi = self.db.get_metadata(self.ids[i], index_is_id=True)
src = unicode(self.search_field.currentText())
t = self.s_r_get_field(mi, src)
w.setText(''.join(t[0:1]))
- self.s_r_paint_results(None)
+
+ if self.search_mode.currentIndex() == 0:
+ self.destination_field.setCurrentIndex(idx)
+ else:
+ self.s_r_paint_results(None)
def s_r_destination_field_changed(self, txt):
txt = unicode(txt)
@@ -314,7 +326,11 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
self.s_r_paint_results(None)
def s_r_search_mode_changed(self, val):
+ self.search_field.clear()
+ self.destination_field.clear()
if val == 0:
+ self.search_field.addItems(self.writable_fields)
+ self.destination_field.addItems(self.writable_fields)
self.destination_field.setCurrentIndex(0)
self.destination_field.setVisible(False)
self.destination_field_label.setVisible(False)
@@ -324,6 +340,8 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
self.comma_separated.setVisible(False)
self.s_r_heading.setText('
'+self.main_heading + self.character_heading)
else:
+ self.search_field.addItems(self.all_fields)
+ self.destination_field.addItems(self.writable_fields)
self.destination_field.setVisible(True)
self.destination_field_label.setVisible(True)
self.replace_mode.setVisible(True)
@@ -367,6 +385,8 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
return ''
dest = unicode(self.destination_field.currentText())
if dest == '':
+ if self.db.metadata_for_field(src)['datatype'] == 'composite':
+ raise Exception(_('You must specify a destination when source is a composite field'))
dest = src
dest_mode = self.replace_mode.currentIndex()
@@ -433,8 +453,6 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
t = self.s_r_replace_mode_separator().join(t)
wr.setText(t)
except Exception as e:
- import traceback
- traceback.print_exc()
self.s_r_error = e
self.s_r_set_colors()
break
@@ -592,11 +610,12 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
do_swap_ta = self.swap_title_and_author.isChecked()
do_remove_conv = self.remove_conversion_settings.isChecked()
do_auto_author = self.auto_author_sort.isChecked()
+ do_title_case = self.change_title_to_title_case.isChecked()
args = (remove, add, au, aus, do_aus, rating, pub, do_series,
do_autonumber, do_remove_format, remove_format, do_swap_ta,
do_remove_conv, do_auto_author, series, do_series_restart,
- series_start_value)
+ series_start_value, do_title_case)
bb = BlockingBusy(_('Applying changes to %d books. This may take a while.')
%len(self.ids), parent=self)
diff --git a/src/calibre/gui2/dialogs/metadata_bulk.ui b/src/calibre/gui2/dialogs/metadata_bulk.ui
index 10e22c5df9..e03a59b7ea 100644
--- a/src/calibre/gui2/dialogs/metadata_bulk.ui
+++ b/src/calibre/gui2/dialogs/metadata_bulk.ui
@@ -270,6 +270,17 @@
+
+
+
+ Change title to title case
+
+
+ Force the title to be in title case. If both this and swap authors are checked,
+title and author are swapped before the title case is set
+
+
+
@@ -340,7 +351,7 @@ Future conversion of these books will use the default settings.
-
+ Qt::Vertical
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index 3e7b932808..c72e990738 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -535,7 +535,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
### The field-style interface. These use field keys.
def get_field(self, idx, key, default=None, index_is_id=False):
- mi = self.get_metadata(idx, index_is_id=index_is_id, get_cover=True)
+ mi = self.get_metadata(idx, index_is_id=index_is_id,
+ get_cover=key == 'cover')
return mi.get(key, default)
def standard_field_keys(self):
@@ -590,7 +591,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
mi.pubdate = self.pubdate(idx, index_is_id=index_is_id)
mi.uuid = self.uuid(idx, index_is_id=index_is_id)
mi.title_sort = self.title_sort(idx, index_is_id=index_is_id)
- mi.formats = self.formats(idx, index_is_id=index_is_id)
+ mi.formats = self.formats(idx, index_is_id=index_is_id,
+ verify_formats=False)
if hasattr(mi.formats, 'split'):
mi.formats = mi.formats.split(',')
else:
@@ -731,7 +733,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
return set([])
return set([f[0] for f in formats])
- def formats(self, index, index_is_id=False):
+ def formats(self, index, index_is_id=False, verify_formats=True):
''' Return available formats as a comma separated list or None if there are no available formats '''
id = index if index_is_id else self.id(index)
try:
@@ -739,6 +741,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
formats = map(lambda x:x[0], formats)
except:
return None
+ if not verify_formats:
+ return formats
ans = []
for format in formats:
if self.format_abspath(id, format, index_is_id=True) is not None:
From b1250a6db18366d599ea63e27658d7851d1e1f35 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Wed, 22 Sep 2010 16:43:28 +0100
Subject: [PATCH 081/207] Add prefix and postfix to template values. Syntax:
either '{key}' or '{txt1|key|txt2}'. In the second case, if val[key] is not
empty, then the result is 'txt1' + val[key] + txt2.
---
src/calibre/ebooks/metadata/book/base.py | 18 +++++++++++++-----
src/calibre/gui2/dialogs/metadata_bulk.py | 9 ---------
src/calibre/library/save_to_disk.py | 17 ++++++++++++++++-
3 files changed, 29 insertions(+), 15 deletions(-)
diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index 3d6d6b1bb8..d5a86264bf 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -14,8 +14,8 @@ from calibre.ebooks.metadata.book import STANDARD_METADATA_FIELDS
from calibre.ebooks.metadata.book import TOP_LEVEL_CLASSIFIERS
from calibre.ebooks.metadata.book import ALL_METADATA_FIELDS
from calibre.library.field_metadata import FieldMetadata
-from calibre.utils.date import isoformat, format_date
+from calibre.utils.date import isoformat, format_date
NULL_VALUES = {
@@ -38,10 +38,17 @@ class SafeFormat(string.Formatter):
Provides a format function that substitutes '' for any missing value
'''
def get_value(self, key, args, mi):
- ign, v = mi.format_field(key, series_with_index=False)
- if v is None:
- return ''
- return v
+ from calibre.library.save_to_disk import explode_string_template_value
+ try:
+ prefix, key, suffix = explode_string_template_value(key)
+ ign, v = mi.format_field(key, series_with_index=False)
+ if v is None:
+ return ''
+ if v is '':
+ return ''
+ return '%s%s%s'%(prefix, v, suffix)
+ except:
+ return key
composite_formatter = SafeFormat()
compress_spaces = re.compile(r'\s+')
@@ -50,6 +57,7 @@ def format_composite(x, mi):
try:
ans = composite_formatter.vformat(x, [], mi).strip()
except:
+ traceback.print_exc()
ans = x
return compress_spaces.sub(' ', ans)
diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py
index fa3b1a9aa7..a9e45087fd 100644
--- a/src/calibre/gui2/dialogs/metadata_bulk.py
+++ b/src/calibre/gui2/dialogs/metadata_bulk.py
@@ -122,15 +122,6 @@ class SafeFormat(string.Formatter):
v = ','.join(v)
return v
-composite_formatter = SafeFormat()
-
-def format_composite(x, mi):
- try:
- ans = composite_formatter.vformat(x, [], mi).strip()
- except:
- ans = x
- return ans
-
class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
s_r_functions = { '' : lambda x: x,
diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py
index fe62dcb7fd..6f7929a072 100644
--- a/src/calibre/library/save_to_disk.py
+++ b/src/calibre/library/save_to_disk.py
@@ -101,15 +101,30 @@ def preprocess_template(template):
template = template.decode(preferred_encoding, 'replace')
return template
+template_value_re = re.compile(r'^([^\|]*(?=\|))(?:\|?)([^\|]*)(?:\|?)((?<=\|).*?)$')
+
+def explode_string_template_value(key):
+ try:
+ matches = template_value_re.match(key)
+ if matches.lastindex != 3:
+ return key
+ return matches.groups()
+ except:
+ return '', key, ''
+
class SafeFormat(string.Formatter):
'''
Provides a format function that substitutes '' for any missing value
'''
def get_value(self, key, args, kwargs):
try:
- return kwargs[key]
+ prefix, key, suffix = explode_string_template_value(key)
+ if kwargs[key]:
+ return '%s%s%s'%(prefix, kwargs[key], suffix)
+ return ''
except:
return ''
+
safe_formatter = SafeFormat()
def safe_format(x, format_args):
From 9112277d00187936e7cc8ffd3bf1eba3b87cfbe7 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Wed, 22 Sep 2010 09:50:01 -0600
Subject: [PATCH 082/207] ...
---
src/calibre/manual/index.rst | 8 +-------
src/calibre/manual/tutorials.rst | 17 +++++++++++++++++
2 files changed, 18 insertions(+), 7 deletions(-)
create mode 100644 src/calibre/manual/tutorials.rst
diff --git a/src/calibre/manual/index.rst b/src/calibre/manual/index.rst
index 40c260b8b5..d63b0b71a9 100644
--- a/src/calibre/manual/index.rst
+++ b/src/calibre/manual/index.rst
@@ -33,16 +33,10 @@ Sections
conversion
metadata
faq
- xpath
+ tutorials
customize
cli/cli-index
develop
glossary
-.. toctree::
- :hidden:
- :maxdepth: 2
-
- template_lang
- portable
diff --git a/src/calibre/manual/tutorials.rst b/src/calibre/manual/tutorials.rst
new file mode 100644
index 0000000000..d07316deb9
--- /dev/null
+++ b/src/calibre/manual/tutorials.rst
@@ -0,0 +1,17 @@
+
+.. include:: global.rst
+
+.. _tutorials:
+
+Tutorials
+=======================================================
+
+Here you will find tutorials to get you started using |app|'s more advanced features, like XPath and templates.
+
+.. toctree::
+ :maxdepth: 1
+
+ xpath
+ template_lang
+ portable
+
From bfc3e63980dd0fc47601529ca1f224602e10f777 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Wed, 22 Sep 2010 09:57:33 -0600
Subject: [PATCH 083/207] ...
---
src/calibre/library/save_to_disk.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py
index 6f7929a072..19727deb17 100644
--- a/src/calibre/library/save_to_disk.py
+++ b/src/calibre/library/save_to_disk.py
@@ -120,7 +120,7 @@ class SafeFormat(string.Formatter):
try:
prefix, key, suffix = explode_string_template_value(key)
if kwargs[key]:
- return '%s%s%s'%(prefix, kwargs[key], suffix)
+ return prefix + kwargs[key] + suffix
return ''
except:
return ''
From bd8a206219b69fb92305db8135b28f98021a3b62 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Wed, 22 Sep 2010 17:10:57 +0100
Subject: [PATCH 084/207] Changes to faq
---
src/calibre/manual/template_lang.rst | 15 ++++++++++++++-
1 file changed, 14 insertions(+), 1 deletion(-)
diff --git a/src/calibre/manual/template_lang.rst b/src/calibre/manual/template_lang.rst
index 541b5da138..780d742eff 100644
--- a/src/calibre/manual/template_lang.rst
+++ b/src/calibre/manual/template_lang.rst
@@ -40,7 +40,20 @@ and if a book does not have a series::
Advanced formatting
----------------------
-You can do more than just simple substitution with the templates. You can also control how the substituted data is formatted. For instance, suppose you wanted to ensure that the series_index is always formatted as three digits with leading zeros. This would do the trick::
+You can do more than just simple substitution with the templates. You can also conditionally include text and control how the substituted data is formatted.
+
+Regarding conditionally including text: there are cases where you might want to have text appear in the output only if a field is not empty. A common case is series and series_index, where you want either nothing or the two values with a hyphen between them. Calibre handles this case using a special field syntax.
+For example, assume you want to use the template
+
+ {series} - {series_index} - {title}
+
+Unfortunately, if the book has no series, the answer will be '- - title'. Many people would rather it be simply 'title', without the hyphens. To do this, use the extended syntax {some_text|field|other_text}. When you use this syntax, if field has the value SERIES then the result will be some_textSERIESother_text. If field has no value, then the result will be the empty string (nothing). Using this syntax, we can solve the above series problem with the template
+
+ {series}{ - |series_index| - }{title}
+
+The hyphens will be included only if the book has a series index. Note: you must either use no | characters or both of them. Using one, such as in {field| - }, is not allowed. It is OK to not provide any text for one side or the other, such as in {|series| - }. Using {|title|} is the same as using {title}.
+
+Now to formatting. Suppose you wanted to ensure that the series_index is always formatted as three digits with leading zeros. This would do the trick::
{series_index:0>3s} - Three digits with leading zeros
From f280cc956fa7b234212ed995fd7497b29b9ac5ea Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Wed, 22 Sep 2010 20:56:08 +0100
Subject: [PATCH 085/207] Fix template bugs introduced by using + instead of
'%s'
---
src/calibre/ebooks/metadata/book/base.py | 4 ++--
src/calibre/library/save_to_disk.py | 5 +++--
2 files changed, 5 insertions(+), 4 deletions(-)
diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index 39b9b34174..4e78bf5a48 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -43,9 +43,9 @@ class SafeFormat(string.Formatter):
ign, v = mi.format_field(key, series_with_index=False)
if v is None:
return ''
- if v is '':
+ if v == '':
return ''
- return prefix + v + suffix
+ return prefix + unicode(v) + suffix
except:
return key
diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py
index 19727deb17..90e5413389 100644
--- a/src/calibre/library/save_to_disk.py
+++ b/src/calibre/library/save_to_disk.py
@@ -101,7 +101,8 @@ def preprocess_template(template):
template = template.decode(preferred_encoding, 'replace')
return template
-template_value_re = re.compile(r'^([^\|]*(?=\|))(?:\|?)([^\|]*)(?:\|?)((?<=\|).*?)$')
+template_value_re = re.compile(r'^([^\|]*(?=\|))(?:\|?)([^\|]*)(?:\|?)((?<=\|).*?)$',
+ flags= re.UNICODE)
def explode_string_template_value(key):
try:
@@ -120,7 +121,7 @@ class SafeFormat(string.Formatter):
try:
prefix, key, suffix = explode_string_template_value(key)
if kwargs[key]:
- return prefix + kwargs[key] + suffix
+ return prefix + unicode(kwargs[key]) + suffix
return ''
except:
return ''
From 8a9ae38ebff2cc641d2d661da2d6577db7f0acd0 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Wed, 22 Sep 2010 21:00:25 +0100
Subject: [PATCH 086/207] Change to fix to make the value unicode in
format_field_extended. This is a more general fix. Note that the orig_field
has not been changed.
---
src/calibre/ebooks/metadata/book/base.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index 4e78bf5a48..a2b2790ed9 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -45,7 +45,7 @@ class SafeFormat(string.Formatter):
return ''
if v == '':
return ''
- return prefix + unicode(v) + suffix
+ return prefix + v + suffix
except:
return key
@@ -444,7 +444,7 @@ class Metadata(object):
res = format_date(res, cmeta['display'].get('date_format','dd MMM yyyy'))
elif datatype == 'bool':
res = _('Yes') if res else _('No')
- return (name, res, orig_res, cmeta)
+ return (name, unicode(res), orig_res, cmeta)
if key in field_metadata and field_metadata[key]['kind'] == 'field':
res = self.get(key, None)
@@ -462,7 +462,7 @@ class Metadata(object):
res = res + ' [%s]'%self.format_series_index()
elif datatype == 'datetime':
res = format_date(res, fmeta['display'].get('date_format','dd MMM yyyy'))
- return (name, res, orig_res, fmeta)
+ return (name, unicode(res), orig_res, fmeta)
return (None, None, None, None)
From 7e233696a1c2bbe3ca4c0469a99540da665fc94f Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Wed, 22 Sep 2010 16:11:53 -0600
Subject: [PATCH 087/207] ...
---
src/calibre/manual/tutorials.rst | 1 +
src/calibre/utils/pyconsole/__init__.py | 4 ++--
src/calibre/utils/pyconsole/main.py | 22 +++++++++++++++++-----
3 files changed, 20 insertions(+), 7 deletions(-)
diff --git a/src/calibre/manual/tutorials.rst b/src/calibre/manual/tutorials.rst
index d07316deb9..1e4cab8493 100644
--- a/src/calibre/manual/tutorials.rst
+++ b/src/calibre/manual/tutorials.rst
@@ -11,6 +11,7 @@ Here you will find tutorials to get you started using |app|'s more advanced feat
.. toctree::
:maxdepth: 1
+ news
xpath
template_lang
portable
diff --git a/src/calibre/utils/pyconsole/__init__.py b/src/calibre/utils/pyconsole/__init__.py
index 32eb926143..cf7e5eab50 100644
--- a/src/calibre/utils/pyconsole/__init__.py
+++ b/src/calibre/utils/pyconsole/__init__.py
@@ -8,7 +8,7 @@ __docformat__ = 'restructuredtext en'
import sys
from calibre import prints as prints_
-from calibre.utils.config import Config, ConfigProxy
+from calibre.utils.config import Config, ConfigProxy, JSONConfig
def console_config():
@@ -20,7 +20,7 @@ def console_config():
return c
prefs = ConfigProxy(console_config())
-
+dynamic = JSONConfig('console')
def prints(*args, **kwargs):
kwargs['file'] = sys.__stdout__
diff --git a/src/calibre/utils/pyconsole/main.py b/src/calibre/utils/pyconsole/main.py
index a708ca1652..4027897ddd 100644
--- a/src/calibre/utils/pyconsole/main.py
+++ b/src/calibre/utils/pyconsole/main.py
@@ -12,6 +12,7 @@ from PyQt4.Qt import QDialog, QToolBar, QStatusBar, QLabel, QFont, Qt, \
QApplication, QIcon, QVBoxLayout, QAction
from calibre.constants import __appname__, __version__
+from calibre.utils.pyconsole import dynamic
from calibre.utils.pyconsole.console import Console
class MainWindow(QDialog):
@@ -26,6 +27,9 @@ class MainWindow(QDialog):
self.setLayout(self.l)
self.resize(800, 600)
+ geom = dynamic.get('console_window_geometry', None)
+ if geom is not None:
+ self.restoreGeometry(geom)
# Setup tool bar {{{
self.tool_bar = QToolBar(self)
@@ -62,17 +66,25 @@ class MainWindow(QDialog):
self.restart_requested = True
self.reject()
-def main():
- QApplication.setApplicationName(__appname__+' console')
- QApplication.setOrganizationName('Kovid Goyal')
- app = QApplication([])
- app
+ def closeEvent(self, *args):
+ dynamic.set('console_window_geometry',
+ bytearray(self.saveGeometry()))
+ return QDialog.closeEvent(self, *args)
+
+
+def show():
while True:
m = MainWindow()
m.exec_()
if not m.restart_requested:
break
+def main():
+ QApplication.setApplicationName(__appname__+' console')
+ QApplication.setOrganizationName('Kovid Goyal')
+ app = QApplication([])
+ app
+ show()
if __name__ == '__main__':
main()
From 69e4cb006545c9db1bc7704c4363a9a111d9c3c1 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Wed, 22 Sep 2010 17:05:35 -0600
Subject: [PATCH 088/207] ...
---
src/calibre/utils/pyconsole/__init__.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/src/calibre/utils/pyconsole/__init__.py b/src/calibre/utils/pyconsole/__init__.py
index cf7e5eab50..b3f811faca 100644
--- a/src/calibre/utils/pyconsole/__init__.py
+++ b/src/calibre/utils/pyconsole/__init__.py
@@ -9,6 +9,7 @@ import sys
from calibre import prints as prints_
from calibre.utils.config import Config, ConfigProxy, JSONConfig
+from calibre.utils.ipc.launch import Worker
def console_config():
@@ -26,4 +27,6 @@ def prints(*args, **kwargs):
kwargs['file'] = sys.__stdout__
prints_(*args, **kwargs)
+class Process(Worker):
+ pass
From c5e26ad9d5660aebdc21afc57e0626e83b2c0b92 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Wed, 22 Sep 2010 21:43:08 -0600
Subject: [PATCH 089/207] Refactor console to run interpreter in separate
process
---
src/calibre/utils/pyconsole/__init__.py | 3 +
src/calibre/utils/pyconsole/console.py | 114 ++++++++-----
src/calibre/utils/pyconsole/controller.py | 125 +++++++++++++++
src/calibre/utils/pyconsole/interpreter.py | 177 +++++++++++++++++++++
src/calibre/utils/pyconsole/main.py | 1 +
src/calibre/utils/pyconsole/repl.py | 67 --------
6 files changed, 381 insertions(+), 106 deletions(-)
create mode 100644 src/calibre/utils/pyconsole/controller.py
create mode 100644 src/calibre/utils/pyconsole/interpreter.py
delete mode 100644 src/calibre/utils/pyconsole/repl.py
diff --git a/src/calibre/utils/pyconsole/__init__.py b/src/calibre/utils/pyconsole/__init__.py
index 3b32a5a9f3..3be9382413 100644
--- a/src/calibre/utils/pyconsole/__init__.py
+++ b/src/calibre/utils/pyconsole/__init__.py
@@ -13,6 +13,9 @@ from calibre.utils.ipc.launch import Worker
from calibre.constants import __appname__, __version__, iswindows
from calibre.gui2 import error_dialog
+# Time to wait for communication to/from the interpreter process
+POLL_TIMEOUT = 0.01 # seconds
+
preferred_encoding, isbytestring, __appname__, __version__, error_dialog, \
iswindows
diff --git a/src/calibre/utils/pyconsole/console.py b/src/calibre/utils/pyconsole/console.py
index 77c8d9a2f6..1acb6e96a9 100644
--- a/src/calibre/utils/pyconsole/console.py
+++ b/src/calibre/utils/pyconsole/console.py
@@ -9,13 +9,13 @@ import sys, textwrap, traceback, StringIO
from functools import partial
from PyQt4.Qt import QTextEdit, Qt, QTextFrameFormat, pyqtSignal, \
- QApplication, QColor, QPalette, QMenu, QActionGroup
+ QApplication, QColor, QPalette, QMenu, QActionGroup, QTimer
from pygments.lexers import PythonLexer, PythonTracebackLexer
from pygments.styles import get_all_styles
from calibre.utils.pyconsole.formatter import Formatter
-from calibre.utils.pyconsole.repl import Interpreter, DummyFile
+from calibre.utils.pyconsole.controller import Controller
from calibre.utils.pyconsole import prints, prefs, __appname__, \
__version__, error_dialog
@@ -113,7 +113,8 @@ class Console(QTextEdit):
continuation='... ',
parent=None):
QTextEdit.__init__(self, parent)
- self.buf = []
+ self.shutting_down = False
+ self.buf = self.old_buf = []
self.prompt_frame = None
self.allow_output = False
self.prompt_frame_format = QTextFrameFormat()
@@ -152,20 +153,80 @@ class Console(QTextEdit):
'''.format(sys.version.splitlines()[0], __appname__,
__version__))
+ self.controllers = []
+ QTimer.singleShot(0, self.launch_controller)
+
+ sys.excepthook = self.unhandled_exception
+
with EditBlock(self.cursor):
self.render_block(motd)
- sys.stdout = sys.stderr = DummyFile(parent=self)
- sys.stdout.write_output.connect(self.show_output)
- self.interpreter = Interpreter(parent=self)
- self.interpreter.show_error.connect(self.show_error)
-
- sys.excepthook = self.unhandled_exception
+ def shutdown(self):
+ self.shutton_down = True
+ for c in self.controllers:
+ c.kill()
def contextMenuEvent(self, event):
self.context_menu.popup(event.globalPos())
event.accept()
+ # Controller management {{{
+ @property
+ def controller(self):
+ return self.controllers[-1]
+
+ def no_controller_error(self):
+ error_dialog(self, _('No interpreter'),
+ _('No active interpreter found. Try restarting the'
+ ' console'), show=True)
+
+ def launch_controller(self, *args):
+ c = Controller(self)
+ c.write_output.connect(self.show_output, type=Qt.QueuedConnection)
+ c.show_error.connect(self.show_error, type=Qt.QueuedConnection)
+ c.interpreter_died.connect(self.interpreter_died,
+ type=Qt.QueuedConnection)
+ c.interpreter_done.connect(self.execution_done)
+ self.controllers.append(c)
+
+ def interpreter_died(self, controller, returncode):
+ if not self.shutting_down and controller.current_command is not None:
+ error_dialog(self, _('Interpreter died'),
+ _('Interpreter dies while excuting a command. To see '
+ 'the command, click Show details'),
+ det_msg=controller.current_command, show=True)
+
+ def execute(self, prompt_lines):
+ c = self.root_frame.lastCursorPosition()
+ self.setTextCursor(c)
+ self.old_prompt_frame = self.prompt_frame
+ self.prompt_frame = None
+ self.old_buf = self.buf
+ self.buf = []
+ self.running.emit()
+ self.controller.runsource('\n'.join(prompt_lines))
+
+ def execution_done(self, controller, ret):
+ if controller is self.controller:
+ self.running_done.emit()
+ if ret: # Incomplete command
+ self.buf = self.old_buf
+ self.prompt_frame = self.old_prompt_frame
+ c = self.prompt_frame.lastCursorPosition()
+ c.insertBlock()
+ self.setTextCursor(c)
+ else: # Command completed
+ try:
+ self.old_prompt_frame.setFrameFormat(QTextFrameFormat())
+ except RuntimeError:
+ # Happens if enough lines of output that the old
+ # frame was deleted
+ pass
+
+ self.render_current_prompt()
+
+ # }}}
+
# Prompt management {{{
@dynamic_property
@@ -264,7 +325,7 @@ class Console(QTextEdit):
if restore_prompt:
self.render_current_prompt()
- def show_error(self, is_syntax_err, tb):
+ def show_error(self, is_syntax_err, tb, controller=None):
if self.prompt_frame is not None:
# At a prompt, so redirect output
return prints(tb, end='')
@@ -279,7 +340,7 @@ class Console(QTextEdit):
self.ensureCursorVisible()
QApplication.processEvents()
- def show_output(self, raw):
+ def show_output(self, raw, which='stdout', controller=None):
def do_show():
try:
self.buf.append(raw)
@@ -384,36 +445,11 @@ class Console(QTextEdit):
def enter_pressed(self):
if self.prompt_frame is None:
return
+ if not self.controller.is_alive:
+ return self.no_controller_error()
cp = list(self.prompt())
if cp[0]:
- c = self.root_frame.lastCursorPosition()
- self.setTextCursor(c)
- old_pf = self.prompt_frame
- self.prompt_frame = None
- oldbuf = self.buf
- self.buf = []
- self.running.emit()
- try:
- ret = self.interpreter.runsource('\n'.join(cp))
- except SystemExit:
- ret = False
- self.show_output('Raising SystemExit not allowed\n')
- self.running_done.emit()
- if ret: # Incomplete command
- self.buf = oldbuf
- self.prompt_frame = old_pf
- c = old_pf.lastCursorPosition()
- c.insertBlock()
- self.setTextCursor(c)
- else: # Command completed
- try:
- old_pf.setFrameFormat(QTextFrameFormat())
- except RuntimeError:
- # Happens if enough lines of output that the old
- # frame was deleted
- pass
-
- self.render_current_prompt()
+ self.execute(cp)
def text_typed(self, text):
if self.prompt_frame is not None:
diff --git a/src/calibre/utils/pyconsole/controller.py b/src/calibre/utils/pyconsole/controller.py
new file mode 100644
index 0000000000..173881d14e
--- /dev/null
+++ b/src/calibre/utils/pyconsole/controller.py
@@ -0,0 +1,125 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+__license__ = 'GPL v3'
+__copyright__ = '2010, Kovid Goyal '
+__docformat__ = 'restructuredtext en'
+
+import os, cPickle, signal, time
+from Queue import Queue, Empty
+from multiprocessing.connection import Listener, arbitrary_address
+from binascii import hexlify
+
+from PyQt4.Qt import QThread, pyqtSignal
+
+from calibre.utils.pyconsole import Process, iswindows, POLL_TIMEOUT
+
+class Controller(QThread):
+
+ # show_error(is_syntax_error, traceback, self)
+ show_error = pyqtSignal(object, object, object)
+ # write_output(unicode_object, stdout or stderr, self)
+ write_output = pyqtSignal(object, object, object)
+ # Indicates interpreter has finished evaluating current command
+ interpreter_done = pyqtSignal(object, object)
+ # interpreter_died(self, returncode or None if no return code available)
+ interpreter_died = pyqtSignal(object, object)
+
+ def __init__(self, parent):
+ QThread.__init__(self, parent)
+ self.keep_going = True
+ self.current_command = None
+
+ self.out_queue = Queue()
+ self.address = arbitrary_address('AF_PIPE' if iswindows else 'AF_UNIX')
+ self.auth_key = os.urandom(32)
+ if iswindows and self.address[1] == ':':
+ self.address = self.address[2:]
+ self.listener = Listener(address=self.address,
+ authkey=self.auth_key, backlog=4)
+
+ self.env = {
+ 'CALIBRE_LAUNCH_INTERPRETER': '1',
+ 'CALIBRE_WORKER_ADDRESS':
+ hexlify(cPickle.dumps(self.listener.address, -1)),
+ 'CALIBRE_WORKER_KEY': hexlify(self.auth_key)
+ }
+ self.process = Process(self.env)
+ self.output_file_buf = self.process(redirect_output=False)
+ self.conn = self.listener.accept()
+ self.start()
+
+ def run(self):
+ while self.keep_going and self.is_alive:
+ try:
+ self.communicate()
+ except KeyboardInterrupt:
+ pass
+ except EOFError:
+ break
+ self.interpreter_died.emit(self, self.returncode)
+ try:
+ self.listener.close()
+ except:
+ pass
+
+ def communicate(self):
+ if self.conn.poll(POLL_TIMEOUT):
+ self.dispatch_incoming_message(self.conn.recv())
+ try:
+ obj = self.out_queue.get_nowait()
+ except Empty:
+ pass
+ else:
+ try:
+ self.conn.send(obj)
+ except:
+ raise EOFError('controller failed to send')
+
+ def dispatch_incoming_message(self, obj):
+ try:
+ cmd, data = obj
+ except:
+ print 'Controller received invalid message'
+ print repr(obj)
+ return
+ if cmd in ('stdout', 'stderr'):
+ self.write_output.emit(data, cmd, self)
+ elif cmd == 'syntaxerror':
+ self.show_error.emit(True, data, self)
+ elif cmd == 'traceback':
+ self.show_error(self, False, data)
+ elif cmd == 'done':
+ self.current_command = None
+ self.interpreter_done.emit(self, data)
+
+ def runsource(self, cmd):
+ self.current_command = cmd
+ self.out_queue.put(('run', cmd))
+
+ def __nonzero__(self):
+ return self.process.is_alive
+
+ @property
+ def returncode(self):
+ return self.process.returncode
+
+ @property
+ def interrupt(self):
+ if hasattr(signal, 'SIGINT'):
+ os.kill(self.process.pid, signal.SIGINT)
+ elif hasattr(signal, 'CTRL_C_EVENT'):
+ os.kill(self.process.pid, signal.CTRL_C_EVENT)
+
+ @property
+ def is_alive(self):
+ return self.process.is_alive
+
+ def kill(self):
+ self.out_queue.put(('quit', 0))
+ t = 0
+ while self.is_alive and t < 10:
+ time.sleep(0.1)
+ self.process.kill()
+ self.keep_going = False
+
diff --git a/src/calibre/utils/pyconsole/interpreter.py b/src/calibre/utils/pyconsole/interpreter.py
new file mode 100644
index 0000000000..6a1aff26c9
--- /dev/null
+++ b/src/calibre/utils/pyconsole/interpreter.py
@@ -0,0 +1,177 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+__license__ = 'GPL v3'
+__copyright__ = '2010, Kovid Goyal '
+__docformat__ = 'restructuredtext en'
+
+import sys, cPickle, os
+from code import InteractiveInterpreter
+from Queue import Queue, Empty
+from threading import Thread
+from binascii import unhexlify
+from multiprocessing.connection import Client
+
+from calibre.utils.pyconsole import preferred_encoding, isbytestring, \
+ POLL_TIMEOUT
+
+'''
+Messages sent by client:
+
+ (stdout, unicode)
+ (stderr, unicode)
+ (syntaxerror, unicode)
+ (traceback, unicode)
+ (done, True iff incomplete command)
+
+Messages that can be received by client:
+ (quit, return code)
+ (run, unicode)
+
+'''
+
+def tounicode(raw): # {{{
+ if isbytestring(raw):
+ try:
+ raw = raw.decode(preferred_encoding, 'replace')
+ except:
+ raw = repr(raw)
+
+ if isbytestring(raw):
+ try:
+ raw.decode('utf-8', 'replace')
+ except:
+ raw = u'Undecodable bytestring'
+ return raw
+# }}}
+
+class DummyFile(object): # {{{
+
+ def __init__(self, what, out_queue):
+ self.closed = False
+ self.name = 'console'
+ self.softspace = 0
+ self.what = what
+ self.out_queue = out_queue
+
+ def flush(self):
+ pass
+
+ def close(self):
+ pass
+
+ def write(self, raw):
+ self.out_queue.put((self.what, tounicode(raw)))
+# }}}
+
+class Comm(Thread): # {{{
+
+ def __init__(self, conn, out_queue, in_queue):
+ Thread.__init__(self)
+ self.daemon = True
+ self.conn = conn
+ self.out_queue = out_queue
+ self.in_queue = in_queue
+ self.keep_going = True
+
+ def run(self):
+ while self.keep_going:
+ try:
+ self.communicate()
+ except KeyboardInterrupt:
+ pass
+ except EOFError:
+ pass
+
+ def communicate(self):
+ if self.conn.poll(POLL_TIMEOUT):
+ try:
+ obj = self.conn.recv()
+ except:
+ pass
+ else:
+ self.in_queue.put(obj)
+ try:
+ obj = self.out_queue.get_nowait()
+ except Empty:
+ pass
+ else:
+ try:
+ self.conn.send(obj)
+ except:
+ raise EOFError('interpreter failed to send')
+# }}}
+
+class Interpreter(InteractiveInterpreter): # {{{
+
+ def __init__(self, queue, local={}):
+ if '__name__' not in local:
+ local['__name__'] = '__console__'
+ if '__doc__' not in local:
+ local['__doc__'] = None
+ self.out_queue = queue
+ sys.stdout = DummyFile('stdout', queue)
+ sys.stderr = DummyFile('sdterr', queue)
+ InteractiveInterpreter.__init__(self, locals=local)
+
+ def showtraceback(self, *args, **kwargs):
+ self.is_syntax_error = False
+ InteractiveInterpreter.showtraceback(self, *args, **kwargs)
+
+ def showsyntaxerror(self, *args, **kwargs):
+ self.is_syntax_error = True
+ InteractiveInterpreter.showsyntaxerror(self, *args, **kwargs)
+
+ def write(self, raw):
+ what = 'syntaxerror' if self.is_syntax_error else 'traceback'
+ self.out_queue.put((what, tounicode(raw)))
+
+# }}}
+
+def connect():
+ os.chdir(os.environ['ORIGWD'])
+ address = cPickle.loads(unhexlify(os.environ['CALIBRE_WORKER_ADDRESS']))
+ key = unhexlify(os.environ['CALIBRE_WORKER_KEY'])
+ return Client(address, authkey=key)
+
+def main():
+ out_queue = Queue()
+ in_queue = Queue()
+ conn = connect()
+ comm = Comm(conn, out_queue, in_queue)
+ comm.start()
+ interpreter = Interpreter(out_queue)
+
+ ret = 0
+
+ while True:
+ try:
+ try:
+ cmd, data = in_queue.get(1)
+ except Empty:
+ pass
+ else:
+ if cmd == 'quit':
+ ret = data
+ comm.keep_going = False
+ comm.join()
+ break
+ elif cmd == 'run':
+ if not comm.is_alive():
+ ret = 1
+ break
+ ret = False
+ try:
+ ret = interpreter.runsource(data)
+ except KeyboardInterrupt:
+ pass
+ except SystemExit:
+ out_queue.put(('stderr', 'SystemExit ignored\n'))
+ out_queue.put(('done', ret))
+ except KeyboardInterrupt:
+ pass
+
+ return ret
+
+if __name__ == '__main__':
+ main()
diff --git a/src/calibre/utils/pyconsole/main.py b/src/calibre/utils/pyconsole/main.py
index a64bc15ec7..664f41ef2e 100644
--- a/src/calibre/utils/pyconsole/main.py
+++ b/src/calibre/utils/pyconsole/main.py
@@ -68,6 +68,7 @@ class MainWindow(QDialog):
def closeEvent(self, *args):
dynamic.set('console_window_geometry',
bytearray(self.saveGeometry()))
+ self.console.shutdown()
return QDialog.closeEvent(self, *args)
diff --git a/src/calibre/utils/pyconsole/repl.py b/src/calibre/utils/pyconsole/repl.py
deleted file mode 100644
index de6262de14..0000000000
--- a/src/calibre/utils/pyconsole/repl.py
+++ /dev/null
@@ -1,67 +0,0 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
-
-__license__ = 'GPL v3'
-__copyright__ = '2010, Kovid Goyal '
-__docformat__ = 'restructuredtext en'
-
-from code import InteractiveInterpreter
-
-from PyQt4.Qt import QObject, pyqtSignal
-
-from calibre import isbytestring
-from calibre.constants import preferred_encoding
-
-class Interpreter(QObject, InteractiveInterpreter):
-
- # show_error(is_syntax_error, traceback)
- show_error = pyqtSignal(object, object)
-
- def __init__(self, local={}, parent=None):
- QObject.__init__(self, parent)
- if '__name__' not in local:
- local['__name__'] = '__console__'
- if '__doc__' not in local:
- local['__doc__'] = None
- InteractiveInterpreter.__init__(self, locals=local)
-
- def showtraceback(self, *args, **kwargs):
- self.is_syntax_error = False
- InteractiveInterpreter.showtraceback(self, *args, **kwargs)
-
- def showsyntaxerror(self, *args, **kwargs):
- self.is_syntax_error = True
- InteractiveInterpreter.showsyntaxerror(self, *args, **kwargs)
-
- def write(self, tb):
- self.show_error.emit(self.is_syntax_error, tb)
-
-class DummyFile(QObject):
-
- # write_output(unicode_object)
- write_output = pyqtSignal(object)
-
- def __init__(self, parent=None):
- QObject.__init__(self, parent)
- self.closed = False
- self.name = 'console'
- self.softspace = 0
-
- def flush(self):
- pass
-
- def close(self):
- pass
-
- def write(self, raw):
- #import sys, traceback
- #print >> sys.__stdout__, 'file,write stack:\n', ''.join(traceback.format_stack())
- if isbytestring(raw):
- try:
- raw = raw.decode(preferred_encoding, 'replace')
- except:
- raw = repr(raw)
- if isbytestring(raw):
- raw = raw.decode(preferred_encoding, 'replace')
- self.write_output.emit(raw)
-
From 950f592b87173daaeb84244560c276adb0cb49f8 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Wed, 22 Sep 2010 21:48:17 -0600
Subject: [PATCH 090/207] ...
---
src/calibre/utils/pyconsole/console.py | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/src/calibre/utils/pyconsole/console.py b/src/calibre/utils/pyconsole/console.py
index 1acb6e96a9..2611965345 100644
--- a/src/calibre/utils/pyconsole/console.py
+++ b/src/calibre/utils/pyconsole/console.py
@@ -268,6 +268,11 @@ class Console(QTextEdit):
return property(fget=fget, fset=fset, doc=doc)
+ def move_cursor_to_prompt(self):
+ if self.prompt_frame is not None and self.cursor_pos[0] < 0:
+ c = self.prompt_frame.lastCursorPosition()
+ self.setTextCursor(c)
+
def prompt(self, strip_prompt_strings=True):
if not self.prompt_frame:
yield u'' if strip_prompt_strings else self.formatter.prompt
@@ -453,6 +458,7 @@ class Console(QTextEdit):
def text_typed(self, text):
if self.prompt_frame is not None:
+ self.move_cursor_to_prompt()
self.cursor.insertText(text)
self.render_current_prompt(restore_cursor=True)
From 9b44f557850a7cdbceac98194cc9bda77d76330c Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Wed, 22 Sep 2010 21:54:58 -0600
Subject: [PATCH 091/207] ...
---
src/calibre/utils/pyconsole/controller.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/calibre/utils/pyconsole/controller.py b/src/calibre/utils/pyconsole/controller.py
index 173881d14e..368e665079 100644
--- a/src/calibre/utils/pyconsole/controller.py
+++ b/src/calibre/utils/pyconsole/controller.py
@@ -88,7 +88,7 @@ class Controller(QThread):
elif cmd == 'syntaxerror':
self.show_error.emit(True, data, self)
elif cmd == 'traceback':
- self.show_error(self, False, data)
+ self.show_error.emit(False, data, self)
elif cmd == 'done':
self.current_command = None
self.interpreter_done.emit(self, data)
From d9e5e74695e82e56b8cc0f61399036ea17a485dd Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Thu, 23 Sep 2010 01:02:03 -0600
Subject: [PATCH 092/207] Console now has history
---
src/calibre/utils/pyconsole/__init__.py | 2 +
src/calibre/utils/pyconsole/console.py | 62 ++++++++++++++++++++--
src/calibre/utils/pyconsole/controller.py | 1 -
src/calibre/utils/pyconsole/interpreter.py | 3 +-
4 files changed, 61 insertions(+), 7 deletions(-)
diff --git a/src/calibre/utils/pyconsole/__init__.py b/src/calibre/utils/pyconsole/__init__.py
index 3be9382413..6ef9f04d4b 100644
--- a/src/calibre/utils/pyconsole/__init__.py
+++ b/src/calibre/utils/pyconsole/__init__.py
@@ -24,6 +24,8 @@ def console_config():
c = Config('console', desc)
c.add_opt('theme', default='native', help='The color theme')
+ c.add_opt('scrollback', default=10000,
+ help='Max number of lines to keep in the scrollback buffer')
return c
diff --git a/src/calibre/utils/pyconsole/console.py b/src/calibre/utils/pyconsole/console.py
index 2611965345..14670fdb59 100644
--- a/src/calibre/utils/pyconsole/console.py
+++ b/src/calibre/utils/pyconsole/console.py
@@ -7,6 +7,7 @@ __docformat__ = 'restructuredtext en'
import sys, textwrap, traceback, StringIO
from functools import partial
+from codeop import CommandCompiler
from PyQt4.Qt import QTextEdit, Qt, QTextFrameFormat, pyqtSignal, \
QApplication, QColor, QPalette, QMenu, QActionGroup, QTimer
@@ -16,8 +17,9 @@ from pygments.styles import get_all_styles
from calibre.utils.pyconsole.formatter import Formatter
from calibre.utils.pyconsole.controller import Controller
+from calibre.utils.pyconsole.history import History
from calibre.utils.pyconsole import prints, prefs, __appname__, \
- __version__, error_dialog
+ __version__, error_dialog, dynamic
class EditBlock(object): # {{{
@@ -73,6 +75,7 @@ class ThemeMenu(QMenu): # {{{
# }}}
+
class Console(QTextEdit):
running = pyqtSignal()
@@ -114,7 +117,9 @@ class Console(QTextEdit):
parent=None):
QTextEdit.__init__(self, parent)
self.shutting_down = False
+ self.compiler = CommandCompiler()
self.buf = self.old_buf = []
+ self.history = History([''], dynamic.get('console_history', []))
self.prompt_frame = None
self.allow_output = False
self.prompt_frame_format = QTextFrameFormat()
@@ -122,7 +127,7 @@ class Console(QTextEdit):
self.prompt_frame_format.setBorderStyle(QTextFrameFormat.BorderStyle_Solid)
self.prompt_len = len(prompt)
- self.doc.setMaximumBlockCount(10000)
+ self.doc.setMaximumBlockCount(int(prefs['scrollback']))
self.lexer = PythonLexer(ensurenl=False)
self.tb_lexer = PythonTracebackLexer()
@@ -139,6 +144,8 @@ class Console(QTextEdit):
self.key_dispatcher = { # {{{
Qt.Key_Enter : self.enter_pressed,
Qt.Key_Return : self.enter_pressed,
+ Qt.Key_Up : self.up_pressed,
+ Qt.Key_Down : self.down_pressed,
Qt.Key_Home : self.home_pressed,
Qt.Key_End : self.end_pressed,
Qt.Key_Left : self.left_pressed,
@@ -153,15 +160,17 @@ class Console(QTextEdit):
'''.format(sys.version.splitlines()[0], __appname__,
__version__))
+ sys.excepthook = self.unhandled_exception
+
self.controllers = []
QTimer.singleShot(0, self.launch_controller)
- sys.excepthook = self.unhandled_exception
with EditBlock(self.cursor):
self.render_block(motd)
def shutdown(self):
+ dynamic.set('console_history', self.history.serialize())
self.shutton_down = True
for c in self.controllers:
c.kill()
@@ -365,7 +374,7 @@ class Console(QTextEdit):
# }}}
- # Keyboard handling {{{
+ # Keyboard management {{{
def keyPressEvent(self, ev):
text = unicode(ev.text())
@@ -394,6 +403,20 @@ class Console(QTextEdit):
self.setTextCursor(c)
self.ensureCursorVisible()
+ def up_pressed(self):
+ lineno, pos = self.cursor_pos
+ if lineno < 0: return
+ if lineno == 0:
+ b = self.history.back()
+ if b is not None:
+ self.set_prompt(b)
+ else:
+ c = self.cursor
+ c.movePosition(c.Up)
+ self.setTextCursor(c)
+ self.ensureCursorVisible()
+
+
def backspace_pressed(self):
lineno, pos = self.cursor_pos
if lineno < 0: return
@@ -414,7 +437,6 @@ class Console(QTextEdit):
lineno, pos = self.cursor_pos
if lineno < 0: return
c = self.cursor
- lineno, pos = self.cursor_pos
cp = list(self.prompt(False))
if pos < len(cp[lineno]):
c.movePosition(c.NextCharacter)
@@ -423,6 +445,22 @@ class Console(QTextEdit):
self.setTextCursor(c)
self.ensureCursorVisible()
+ def down_pressed(self):
+ lineno, pos = self.cursor_pos
+ if lineno < 0: return
+ c = self.cursor
+ cp = list(self.prompt(False))
+ if lineno >= len(cp) - 1:
+ b = self.history.forward()
+ if b is not None:
+ self.set_prompt(b)
+ else:
+ c = self.cursor
+ c.movePosition(c.Down)
+ self.setTextCursor(c)
+ self.ensureCursorVisible()
+
+
def home_pressed(self):
if self.prompt_frame is not None:
mods = QApplication.keyboardModifiers()
@@ -454,6 +492,19 @@ class Console(QTextEdit):
return self.no_controller_error()
cp = list(self.prompt())
if cp[0]:
+ try:
+ ret = self.compiler('\n'.join(cp))
+ except:
+ pass
+ else:
+ if ret is None:
+ c = self.prompt_frame.lastCursorPosition()
+ c.insertBlock()
+ self.setTextCursor(c)
+ self.render_current_prompt()
+ return
+ else:
+ self.history.enter(cp)
self.execute(cp)
def text_typed(self, text):
@@ -461,6 +512,7 @@ class Console(QTextEdit):
self.move_cursor_to_prompt()
self.cursor.insertText(text)
self.render_current_prompt(restore_cursor=True)
+ self.history.current = list(self.prompt())
# }}}
diff --git a/src/calibre/utils/pyconsole/controller.py b/src/calibre/utils/pyconsole/controller.py
index 368e665079..d372cb4ebc 100644
--- a/src/calibre/utils/pyconsole/controller.py
+++ b/src/calibre/utils/pyconsole/controller.py
@@ -104,7 +104,6 @@ class Controller(QThread):
def returncode(self):
return self.process.returncode
- @property
def interrupt(self):
if hasattr(signal, 'SIGINT'):
os.kill(self.process.pid, signal.SIGINT)
diff --git a/src/calibre/utils/pyconsole/interpreter.py b/src/calibre/utils/pyconsole/interpreter.py
index 6a1aff26c9..3cd0d94711 100644
--- a/src/calibre/utils/pyconsole/interpreter.py
+++ b/src/calibre/utils/pyconsole/interpreter.py
@@ -11,6 +11,7 @@ from Queue import Queue, Empty
from threading import Thread
from binascii import unhexlify
from multiprocessing.connection import Client
+from repr import repr as safe_repr
from calibre.utils.pyconsole import preferred_encoding, isbytestring, \
POLL_TIMEOUT
@@ -35,7 +36,7 @@ def tounicode(raw): # {{{
try:
raw = raw.decode(preferred_encoding, 'replace')
except:
- raw = repr(raw)
+ raw = safe_repr(raw)
if isbytestring(raw):
try:
From ea29f4b683ada1c41593ff90664cfa146008be5f Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Thu, 23 Sep 2010 14:36:47 +0100
Subject: [PATCH 093/207] Changes:
1) complete rewrite of composite field processing
-- creation of of formatter class in utils
-- change template validator (prefs/save_template.py) to use new formatting class
-- change save_to_disk to use new formatting class
-- change Metadata class to use new formatting class
-- Check for mutually recursive composite fields
-- change caches.py to use the 'get' interface (now the right one) for composites
2) Add template validation to the base deviceconfig plugin. It checks if the display widget has a 'validate' method, and if so, it calls it.
3) Change models.py so that composite templates can be edited on the library display.
-- back out the changes that set 'editable = False'
4) Fix problem in models.py where book info view was not being updated when a field is changed on library display
5) Changed save_to_disk to permit slashes in field specifications. Did this by splitting the template after template processing. This gives us basic variable folder structures
Example: Simple example: we want the folder structure series/series_index - title. If series does not exist, then the title should be in the top folder.
Template: {series:||/}{series_index:|| - }{title}
6) Change syntax for extended templates
-- prefixes and suffixes have moved to the end of the field specification.
Syntax: {series:|prefix value|suffix value}
You can put a standard python format specification between the : and the first |.
Either zero or two bars must be present.
7) Addition of some built-in functions to template processing. These appear in the format part.
Syntax: {title:uppercase()|prefix value|suffix value}
Functions apply to the value of the field in the format specification.
The functions available are:
-- uppercase(), lowercase(), titlecase(), capitalise()
-- ifempty(text)
If the field is empty, replace it with text.
-- shorten(from start, center string, from end)
Replace the field with a shortened version. The shortened version is found by joining the field's first 'from start' characters, the center string, and the field's last 'from end' characters.
Example: assume that the title is 'Values of beta will give rise to dom'. The field specification
{title:shorten(6,---,6)} will produce the result 'Values---to dom'
-- lookup(key if field not empty, key if field empty)
Replace the value of 'field' with the value of another field. The first field key (lookup name) is used if 'field' is not empty. The second field key is used if field is empty. This, coupled with composite fields and the change to save_to_disk above, facilitates complex variable folder trees on devices.
Example: If a book has a series, then we want the folder structure series/series index - title.fmt. If the book does not have a series, then we want the folder structure genre/author_sort/title.fmt. If the book has no genre, use 'Unknown'. To accomplish this, we:
1) create a composite field named AA containing '{series:||}/{series_index} - {title'.
2) create a composite field named BB containing '{#genre:ifempty(Unknown)}/{author_sort}/{title}
3) set the save template to '{series:lookup(AA,BB)}
---
src/calibre/ebooks/metadata/book/base.py | 50 ++++----
src/calibre/gui2/custom_column_widgets.py | 2 +
.../gui2/device_drivers/configwidget.py | 15 +++
src/calibre/gui2/library/delegates.py | 28 ++++-
src/calibre/gui2/library/models.py | 12 +-
src/calibre/gui2/library/views.py | 6 +-
src/calibre/gui2/preferences/columns.py | 3 +-
.../gui2/preferences/create_custom_column.py | 4 -
src/calibre/gui2/preferences/plugins.py | 6 +-
src/calibre/gui2/preferences/save_template.py | 14 +--
src/calibre/library/caches.py | 2 +-
src/calibre/library/save_to_disk.py | 29 +----
src/calibre/utils/formatter.py | 113 ++++++++++++++++++
13 files changed, 210 insertions(+), 74 deletions(-)
create mode 100644 src/calibre/utils/formatter.py
diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index a2b2790ed9..16819cbd39 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -5,7 +5,7 @@ __license__ = 'GPL v3'
__copyright__ = '2010, Kovid Goyal '
__docformat__ = 'restructuredtext en'
-import copy, re, string, traceback
+import copy, re, traceback
from calibre import prints
from calibre.ebooks.metadata.book import SC_COPYABLE_FIELDS
@@ -15,6 +15,7 @@ from calibre.ebooks.metadata.book import TOP_LEVEL_CLASSIFIERS
from calibre.ebooks.metadata.book import ALL_METADATA_FIELDS
from calibre.library.field_metadata import FieldMetadata
from calibre.utils.date import isoformat, format_date
+from calibre.utils.formatter import TemplateFormatter
NULL_VALUES = {
@@ -32,33 +33,19 @@ NULL_VALUES = {
field_metadata = FieldMetadata()
-class SafeFormat(string.Formatter):
- '''
- Provides a format function that substitutes '' for any missing value
- '''
+class SafeFormat(TemplateFormatter):
def get_value(self, key, args, mi):
- from calibre.library.save_to_disk import explode_string_template_value
try:
- prefix, key, suffix = explode_string_template_value(key)
ign, v = mi.format_field(key, series_with_index=False)
if v is None:
return ''
if v == '':
return ''
- return prefix + v + suffix
+ return v
except:
return key
composite_formatter = SafeFormat()
-compress_spaces = re.compile(r'\s+')
-
-def format_composite(x, mi):
- try:
- ans = composite_formatter.vformat(x, [], mi).strip()
- except:
- traceback.print_exc()
- ans = x
- return compress_spaces.sub(' ', ans)
class Metadata(object):
@@ -75,7 +62,9 @@ class Metadata(object):
@param authors: List of strings or []
@param other: None or a metadata object
'''
- object.__setattr__(self, '_data', copy.deepcopy(NULL_VALUES))
+ _data = copy.deepcopy(NULL_VALUES)
+ object.__setattr__(self, '_data', _data)
+ _data['_curseq'] = _data['_compseq'] = 0
if other is not None:
self.smart_update(other)
else:
@@ -98,14 +87,28 @@ class Metadata(object):
pass
if field in _data['user_metadata'].iterkeys():
d = _data['user_metadata'][field]
- if d['datatype'] != 'composite':
- return d['#value#']
- return format_composite(d['display']['composite_template'], self)
+ val = d['#value#']
+ if d['datatype'] != 'composite' or \
+ (_data['_curseq'] == _data['_compseq'] and val is not None):
+ return val
+ # Data in the structure has changed. Recompute the composite fields
+ _data['_compseq'] = _data['_curseq']
+ for ck in _data['user_metadata']:
+ cf = _data['user_metadata'][ck]
+ if cf['datatype'] != 'composite':
+ continue
+ cf['#value#'] = 'RECURSIVE_COMPOSITE FIELD ' + field
+ cf['#value#'] = composite_formatter.safe_format(
+ d['display']['composite_template'],
+ self, _('TEMPLATE ERROR')).strip()
+ return d['#value#']
+
raise AttributeError(
'Metadata object has no attribute named: '+ repr(field))
def __setattr__(self, field, val, extra=None):
_data = object.__getattribute__(self, '_data')
+ _data['_curseq'] += 1
if field in TOP_LEVEL_CLASSIFIERS:
_data['classifiers'].update({field: val})
elif field in STANDARD_METADATA_FIELDS:
@@ -193,7 +196,7 @@ class Metadata(object):
if v is not None:
result[attr] = v
for attr in _data['user_metadata'].iterkeys():
- v = _data['user_metadata'][attr]['#value#']
+ v = self.get(attr, None)
if v is not None:
result[attr] = v
if _data['user_metadata'][attr]['datatype'] == 'series':
@@ -466,9 +469,6 @@ class Metadata(object):
return (None, None, None, None)
- def expand_template(self, template):
- return format_composite(template, self)
-
def __unicode__(self):
from calibre.ebooks.metadata import authors_to_string
ans = []
diff --git a/src/calibre/gui2/custom_column_widgets.py b/src/calibre/gui2/custom_column_widgets.py
index d16233be1a..90abfc2474 100644
--- a/src/calibre/gui2/custom_column_widgets.py
+++ b/src/calibre/gui2/custom_column_widgets.py
@@ -351,6 +351,8 @@ def populate_metadata_page(layout, db, book_id, bulk=False, two_column=False, pa
if not x[col]['editable']:
continue
dt = x[col]['datatype']
+ if dt == 'composite':
+ continue
if dt == 'comments':
continue
w = widget_factory(dt, col)
diff --git a/src/calibre/gui2/device_drivers/configwidget.py b/src/calibre/gui2/device_drivers/configwidget.py
index 3d9c9ab2ee..1d6c84ef7c 100644
--- a/src/calibre/gui2/device_drivers/configwidget.py
+++ b/src/calibre/gui2/device_drivers/configwidget.py
@@ -6,7 +6,9 @@ __docformat__ = 'restructuredtext en'
from PyQt4.Qt import QWidget, QListWidgetItem, Qt, QVariant, SIGNAL
+from calibre.gui2 import error_dialog
from calibre.gui2.device_drivers.configwidget_ui import Ui_ConfigWidget
+from calibre.utils.formatter import validation_formatter
class ConfigWidget(QWidget, Ui_ConfigWidget):
@@ -77,3 +79,16 @@ class ConfigWidget(QWidget, Ui_ConfigWidget):
def use_author_sort(self):
return self.opt_use_author_sort.isChecked()
+
+ def validate(self):
+ print 'here in validate'
+ tmpl = unicode(self.opt_save_template.text())
+ try:
+ validation_formatter.validate(tmpl)
+ return True
+ except Exception, err:
+ error_dialog(self, _('Invalid template'),
+ '
'+_('The template %s is invalid:')%tmpl + \
+ ' '+str(err), show=True)
+
+ return False
diff --git a/src/calibre/gui2/library/delegates.py b/src/calibre/gui2/library/delegates.py
index bf233b1175..ceb1cf14a8 100644
--- a/src/calibre/gui2/library/delegates.py
+++ b/src/calibre/gui2/library/delegates.py
@@ -15,10 +15,11 @@ from PyQt4.Qt import QColor, Qt, QModelIndex, QSize, \
QStyledItemDelegate, QCompleter, \
QComboBox
-from calibre.gui2 import UNDEFINED_QDATE
+from calibre.gui2 import UNDEFINED_QDATE, error_dialog
from calibre.gui2.widgets import EnLineEdit, TagsLineEdit
from calibre.utils.date import now, format_date
from calibre.utils.config import tweaks
+from calibre.utils.formatter import validation_formatter
from calibre.gui2.dialogs.comments_dialog import CommentsDialog
class RatingDelegate(QStyledItemDelegate): # {{{
@@ -303,6 +304,31 @@ class CcBoolDelegate(QStyledItemDelegate): # {{{
val = 2 if val is None else 1 if not val else 0
editor.setCurrentIndex(val)
+class CcTemplateDelegate(QStyledItemDelegate): # {{{
+ def __init__(self, parent):
+ '''
+ Delegate for custom_column bool data.
+ '''
+ QStyledItemDelegate.__init__(self, parent)
+
+ def createEditor(self, parent, option, index):
+ return EnLineEdit(parent)
+
+ def setModelData(self, editor, model, index):
+ val = unicode(editor.text())
+ try:
+ validation_formatter.validate(val)
+ except Exception, err:
+ error_dialog(self.parent(), _('Invalid template'),
+ '
'+_('The template %s is invalid:')%val + \
+ ' '+str(err), show=True)
+ model.setData(index, QVariant(val), Qt.EditRole)
+
+ def setEditorData(self, editor, index):
+ m = index.model()
+ val = m.custom_columns[m.column_map[index.column()]]['display']['composite_template']
+ editor.setText(val)
+
# }}}
diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index 4b1e974b12..fe64a33c47 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -696,7 +696,8 @@ class BooksModel(QAbstractTableModel): # {{{
return flags
def set_custom_column_data(self, row, colhead, value):
- typ = self.custom_columns[colhead]['datatype']
+ cc = self.custom_columns[colhead]
+ typ = cc['datatype']
label=self.db.field_metadata.key_to_label(colhead)
s_index = None
if typ in ('text', 'comments'):
@@ -722,6 +723,14 @@ class BooksModel(QAbstractTableModel): # {{{
val = qt_to_dt(val, as_utc=False)
elif typ == 'series':
val, s_index = parse_series_string(self.db, label, value.toString())
+ elif typ == 'composite':
+ tmpl = unicode(value.toString()).strip()
+ disp = cc['display']
+ disp['composite_template'] = tmpl
+ self.db.set_custom_column_metadata(cc['colnum'], display = disp)
+ self.refresh(reset=True)
+ return True
+
self.db.set_custom(self.db.id(row), val, extra=s_index,
label=label, num=None, append=False, notify=True)
return True
@@ -768,6 +777,7 @@ class BooksModel(QAbstractTableModel): # {{{
self.db.set_pubdate(id, qt_to_dt(val, as_utc=False))
else:
self.db.set(row, column, val)
+ self.refresh_rows([row], row)
self.dataChanged.emit(index, index)
return True
diff --git a/src/calibre/gui2/library/views.py b/src/calibre/gui2/library/views.py
index d3ead429cf..b113866ecc 100644
--- a/src/calibre/gui2/library/views.py
+++ b/src/calibre/gui2/library/views.py
@@ -13,7 +13,7 @@ from PyQt4.Qt import QTableView, Qt, QAbstractItemView, QMenu, pyqtSignal, \
from calibre.gui2.library.delegates import RatingDelegate, PubDateDelegate, \
TextDelegate, DateDelegate, TagsDelegate, CcTextDelegate, \
- CcBoolDelegate, CcCommentsDelegate, CcDateDelegate
+ CcBoolDelegate, CcCommentsDelegate, CcDateDelegate, CcTemplateDelegate
from calibre.gui2.library.models import BooksModel, DeviceBooksModel
from calibre.utils.config import tweaks, prefs
from calibre.gui2 import error_dialog, gprefs
@@ -47,6 +47,7 @@ class BooksView(QTableView): # {{{
self.cc_text_delegate = CcTextDelegate(self)
self.cc_bool_delegate = CcBoolDelegate(self)
self.cc_comments_delegate = CcCommentsDelegate(self)
+ self.cc_template_delegate = CcTemplateDelegate(self)
self.display_parent = parent
self._model = modelcls(self)
self.setModel(self._model)
@@ -392,8 +393,7 @@ class BooksView(QTableView): # {{{
elif cc['datatype'] == 'rating':
self.setItemDelegateForColumn(cm.index(colhead), self.rating_delegate)
elif cc['datatype'] == 'composite':
- pass
- # no delegate for composite columns, as they are not editable
+ self.setItemDelegateForColumn(cm.index(colhead), self.cc_template_delegate)
else:
dattr = colhead+'_delegate'
delegate = colhead if hasattr(self, dattr) else 'text'
diff --git a/src/calibre/gui2/preferences/columns.py b/src/calibre/gui2/preferences/columns.py
index 761a9880b1..c1b9230f42 100644
--- a/src/calibre/gui2/preferences/columns.py
+++ b/src/calibre/gui2/preferences/columns.py
@@ -155,8 +155,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
name=self.custcols[c]['name'],
datatype=self.custcols[c]['datatype'],
is_multiple=self.custcols[c]['is_multiple'],
- display = self.custcols[c]['display'],
- editable = self.custcols[c]['editable'])
+ display = self.custcols[c]['display'])
must_restart = True
elif '*deleteme' in self.custcols[c]:
db.delete_custom_column(label=self.custcols[c]['label'])
diff --git a/src/calibre/gui2/preferences/create_custom_column.py b/src/calibre/gui2/preferences/create_custom_column.py
index e88949a23c..bec21270df 100644
--- a/src/calibre/gui2/preferences/create_custom_column.py
+++ b/src/calibre/gui2/preferences/create_custom_column.py
@@ -156,9 +156,6 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
return self.simple_error('', _('You must enter a template for'
' composite columns'))
display_dict = {'composite_template':unicode(self.composite_box.text())}
- is_editable = False
- else:
- is_editable = True
db = self.parent.gui.library_view.model().db
key = db.field_metadata.custom_field_prefix+col
@@ -168,7 +165,6 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
'label':col,
'name':col_heading,
'datatype':col_type,
- 'editable':is_editable,
'display':display_dict,
'normalized':None,
'colnum':None,
diff --git a/src/calibre/gui2/preferences/plugins.py b/src/calibre/gui2/preferences/plugins.py
index a26553db1c..388227e438 100644
--- a/src/calibre/gui2/preferences/plugins.py
+++ b/src/calibre/gui2/preferences/plugins.py
@@ -199,7 +199,11 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
config_dialog.exec_()
if config_dialog.result() == QDialog.Accepted:
- plugin.save_settings(config_widget)
+ if hasattr(config_widget, 'validate'):
+ if config_widget.validate():
+ plugin.save_settings(config_widget)
+ else:
+ plugin.save_settings(config_widget)
self._plugin_model.refresh_plugin(plugin)
else:
help_text = plugin.customization_help(gui=True)
diff --git a/src/calibre/gui2/preferences/save_template.py b/src/calibre/gui2/preferences/save_template.py
index 0f48893b69..5b3f0321b2 100644
--- a/src/calibre/gui2/preferences/save_template.py
+++ b/src/calibre/gui2/preferences/save_template.py
@@ -13,17 +13,8 @@ from PyQt4.Qt import QWidget, pyqtSignal
from calibre.gui2 import error_dialog
from calibre.gui2.preferences.save_template_ui import Ui_Form
from calibre.library.save_to_disk import FORMAT_ARG_DESCS, preprocess_template
+from calibre.utils.formatter import validation_formatter
-class ValidateFormat(string.Formatter):
- '''
- Provides a format function that substitutes '' for any missing value
- '''
- def get_value(self, key, args, kwargs):
- return 'this is some text that should be long enough'
-
-validate_formatter = ValidateFormat()
-def validate_format(x, format_args):
- return validate_formatter.vformat(x, [], format_args).strip()
class SaveTemplate(QWidget, Ui_Form):
@@ -62,9 +53,8 @@ class SaveTemplate(QWidget, Ui_Form):
custom fields, because they may or may not exist.
'''
tmpl = preprocess_template(self.opt_template.text())
- fa = {}
try:
- validate_format(tmpl, fa)
+ validation_formatter.validate(tmpl)
except Exception, err:
error_dialog(self, _('Invalid template'),
'
'+_('The template %s is invalid:')%tmpl + \
diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index 42feb6f8fa..7849eecb2e 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -546,7 +546,7 @@ class ResultCache(SearchQueryParser):
if len(self.composites) > 0:
mi = db.get_metadata(id, index_is_id=True)
for k,c in self.composites:
- self._data[id][c] = mi.format_field(k)[1]
+ self._data[id][c] = mi.get(k, None)
except IndexError:
return None
try:
diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py
index 90e5413389..a0f739e4c2 100644
--- a/src/calibre/library/save_to_disk.py
+++ b/src/calibre/library/save_to_disk.py
@@ -9,6 +9,7 @@ __docformat__ = 'restructuredtext en'
import os, traceback, cStringIO, re, string
from calibre.utils.config import Config, StringConfig, tweaks
+from calibre.utils.formatter import TemplateFormatter
from calibre.utils.filenames import shorten_components_to, supports_long_names, \
ascii_filename, sanitize_file_name
from calibre.ebooks.metadata.opf2 import metadata_to_opf
@@ -101,40 +102,20 @@ def preprocess_template(template):
template = template.decode(preferred_encoding, 'replace')
return template
-template_value_re = re.compile(r'^([^\|]*(?=\|))(?:\|?)([^\|]*)(?:\|?)((?<=\|).*?)$',
- flags= re.UNICODE)
-
-def explode_string_template_value(key):
- try:
- matches = template_value_re.match(key)
- if matches.lastindex != 3:
- return key
- return matches.groups()
- except:
- return '', key, ''
-
-class SafeFormat(string.Formatter):
+class SafeFormat(TemplateFormatter):
'''
Provides a format function that substitutes '' for any missing value
'''
def get_value(self, key, args, kwargs):
try:
- prefix, key, suffix = explode_string_template_value(key)
if kwargs[key]:
- return prefix + unicode(kwargs[key]) + suffix
+ return kwargs[key]
return ''
except:
return ''
safe_formatter = SafeFormat()
-def safe_format(x, format_args):
- try:
- ans = safe_formatter.vformat(x, [], format_args).strip()
- except:
- ans = ''
- return re.sub(r'\s+', ' ', ans)
-
def get_components(template, mi, id, timefmt='%b %Y', length=250,
sanitize_func=ascii_filename, replace_whitespace=False,
to_lowercase=False):
@@ -178,8 +159,8 @@ def get_components(template, mi, id, timefmt='%b %Y', length=250,
elif custom_metadata[key]['datatype'] == 'bool':
format_args[key] = _('yes') if format_args[key] else _('no')
- components = [x.strip() for x in template.split('/') if x.strip()]
- components = [safe_format(x, format_args) for x in components]
+ components = safe_formatter.safe_format(template, format_args, '')
+ components = [x.strip() for x in components.split('/') if x.strip()]
components = [sanitize_func(x) for x in components if x]
if not components:
components = [str(id)]
diff --git a/src/calibre/utils/formatter.py b/src/calibre/utils/formatter.py
new file mode 100644
index 0000000000..f9ef4e0846
--- /dev/null
+++ b/src/calibre/utils/formatter.py
@@ -0,0 +1,113 @@
+'''
+Created on 23 Sep 2010
+
+@author: charles
+'''
+
+import re, string
+
+def _lookup(val, mi, field_if_set, field_not_set):
+ if hasattr(mi, 'format_field'):
+ if val:
+ return mi.format_field(field_if_set.strip())[1]
+ else:
+ return mi.format_field(field_not_set.strip())[1]
+ else:
+ if val:
+ return mi.get(field_if_set.strip(), '')
+ else:
+ return mi.get(field_not_set.strip(), '')
+
+def _ifempty(val, mi, value_if_empty):
+ if val:
+ return val
+ else:
+ return value_if_empty
+
+def _shorten(val, mi, leading, center_string, trailing):
+ l = int(leading)
+ t = int(trailing)
+ if len(val) > l + len(center_string) + t:
+ return val[0:l] + center_string + val[-t:]
+ else:
+ return val
+
+class TemplateFormatter(string.Formatter):
+ '''
+ Provides a format function that substitutes '' for any missing value
+ '''
+
+ functions = {
+ 'uppercase' : (0, lambda x: x.upper()),
+ 'lowercase' : (0, lambda x: x.lower()),
+ 'titlecase' : (0, lambda x: x.title()),
+ 'capitalize' : (0, lambda x: x.capitalize()),
+ 'ifempty' : (1, _ifempty),
+ 'lookup' : (2, _lookup),
+ 'shorten' : (3, _shorten),
+ }
+
+ def get_value(self, key, args, mi):
+ raise Exception('get_value must be implemented in the subclass')
+
+ format_string_re = re.compile(r'^(.*)\|(.*)\|(.*)$')
+
+ def _explode_format_string(self, fmt):
+ try:
+ matches = self.format_string_re.match(fmt)
+ if matches is None or matches.lastindex != 3:
+ return fmt, '', ''
+ return matches.groups()
+ except:
+ import traceback
+ traceback.print_exc()
+ return fmt, '', ''
+
+ def format_field(self, val, fmt):
+ fmt, prefix, suffix = self._explode_format_string(fmt)
+
+ p = fmt.find('(')
+ if p >= 0 and fmt[-1] == ')' and fmt[0:p] in self.functions:
+ field = fmt[0:p]
+ func = self.functions[field]
+ args = fmt[p+1:-1].split(',')
+ if (func[0] == 0 and (len(args) != 1 or args[0])) or \
+ (func[0] > 0 and func[0] != len(args)):
+ raise Exception ('Incorrect number of arguments for function '+ fmt[0:p])
+ if func[0] == 0:
+ val = func[1](val, self.mi)
+ else:
+ val = func[1](val, self.mi, *args)
+ else:
+ val = string.Formatter.format_field(self, val, fmt)
+ if not val:
+ return ''
+ return prefix + val + suffix
+
+ compress_spaces = re.compile(r'\s+')
+
+ def vformat(self, fmt, args, kwargs):
+ self.mi = kwargs
+ ans = string.Formatter.vformat(self, fmt, args, kwargs)
+ return self.compress_spaces.sub(' ', ans).strip()
+
+ def safe_format(self, fmt, kwargs, error_value):
+ try:
+ ans = self.vformat(fmt, [], kwargs).strip()
+ except:
+ ans = error_value
+ return ans
+
+class ValidateFormat(TemplateFormatter):
+ '''
+ Provides a format function that substitutes '' for any missing value
+ '''
+ def get_value(self, key, args, kwargs):
+ return 'this is some text that should be long enough'
+
+ def validate(self, x):
+ return self.vformat(x, [], {})
+
+validation_formatter = ValidateFormat()
+
+
From 30c96df50546e9730ad1903ac31e54a05d09f723 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Thu, 23 Sep 2010 08:13:51 -0600
Subject: [PATCH 094/207] ...
---
src/calibre/utils/pyconsole/history.py | 56 ++++++++++++++++++++++++++
1 file changed, 56 insertions(+)
create mode 100644 src/calibre/utils/pyconsole/history.py
diff --git a/src/calibre/utils/pyconsole/history.py b/src/calibre/utils/pyconsole/history.py
new file mode 100644
index 0000000000..5440e57153
--- /dev/null
+++ b/src/calibre/utils/pyconsole/history.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+__license__ = 'GPL v3'
+__copyright__ = '2010, Kovid Goyal '
+__docformat__ = 'restructuredtext en'
+
+from collections import deque
+
+class History(object): # {{{
+
+ def __init__(self, current, entries):
+ self.entries = deque(entries, maxlen=max(2000, len(entries)))
+ self.index = len(self.entries) - 1
+ self.current = self.default = current
+ self.last_was_back = False
+
+ def back(self, amt=1):
+ if self.entries:
+ oidx = self.index
+ ans = self.entries[self.index]
+ self.index = max(0, self.index - amt)
+ self.last_was_back = self.index != oidx
+ return ans
+
+ def forward(self, amt=1):
+ if self.entries:
+ d = self.index
+ if self.last_was_back:
+ d += 1
+ if d >= len(self.entries) - 1:
+ self.index = len(self.entries) - 1
+ self.last_was_back = False
+ return self.current
+ if self.last_was_back:
+ amt += 1
+ self.index = min(len(self.entries)-1, self.index + amt)
+ self.last_was_back = False
+ return self.entries[self.index]
+
+ def enter(self, x):
+ try:
+ self.entries.remove(x)
+ except ValueError:
+ pass
+ self.entries.append(x)
+ self.index = len(self.entries) - 1
+ self.current = self.default
+ self.last_was_back = False
+
+ def serialize(self):
+ return list(self.entries)
+
+# }}}
+
+
From 36ce8740816958c064a10334df0cb50e36f4784c Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Thu, 23 Sep 2010 17:18:49 +0100
Subject: [PATCH 095/207] Fix db2.get_metadata to handle format correctly (it
is already a list) Fix Metadata to put composite fields back where they
belong
---
src/calibre/ebooks/metadata/book/base.py | 2 +-
src/calibre/library/database2.py | 4 +---
2 files changed, 2 insertions(+), 4 deletions(-)
diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index 16819cbd39..2bbe76488e 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -99,7 +99,7 @@ class Metadata(object):
continue
cf['#value#'] = 'RECURSIVE_COMPOSITE FIELD ' + field
cf['#value#'] = composite_formatter.safe_format(
- d['display']['composite_template'],
+ cf['display']['composite_template'],
self, _('TEMPLATE ERROR')).strip()
return d['#value#']
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index fd5809f937..fde57e2a2e 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -584,9 +584,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
mi.title_sort = self.title_sort(idx, index_is_id=index_is_id)
mi.formats = self.formats(idx, index_is_id=index_is_id,
verify_formats=False)
- if hasattr(mi.formats, 'split'):
- mi.formats = mi.formats.split(',')
- else:
+ if len(mi.formats) == 0:
mi.formats = None
tags = self.tags(idx, index_is_id=index_is_id)
if tags:
From b2d5f740b5268d369941e337c78c5838060caa98 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Thu, 23 Sep 2010 17:46:46 +0100
Subject: [PATCH 096/207] 1) Put back get_metadata code for format, and fix
format. 2) Ensure that gui editing does an lcase.
---
src/calibre/gui2/library/models.py | 2 +-
src/calibre/library/database2.py | 6 ++++--
2 files changed, 5 insertions(+), 3 deletions(-)
diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index fe64a33c47..bab2a59b1c 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -724,7 +724,7 @@ class BooksModel(QAbstractTableModel): # {{{
elif typ == 'series':
val, s_index = parse_series_string(self.db, label, value.toString())
elif typ == 'composite':
- tmpl = unicode(value.toString()).strip()
+ tmpl = unicode(value.toString()).lower().strip()
disp = cc['display']
disp['composite_template'] = tmpl
self.db.set_custom_column_metadata(cc['colnum'], display = disp)
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index fde57e2a2e..22de8df41f 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -584,7 +584,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
mi.title_sort = self.title_sort(idx, index_is_id=index_is_id)
mi.formats = self.formats(idx, index_is_id=index_is_id,
verify_formats=False)
- if len(mi.formats) == 0:
+ if hasattr(mi.formats, 'split'):
+ mi.formats = mi.formats.split(',')
+ else:
mi.formats = None
tags = self.tags(idx, index_is_id=index_is_id)
if tags:
@@ -731,7 +733,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
except:
return None
if not verify_formats:
- return formats
+ return ','.join(formats)
ans = []
for format in formats:
if self.format_abspath(id, format, index_is_id=True) is not None:
From 232ce4748ddcf03fc21007cc3a1d5ea72e01ce64 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Thu, 23 Sep 2010 17:59:50 +0100
Subject: [PATCH 097/207] Back out the models 'strip' change
---
src/calibre/ebooks/metadata/book/base.py | 2 +-
src/calibre/gui2/library/models.py | 2 +-
src/calibre/library/save_to_disk.py | 4 ++--
3 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index 2bbe76488e..9e1085df25 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -36,7 +36,7 @@ field_metadata = FieldMetadata()
class SafeFormat(TemplateFormatter):
def get_value(self, key, args, mi):
try:
- ign, v = mi.format_field(key, series_with_index=False)
+ ign, v = mi.format_field(key.lower(), series_with_index=False)
if v is None:
return ''
if v == '':
diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index bab2a59b1c..fe64a33c47 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -724,7 +724,7 @@ class BooksModel(QAbstractTableModel): # {{{
elif typ == 'series':
val, s_index = parse_series_string(self.db, label, value.toString())
elif typ == 'composite':
- tmpl = unicode(value.toString()).lower().strip()
+ tmpl = unicode(value.toString()).strip()
disp = cc['display']
disp['composite_template'] = tmpl
self.db.set_custom_column_metadata(cc['colnum'], display = disp)
diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py
index a0f739e4c2..11922b7154 100644
--- a/src/calibre/library/save_to_disk.py
+++ b/src/calibre/library/save_to_disk.py
@@ -108,8 +108,8 @@ class SafeFormat(TemplateFormatter):
'''
def get_value(self, key, args, kwargs):
try:
- if kwargs[key]:
- return kwargs[key]
+ if kwargs[key.lower()]:
+ return kwargs[key.lower()]
return ''
except:
return ''
From 1a782eb0ffa9ca7bfc063902b35b06f6e8399271 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Thu, 23 Sep 2010 20:36:52 +0100
Subject: [PATCH 098/207] - Some cleanups on templates. - Make save_to_disk
templates sanitize all fields except composites
---
src/calibre/ebooks/metadata/book/base.py | 9 ++-
src/calibre/library/save_to_disk.py | 12 ++--
src/calibre/utils/formatter.py | 72 ++++++++++++------------
3 files changed, 51 insertions(+), 42 deletions(-)
diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index 9e1085df25..929dc01aec 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -34,9 +34,10 @@ NULL_VALUES = {
field_metadata = FieldMetadata()
class SafeFormat(TemplateFormatter):
- def get_value(self, key, args, mi):
+
+ def get_value(self, key, args, kwargs):
try:
- ign, v = mi.format_field(key.lower(), series_with_index=False)
+ ign, v = self.book.format_field(key.lower(), series_with_index=False)
if v is None:
return ''
if v == '':
@@ -100,7 +101,9 @@ class Metadata(object):
cf['#value#'] = 'RECURSIVE_COMPOSITE FIELD ' + field
cf['#value#'] = composite_formatter.safe_format(
cf['display']['composite_template'],
- self, _('TEMPLATE ERROR')).strip()
+ self,
+ _('TEMPLATE ERROR'),
+ self).strip()
return d['#value#']
raise AttributeError(
diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py
index 11922b7154..2d0a3d1277 100644
--- a/src/calibre/library/save_to_disk.py
+++ b/src/calibre/library/save_to_disk.py
@@ -108,8 +108,12 @@ class SafeFormat(TemplateFormatter):
'''
def get_value(self, key, args, kwargs):
try:
- if kwargs[key.lower()]:
- return kwargs[key.lower()]
+ b = self.book.get_user_metadata(key, False)
+ key = key.lower()
+ if b is not None and b['datatype'] == 'composite':
+ return self.vformat(b['display']['composite_template'], [], kwargs)
+ if kwargs[key]:
+ return self.sanitize(kwargs[key.lower()])
return ''
except:
return ''
@@ -159,9 +163,9 @@ def get_components(template, mi, id, timefmt='%b %Y', length=250,
elif custom_metadata[key]['datatype'] == 'bool':
format_args[key] = _('yes') if format_args[key] else _('no')
- components = safe_formatter.safe_format(template, format_args, '')
+ components = safe_formatter.safe_format(template, format_args, '', mi,
+ sanitize=sanitize_func)
components = [x.strip() for x in components.split('/') if x.strip()]
- components = [sanitize_func(x) for x in components if x]
if not components:
components = [str(id)]
components = [x.encode(filesystem_encoding, 'replace') if isinstance(x,
diff --git a/src/calibre/utils/formatter.py b/src/calibre/utils/formatter.py
index f9ef4e0846..95870d9c61 100644
--- a/src/calibre/utils/formatter.py
+++ b/src/calibre/utils/formatter.py
@@ -6,48 +6,48 @@ Created on 23 Sep 2010
import re, string
-def _lookup(val, mi, field_if_set, field_not_set):
- if hasattr(mi, 'format_field'):
- if val:
- return mi.format_field(field_if_set.strip())[1]
- else:
- return mi.format_field(field_not_set.strip())[1]
- else:
- if val:
- return mi.get(field_if_set.strip(), '')
- else:
- return mi.get(field_not_set.strip(), '')
-
-def _ifempty(val, mi, value_if_empty):
- if val:
- return val
- else:
- return value_if_empty
-
-def _shorten(val, mi, leading, center_string, trailing):
- l = int(leading)
- t = int(trailing)
- if len(val) > l + len(center_string) + t:
- return val[0:l] + center_string + val[-t:]
- else:
- return val
-
class TemplateFormatter(string.Formatter):
'''
Provides a format function that substitutes '' for any missing value
'''
+ def __init__(self):
+ string.Formatter.__init__(self)
+ self.book = None
+ self.kwargs = None
+ self.sanitize = None
+
+ def _lookup(self, val, field_if_set, field_not_set):
+ if val:
+ return self.vformat('{'+field_if_set.strip()+'}', [], self.kwargs)
+ else:
+ return self.vformat('{'+field_not_set.strip()+'}', [], self.kwargs)
+
+ def _ifempty(self, val, value_if_empty):
+ if val:
+ return val
+ else:
+ return value_if_empty
+
+ def _shorten(self, val, leading, center_string, trailing):
+ l = int(leading)
+ t = int(trailing)
+ if len(val) > l + len(center_string) + t:
+ return val[0:l] + center_string + val[-t:]
+ else:
+ return val
+
functions = {
- 'uppercase' : (0, lambda x: x.upper()),
- 'lowercase' : (0, lambda x: x.lower()),
- 'titlecase' : (0, lambda x: x.title()),
- 'capitalize' : (0, lambda x: x.capitalize()),
+ 'uppercase' : (0, lambda s,x: x.upper()),
+ 'lowercase' : (0, lambda s,x: x.lower()),
+ 'titlecase' : (0, lambda s,x: x.title()),
+ 'capitalize' : (0, lambda s,x: x.capitalize()),
'ifempty' : (1, _ifempty),
'lookup' : (2, _lookup),
'shorten' : (3, _shorten),
}
- def get_value(self, key, args, mi):
+ def get_value(self, key, args):
raise Exception('get_value must be implemented in the subclass')
format_string_re = re.compile(r'^(.*)\|(.*)\|(.*)$')
@@ -75,9 +75,9 @@ class TemplateFormatter(string.Formatter):
(func[0] > 0 and func[0] != len(args)):
raise Exception ('Incorrect number of arguments for function '+ fmt[0:p])
if func[0] == 0:
- val = func[1](val, self.mi)
+ val = func[1](self, val)
else:
- val = func[1](val, self.mi, *args)
+ val = func[1](self, val, *args)
else:
val = string.Formatter.format_field(self, val, fmt)
if not val:
@@ -87,11 +87,13 @@ class TemplateFormatter(string.Formatter):
compress_spaces = re.compile(r'\s+')
def vformat(self, fmt, args, kwargs):
- self.mi = kwargs
ans = string.Formatter.vformat(self, fmt, args, kwargs)
return self.compress_spaces.sub(' ', ans).strip()
- def safe_format(self, fmt, kwargs, error_value):
+ def safe_format(self, fmt, kwargs, error_value, book, sanitize=None):
+ self.kwargs = kwargs
+ self.book = book
+ self.sanitize = sanitize
try:
ans = self.vformat(fmt, [], kwargs).strip()
except:
From 1ad0eebd5658c73913dc0ef4b73a95ae0c8960a5 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Thu, 23 Sep 2010 23:30:16 -0600
Subject: [PATCH 099/207] API for dealing with distributed metadata backup
---
src/calibre/library/custom_columns.py | 5 ++-
src/calibre/library/database2.py | 47 +++++++++++++++++++++++++-
src/calibre/library/schema_upgrades.py | 12 +++++++
3 files changed, 62 insertions(+), 2 deletions(-)
diff --git a/src/calibre/library/custom_columns.py b/src/calibre/library/custom_columns.py
index d74024280e..2d8634659b 100644
--- a/src/calibre/library/custom_columns.py
+++ b/src/calibre/library/custom_columns.py
@@ -382,6 +382,7 @@ class CustomColumns(object):
)
# get rid of the temp tables
self.conn.executescript(drops)
+ self.dirtied(ids, commit=False)
self.conn.commit()
# set the in-memory copies of the tags
@@ -402,19 +403,21 @@ class CustomColumns(object):
same length as ids.
'''
if extras is not None and len(extras) != len(ids):
- raise ValueError('Lentgh of ids and extras is not the same')
+ raise ValueError('Length of ids and extras is not the same')
ev = None
for idx,id in enumerate(ids):
if extras is not None:
ev = extras[idx]
self._set_custom(id, val, label=label, num=num, append=append,
notify=notify, extra=ev)
+ self.dirtied(ids, commit=False)
self.conn.commit()
def set_custom(self, id, val, label=None, num=None,
append=False, notify=True, extra=None, commit=True):
self._set_custom(id, val, label=label, num=num, append=append,
notify=notify, extra=extra)
+ self.dirtied([id], commit=False)
if commit:
self.conn.commit()
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index 1fe77077b9..7a8aef541d 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -13,6 +13,7 @@ from math import floor
from PyQt4.QtGui import QImage
from calibre.ebooks.metadata import title_sort, author_to_author_sort
+from calibre.ebooks.metadata.opf2 import metadata_to_opf
from calibre.library.database import LibraryDatabase
from calibre.library.field_metadata import FieldMetadata, TagsIcons
from calibre.library.schema_upgrades import SchemaUpgrade
@@ -126,6 +127,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
def __init__(self, library_path, row_factory=False):
self.field_metadata = FieldMetadata()
+ self.dirtied_cache = set([])
if not os.path.exists(library_path):
os.makedirs(library_path)
self.listeners = set([])
@@ -337,6 +339,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
setattr(self, 'title_sort', functools.partial(self.get_property,
loc=self.FIELD_MAP['sort']))
+ d = self.conn.get('SELECT book FROM metadata_dirtied', all=True)
+ self.dirtied_cache.update(set([x[0] for x in d]))
+
self.refresh_ondevice = functools.partial(self.data.refresh_ondevice, self)
self.refresh()
self.last_update_check = self.last_modified()
@@ -550,6 +555,33 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
def metadata_for_field(self, key):
return self.field_metadata[key]
+ def dump_metadata(self, book_ids, remove_from_dirtied=True, commit=True):
+ for book_id in book_ids:
+ mi = self.get_metadata(book_id, index_is_id=True, get_cover=True)
+ # Always set cover to cover.jpg. Even if cover doesn't exist,
+ # no harm done. This way no need to call dirtied when
+ # cover is set/removed
+ mi.cover = 'cover.jpg'
+ raw = metadata_to_opf(mi)
+ path = self.abspath(book_id, index_is_id=True)
+ with open(os.path.join(path, 'metadata.opf'), 'wb') as f:
+ f.write(raw)
+ if remove_from_dirtied:
+ self.conn.execute('DELETE FROM metadata_dirtied WHERE book=?',
+ (book_id,))
+ if book_id in self.dirtied_cache:
+ self.dirtied_cache.remove(book_id)
+ if commit:
+ self.conn.commit()
+
+ def dirtied(self, book_ids, commit=True):
+ self.conn.executemany(
+ 'INSERT OR REPLACE INTO metadata_dirtied VALUES (?)',
+ [(x,) for x in book_ids])
+ if commit:
+ self.conn.commit()
+ self.dirtied.update(set(book_ids))
+
def get_metadata(self, idx, index_is_id=False, get_cover=False):
'''
Convenience method to return metadata as a :class:`Metadata` object.
@@ -583,7 +615,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
mi.uuid = self.uuid(idx, index_is_id=index_is_id)
mi.title_sort = self.title_sort(idx, index_is_id=index_is_id)
mi.formats = self.formats(idx, index_is_id=index_is_id,
- verify_formats=False)
+ verify_formats=False)
if hasattr(mi.formats, 'split'):
mi.formats = mi.formats.split(',')
else:
@@ -1242,6 +1274,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
ss = self.author_sort_from_book(id, index_is_id=True)
self.conn.execute('UPDATE books SET author_sort=? WHERE id=?',
(ss, id))
+ self.dirtied([id], commit=False)
if commit:
self.conn.commit()
self.data.set(id, self.FIELD_MAP['authors'],
@@ -1268,6 +1301,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
else:
self.data.set(id, self.FIELD_MAP['sort'], title, row_is_id=True)
self.set_path(id, index_is_id=True)
+ self.dirtied([id], commit=False)
if commit:
self.conn.commit()
if notify:
@@ -1277,6 +1311,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
if dt:
self.conn.execute('UPDATE books SET timestamp=? WHERE id=?', (dt, id))
self.data.set(id, self.FIELD_MAP['timestamp'], dt, row_is_id=True)
+ self.dirtied([id], commit=False)
if commit:
self.conn.commit()
if notify:
@@ -1286,6 +1321,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
if dt:
self.conn.execute('UPDATE books SET pubdate=? WHERE id=?', (dt, id))
self.data.set(id, self.FIELD_MAP['pubdate'], dt, row_is_id=True)
+ self.dirtied([id], commit=False)
if commit:
self.conn.commit()
if notify:
@@ -1304,6 +1340,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
else:
aid = self.conn.execute('INSERT INTO publishers(name) VALUES (?)', (publisher,)).lastrowid
self.conn.execute('INSERT INTO books_publishers_link(book, publisher) VALUES (?,?)', (id, aid))
+ self.dirtied([id], commit=False)
if commit:
self.conn.commit()
self.data.set(id, self.FIELD_MAP['publisher'], publisher, row_is_id=True)
@@ -1594,6 +1631,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
'''.format(tables[0], tables[1])
)
self.conn.executescript(drops)
+ self.dirtied(ids, commit=False)
self.conn.commit()
for x in ids:
@@ -1639,6 +1677,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
(id, tid), all=False):
self.conn.execute('INSERT INTO books_tags_link(book, tag) VALUES (?,?)',
(id, tid))
+ self.dirtied([id], commit=False)
if commit:
self.conn.commit()
tags = u','.join(self.get_tags(id))
@@ -1693,6 +1732,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
else:
aid = self.conn.execute('INSERT INTO series(name) VALUES (?)', (series,)).lastrowid
self.conn.execute('INSERT INTO books_series_link(book, series) VALUES (?,?)', (id, aid))
+ self.dirtied([id], commit=False)
if commit:
self.conn.commit()
self.data.set(id, self.FIELD_MAP['series'], series, row_is_id=True)
@@ -1707,6 +1747,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
except:
idx = 1.0
self.conn.execute('UPDATE books SET series_index=? WHERE id=?', (idx, id))
+ self.dirtied([id], commit=False)
if commit:
self.conn.commit()
self.data.set(id, self.FIELD_MAP['series_index'], idx, row_is_id=True)
@@ -1719,6 +1760,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
rat = self.conn.get('SELECT id FROM ratings WHERE rating=?', (rating,), all=False)
rat = rat if rat else self.conn.execute('INSERT INTO ratings(rating) VALUES (?)', (rating,)).lastrowid
self.conn.execute('INSERT INTO books_ratings_link(book, rating) VALUES (?,?)', (id, rat))
+ self.dirtied([id], commit=False)
if commit:
self.conn.commit()
self.data.set(id, self.FIELD_MAP['rating'], rating, row_is_id=True)
@@ -1731,11 +1773,13 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
if commit:
self.conn.commit()
self.data.set(id, self.FIELD_MAP['comments'], text, row_is_id=True)
+ self.dirtied([id], commit=False)
if notify:
self.notify('metadata', [id])
def set_author_sort(self, id, sort, notify=True, commit=True):
self.conn.execute('UPDATE books SET author_sort=? WHERE id=?', (sort, id))
+ self.dirtied([id], commit=False)
if commit:
self.conn.commit()
self.data.set(id, self.FIELD_MAP['author_sort'], sort, row_is_id=True)
@@ -1744,6 +1788,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
def set_isbn(self, id, isbn, notify=True, commit=True):
self.conn.execute('UPDATE books SET isbn=? WHERE id=?', (isbn, id))
+ self.dirtied([id], commit=False)
if commit:
self.conn.commit()
self.data.set(id, self.FIELD_MAP['isbn'], isbn, row_is_id=True)
diff --git a/src/calibre/library/schema_upgrades.py b/src/calibre/library/schema_upgrades.py
index b08161abf2..167cc0a327 100644
--- a/src/calibre/library/schema_upgrades.py
+++ b/src/calibre/library/schema_upgrades.py
@@ -397,3 +397,15 @@ class SchemaUpgrade(object):
UNIQUE(key));
'''
self.conn.executescript(script)
+
+ def upgrade_version_13(self):
+ 'Dirtied table for OPF metadata backups'
+ script = '''
+ DROP TABLE IF EXISTS metadata_dirtied;
+ CREATE TABLE metadata_dirtied(id INTEGER PRIMARY KEY,
+ book INTEGER NOT NULL,
+ UNIQUE(book));
+ INSERT INTO metadata_dirtied (book) SELECT id FROM books;
+ '''
+ self.conn.executescript(script)
+
From f46d919c751dbccb24f21c348ca130833ffc5a7a Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Thu, 23 Sep 2010 23:50:22 -0600
Subject: [PATCH 100/207] Add thread to GUI for distributed metadata backup
---
src/calibre/gui2/library/models.py | 5 ++++-
src/calibre/gui2/ui.py | 4 ++++
src/calibre/library/caches.py | 30 ++++++++++++++++++++++++++++--
src/calibre/library/database2.py | 15 +++++++++------
4 files changed, 45 insertions(+), 9 deletions(-)
diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index fe64a33c47..9d9de358c8 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -21,7 +21,7 @@ from calibre.utils.date import dt_factory, qt_to_dt, isoformat
from calibre.ebooks.metadata.meta import set_metadata as _set_metadata
from calibre.utils.search_query_parser import SearchQueryParser
from calibre.library.caches import _match, CONTAINS_MATCH, EQUALS_MATCH, \
- REGEXP_MATCH, CoverCache
+ REGEXP_MATCH, CoverCache, MetadataBackup
from calibre.library.cli import parse_series_string
from calibre import strftime, isbytestring, prepare_string_for_xml
from calibre.constants import filesystem_encoding
@@ -153,6 +153,9 @@ class BooksModel(QAbstractTableModel): # {{{
self.cover_cache.stop()
self.cover_cache = CoverCache(db, FunctionDispatcher(self.db.cover))
self.cover_cache.start()
+ self.metadata_backup = MetadataBackup(db,
+ FunctionDispatcher(self.db.dump_metadata))
+ self.metadata_backup.start()
def refresh_cover(event, ids):
if event == 'cover' and self.cover_cache is not None:
self.cover_cache.refresh(ids)
diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py
index 9bc504a001..88a8c68572 100644
--- a/src/calibre/gui2/ui.py
+++ b/src/calibre/gui2/ui.py
@@ -551,6 +551,10 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, # {{{
cc = self.library_view.model().cover_cache
if cc is not None:
cc.stop()
+ mb = self.library_view.model().metadata_backup
+ if mb is not None:
+ mb.stop()
+
self.hide_windows()
self.emailer.stop()
try:
diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index 7849eecb2e..2d37314896 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -21,7 +21,31 @@ from calibre.utils.pyparsing import ParseException
from calibre.ebooks.metadata import title_sort
from calibre import fit_image
-class CoverCache(Thread):
+class MetadataBackup(Thread): # {{{
+
+ def __init__(self, db, dump_func):
+ Thread.__init__(self)
+ self.daemon = True
+ self.db = db
+ self.dump_func = dump_func
+ self.keep_running = True
+
+ def stop(self):
+ self.keep_running = False
+
+ def run(self):
+ while self.keep_running:
+ try:
+ id_ = self.db.dirtied_queue.get(True, 5)
+ except Empty:
+ continue
+ # If there is an exception is dump_func, we
+ # have no way of knowing
+ self.dump_func([id_])
+
+# }}}
+
+class CoverCache(Thread): # {{{
def __init__(self, db, cover_func):
Thread.__init__(self)
@@ -90,6 +114,7 @@ class CoverCache(Thread):
for id_ in ids:
self.cache.pop(id_, None)
self.load_queue.put(id_)
+# }}}
### Global utility function for get_match here and in gui2/library.py
CONTAINS_MATCH = 0
@@ -107,7 +132,7 @@ def _match(query, value, matchkind):
pass
return False
-class ResultCache(SearchQueryParser):
+class ResultCache(SearchQueryParser): # {{{
'''
Stores sorted and filtered metadata in memory.
@@ -694,4 +719,5 @@ class SortKeyGenerator(object):
# }}}
+# }}}
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index 7a8aef541d..92f8cca0db 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -9,6 +9,7 @@ The database used to store ebook metadata
import os, sys, shutil, cStringIO, glob, time, functools, traceback, re
from itertools import repeat
from math import floor
+from Queue import Queue
from PyQt4.QtGui import QImage
@@ -127,7 +128,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
def __init__(self, library_path, row_factory=False):
self.field_metadata = FieldMetadata()
- self.dirtied_cache = set([])
+ self.dirtied_queue = Queue()
if not os.path.exists(library_path):
os.makedirs(library_path)
self.listeners = set([])
@@ -340,7 +341,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
loc=self.FIELD_MAP['sort']))
d = self.conn.get('SELECT book FROM metadata_dirtied', all=True)
- self.dirtied_cache.update(set([x[0] for x in d]))
+ for x in d:
+ self.dirtied_queue.put(x[0])
self.refresh_ondevice = functools.partial(self.data.refresh_ondevice, self)
self.refresh()
@@ -557,6 +559,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
def dump_metadata(self, book_ids, remove_from_dirtied=True, commit=True):
for book_id in book_ids:
+ if not self.data.has_id(book_id):
+ continue
mi = self.get_metadata(book_id, index_is_id=True, get_cover=True)
# Always set cover to cover.jpg. Even if cover doesn't exist,
# no harm done. This way no need to call dirtied when
@@ -569,18 +573,17 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
if remove_from_dirtied:
self.conn.execute('DELETE FROM metadata_dirtied WHERE book=?',
(book_id,))
- if book_id in self.dirtied_cache:
- self.dirtied_cache.remove(book_id)
if commit:
self.conn.commit()
def dirtied(self, book_ids, commit=True):
self.conn.executemany(
- 'INSERT OR REPLACE INTO metadata_dirtied VALUES (?)',
+ 'INSERT OR REPLACE INTO metadata_dirtied (book) VALUES (?)',
[(x,) for x in book_ids])
if commit:
self.conn.commit()
- self.dirtied.update(set(book_ids))
+ for x in book_ids:
+ self.dirtied_queue.put(x)
def get_metadata(self, idx, index_is_id=False, get_cover=False):
'''
From 703ea3c77827a4053d4c020fa79e7d5e8111f24a Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Thu, 23 Sep 2010 23:55:56 -0600
Subject: [PATCH 101/207] propert indentation in generated OPF files
---
src/calibre/ebooks/metadata/opf2.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/calibre/ebooks/metadata/opf2.py b/src/calibre/ebooks/metadata/opf2.py
index 8a4ff6a5bd..5c2477c3dc 100644
--- a/src/calibre/ebooks/metadata/opf2.py
+++ b/src/calibre/ebooks/metadata/opf2.py
@@ -1230,7 +1230,7 @@ def metadata_to_opf(mi, as_string=True):
%(id)s%(uuid)s
-
+
'''%dict(a=__appname__, id=mi.application_id, uuid=mi.uuid)))
From 87d70304bd7c96b3f65c565b1d8f8177f017f7a0 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Fri, 24 Sep 2010 00:05:11 -0600
Subject: [PATCH 102/207] Make metadata backup a little more robust
---
src/calibre/library/caches.py | 14 ++++++++++----
src/calibre/library/database2.py | 1 +
2 files changed, 11 insertions(+), 4 deletions(-)
diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index 2d37314896..0b5a922209 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -19,7 +19,7 @@ from calibre.utils.date import parse_date, now, UNDEFINED_DATE
from calibre.utils.search_query_parser import SearchQueryParser
from calibre.utils.pyparsing import ParseException
from calibre.ebooks.metadata import title_sort
-from calibre import fit_image
+from calibre import fit_image, prints
class MetadataBackup(Thread): # {{{
@@ -39,9 +39,15 @@ class MetadataBackup(Thread): # {{{
id_ = self.db.dirtied_queue.get(True, 5)
except Empty:
continue
- # If there is an exception is dump_func, we
- # have no way of knowing
- self.dump_func([id_])
+ except:
+ # Happens during interpreter shutdown
+ break
+ if self.dump_func([id_]) is None:
+ # An exception occured in dump_func, retry once
+ prints('Failed to backup metadata for id:', id_, 'once')
+ time.sleep(2)
+ if not self.dump_func([id_]):
+ prints('Failed to backup metadata for id:', id_, 'again, giving up')
# }}}
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index 92f8cca0db..6a0d442927 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -575,6 +575,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
(book_id,))
if commit:
self.conn.commit()
+ return True
def dirtied(self, book_ids, commit=True):
self.conn.executemany(
From 992e5c3c087c1e28cb1e5f1aa61ead0e4556de18 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Fri, 24 Sep 2010 08:21:10 +0100
Subject: [PATCH 103/207] Repair damage during conflict resolution
---
src/calibre/utils/formatter.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/calibre/utils/formatter.py b/src/calibre/utils/formatter.py
index f1c2a2cb4d..a98f0e7f45 100644
--- a/src/calibre/utils/formatter.py
+++ b/src/calibre/utils/formatter.py
@@ -50,7 +50,7 @@ class TemplateFormatter(string.Formatter):
format_string_re = re.compile(r'^(.*)\|(.*)\|(.*)$')
compress_spaces = re.compile(r'\s+')
- def get_value(self, key, args):
+ def get_value(self, key, args, kwargs):
raise Exception('get_value must be implemented in the subclass')
From 12768864a59be5ddf477c490072fd682b1f5942a Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Fri, 24 Sep 2010 08:54:06 +0100
Subject: [PATCH 104/207] 1) fix exception in set_metadata related to composite
custom columns 2) make ondevice work with add_books_from_device
---
src/calibre/gui2/actions/add.py | 2 +-
src/calibre/gui2/device.py | 16 +++++++++-------
src/calibre/gui2/library/models.py | 3 +++
src/calibre/library/custom_columns.py | 2 ++
4 files changed, 15 insertions(+), 8 deletions(-)
diff --git a/src/calibre/gui2/actions/add.py b/src/calibre/gui2/actions/add.py
index aa20b8bc16..e0a7b5647e 100644
--- a/src/calibre/gui2/actions/add.py
+++ b/src/calibre/gui2/actions/add.py
@@ -232,7 +232,7 @@ class AddAction(InterfaceAction):
# metadata for this book to the device. This sets the uuid to the
# correct value. Note that set_books_in_library might sync_booklists
self.gui.set_books_in_library(booklists=[model.db], reset=True)
- model.reset()
+ self.gui.refresh_ondevice()
def add_books_from_device(self, view):
rows = view.selectionModel().selectedRows()
diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py
index a7e55c4619..58c5e5d9ad 100644
--- a/src/calibre/gui2/device.py
+++ b/src/calibre/gui2/device.py
@@ -721,14 +721,16 @@ class DeviceMixin(object): # {{{
self.device_manager.device.__class__.get_gui_name()+\
_(' detected.'), 3000)
self.device_connected = device_kind
- self.refresh_ondevice_info (device_connected = True, reset_only = True)
+ self.library_view.set_device_connected(self.device_connected)
+ self.refresh_ondevice (reset_only = True)
else:
self.device_connected = None
self.status_bar.device_disconnected()
if self.current_view() != self.library_view:
self.book_details.reset_info()
self.location_manager.update_devices()
- self.refresh_ondevice_info(device_connected=False)
+ self.library_view.set_device_connected(self.device_connected)
+ self.refresh_ondevice()
def info_read(self, job):
'''
@@ -760,9 +762,9 @@ class DeviceMixin(object): # {{{
self.card_b_view.set_editable(self.device_manager.device.CAN_SET_METADATA)
self.sync_news()
self.sync_catalogs()
- self.refresh_ondevice_info(device_connected = True)
+ self.refresh_ondevice()
- def refresh_ondevice_info(self, device_connected, reset_only = False):
+ def refresh_ondevice(self, reset_only = False):
'''
Force the library view to refresh, taking into consideration new
device books information
@@ -770,7 +772,7 @@ class DeviceMixin(object): # {{{
self.book_on_device(None, reset=True)
if reset_only:
return
- self.library_view.set_device_connected(device_connected)
+ self.library_view.model().refresh_ondevice()
# }}}
@@ -803,7 +805,7 @@ class DeviceMixin(object): # {{{
self.book_on_device(None, reset=True)
# We need to reset the ondevice flags in the library. Use a big hammer,
# so we don't need to worry about whether some succeeded or not.
- self.refresh_ondevice_info(device_connected=True, reset_only=False)
+ self.refresh_ondevice(reset_only=False)
def dispatch_sync_event(self, dest, delete, specific):
rows = self.library_view.selectionModel().selectedRows()
@@ -1300,7 +1302,7 @@ class DeviceMixin(object): # {{{
if not self.set_books_in_library(self.booklists(), reset=True):
self.upload_booklists()
self.book_on_device(None, reset=True)
- self.refresh_ondevice_info(device_connected = True)
+ self.refresh_ondevice()
view = self.card_a_view if on_card == 'carda' else \
self.card_b_view if on_card == 'cardb' else self.memory_view
diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index 9d9de358c8..640a588d29 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -120,6 +120,9 @@ class BooksModel(QAbstractTableModel): # {{{
def set_device_connected(self, is_connected):
self.device_connected = is_connected
+ self.refresh_ondevice()
+
+ def refresh_ondevice(self):
self.db.refresh_ondevice()
self.refresh() # does a resort()
self.research()
diff --git a/src/calibre/library/custom_columns.py b/src/calibre/library/custom_columns.py
index 2d8634659b..97c8565177 100644
--- a/src/calibre/library/custom_columns.py
+++ b/src/calibre/library/custom_columns.py
@@ -427,6 +427,8 @@ class CustomColumns(object):
data = self.custom_column_label_map[label]
if num is not None:
data = self.custom_column_num_map[num]
+ if data['datatype'] == 'composite':
+ return None
if not data['editable']:
raise ValueError('Column %r is not editable'%data['label'])
table, lt = self.custom_table_names(data['num'])
From 97e2c838d0e6e4920ce4e1f4d71688979f57b68d Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Fri, 24 Sep 2010 10:50:50 +0100
Subject: [PATCH 105/207] 1) Fix of json codec. 2) make dump_metadata set
get_cover=False
---
src/calibre/ebooks/metadata/book/json_codec.py | 3 ++-
src/calibre/library/database2.py | 2 +-
2 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/src/calibre/ebooks/metadata/book/json_codec.py b/src/calibre/ebooks/metadata/book/json_codec.py
index 2550089473..c02d4e953d 100644
--- a/src/calibre/ebooks/metadata/book/json_codec.py
+++ b/src/calibre/ebooks/metadata/book/json_codec.py
@@ -75,7 +75,8 @@ class JsonCodec(object):
self.field_metadata = FieldMetadata()
def encode_to_file(self, file, booklist):
- json.dump(self.encode_booklist_metadata(booklist), file, indent=2, encoding='utf-8')
+ file.write(json.dumps(self.encode_booklist_metadata(booklist),
+ indent=2, encoding='utf-8'))
def encode_booklist_metadata(self, booklist):
result = []
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index 6a0d442927..773a4bdc9f 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -561,7 +561,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
for book_id in book_ids:
if not self.data.has_id(book_id):
continue
- mi = self.get_metadata(book_id, index_is_id=True, get_cover=True)
+ mi = self.get_metadata(book_id, index_is_id=True, get_cover=False)
# Always set cover to cover.jpg. Even if cover doesn't exist,
# no harm done. This way no need to call dirtied when
# cover is set/removed
From 8b9b64a8e6bdab03f62c98a4f2c35ec73957cca7 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Fri, 24 Sep 2010 11:34:52 +0100
Subject: [PATCH 106/207] 1) add two tweaks controlling what custom fields the
content server displays 2) add & cleanup some field_metadata methods
---
resources/default_tweaks.py | 18 ++++++++++++++++++
src/calibre/gui2/library/models.py | 2 +-
src/calibre/library/database2.py | 6 ++++++
src/calibre/library/field_metadata.py | 2 +-
src/calibre/library/server/__init__.py | 12 +++++++++++-
src/calibre/library/server/mobile.py | 3 ++-
src/calibre/library/server/opds.py | 3 ++-
src/calibre/library/server/xml.py | 3 ++-
8 files changed, 43 insertions(+), 6 deletions(-)
diff --git a/resources/default_tweaks.py b/resources/default_tweaks.py
index 04b861605e..095eba0c3d 100644
--- a/resources/default_tweaks.py
+++ b/resources/default_tweaks.py
@@ -145,6 +145,24 @@ add_new_book_tags_when_importing_books = False
# Set the maximum number of tags to show per book in the content server
max_content_server_tags_shown=5
+# Set custom metadata fields that the content server will or will not display.
+# content_server_will_display is a list of custom fields to be displayed.
+# content_server_wont_display is a list of custom fields not to be displayed.
+# wont_display has priority over will_display.
+# The special value '*' means all custom fields.
+# Defaults:
+# content_server_will_display = ['*']
+# content_server_wont_display = ['']
+# Examples:
+# To display only the custom fields #mytags and #genre:
+# content_server_will_display = ['#mytags', '#genre']
+# content_server_wont_display = ['']
+# To display all fields except #mycomments:
+# content_server_will_display = ['*']
+# content_server_wont_display['#mycomments']
+content_server_will_display = ['*']
+content_server_wont_display = ['']
+
# Set the maximum number of sort 'levels' that calibre will use to resort the
# library after certain operations such as searches or device insertion. Each
diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index 640a588d29..af1b42bf33 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -132,7 +132,7 @@ class BooksModel(QAbstractTableModel): # {{{
def set_database(self, db):
self.db = db
- self.custom_columns = self.db.field_metadata.get_custom_field_metadata()
+ self.custom_columns = self.db.field_metadata.custom_field_metadata()
self.column_map = list(self.orig_headers.keys()) + \
list(self.custom_columns)
def col_idx(name):
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index 773a4bdc9f..c7c4926b14 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -554,6 +554,12 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
def search_term_to_field_key(self, term):
return self.field_metadata.search_term_to_key(term)
+ def custom_field_metadata(self):
+ return self.field_metadata.custom_field_metadata()
+
+ def all_metadata(self):
+ return self.field_metadata.all_metadata()
+
def metadata_for_field(self, key):
return self.field_metadata[key]
diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py
index bac423f46d..d608dca49d 100644
--- a/src/calibre/library/field_metadata.py
+++ b/src/calibre/library/field_metadata.py
@@ -411,7 +411,7 @@ class FieldMetadata(dict):
l[k] = self._tb_cats[k]
return l
- def get_custom_field_metadata(self):
+ def custom_field_metadata(self):
l = {}
for k in self._tb_cats:
if self._tb_cats[k]['is_custom']:
diff --git a/src/calibre/library/server/__init__.py b/src/calibre/library/server/__init__.py
index 5050dfaa99..7cdea9f602 100644
--- a/src/calibre/library/server/__init__.py
+++ b/src/calibre/library/server/__init__.py
@@ -7,7 +7,7 @@ __docformat__ = 'restructuredtext en'
import os
-from calibre.utils.config import Config, StringConfig, config_dir
+from calibre.utils.config import Config, StringConfig, config_dir, tweaks
listen_on = '0.0.0.0'
@@ -46,6 +46,16 @@ def server_config(defaults=None):
'to disable grouping.'))
return c
+def custom_fields_to_display(db):
+ ckeys = db.custom_field_keys()
+ yes_fields = set(tweaks['content_server_will_display'])
+ no_fields = set(tweaks['content_server_wont_display'])
+ if '*' in yes_fields:
+ yes_fields = set(ckeys)
+ if '*' in no_fields:
+ no_fields = set(ckeys)
+ return frozenset(yes_fields - no_fields)
+
def main():
from calibre.library.server.main import main
return main()
diff --git a/src/calibre/library/server/mobile.py b/src/calibre/library/server/mobile.py
index c0a3c122cd..071c7b1077 100644
--- a/src/calibre/library/server/mobile.py
+++ b/src/calibre/library/server/mobile.py
@@ -13,6 +13,7 @@ from lxml import html
from lxml.html.builder import HTML, HEAD, TITLE, LINK, DIV, IMG, BODY, \
OPTION, SELECT, INPUT, FORM, SPAN, TABLE, TR, TD, A, HR
+from calibre.library.server import custom_fields_to_display
from calibre.library.server.utils import strftime, format_tag_string
from calibre.ebooks.metadata import fmt_sidx
from calibre.constants import __appname__
@@ -197,7 +198,7 @@ class MobileServer(object):
self.sort(items, sort, (order.lower().strip() == 'ascending'))
CFM = self.db.field_metadata
- CKEYS = [key for key in sorted(CFM.get_custom_fields(),
+ CKEYS = [key for key in sorted(custom_fields_to_display(self.db),
cmp=lambda x,y: cmp(CFM[x]['name'].lower(),
CFM[y]['name'].lower()))]
# This method uses its own book dict, not the Metadata dict. The loop
diff --git a/src/calibre/library/server/opds.py b/src/calibre/library/server/opds.py
index d495f58fa1..0e6917c504 100644
--- a/src/calibre/library/server/opds.py
+++ b/src/calibre/library/server/opds.py
@@ -17,6 +17,7 @@ import routes
from calibre.constants import __appname__
from calibre.ebooks.metadata import fmt_sidx
from calibre.library.comments import comments_to_html
+from calibre.library.server import custom_fields_to_display
from calibre.library.server.utils import format_tag_string
from calibre import guess_type
from calibre.utils.ordered_dict import OrderedDict
@@ -277,7 +278,7 @@ class AcquisitionFeed(NavFeed):
db):
NavFeed.__init__(self, id_, updated, version, offsets, page_url, up_url)
CFM = db.field_metadata
- CKEYS = [key for key in sorted(CFM.get_custom_fields(),
+ CKEYS = [key for key in sorted(custom_fields_to_display(db),
cmp=lambda x,y: cmp(CFM[x]['name'].lower(),
CFM[y]['name'].lower()))]
for item in items:
diff --git a/src/calibre/library/server/xml.py b/src/calibre/library/server/xml.py
index 45ffdc2737..12fcc217f0 100644
--- a/src/calibre/library/server/xml.py
+++ b/src/calibre/library/server/xml.py
@@ -11,6 +11,7 @@ import cherrypy
from lxml.builder import ElementMaker
from lxml import etree
+from calibre.library.server import custom_fields_to_display
from calibre.library.server.utils import strftime, format_tag_string
from calibre.ebooks.metadata import fmt_sidx
from calibre.constants import preferred_encoding
@@ -94,7 +95,7 @@ class XMLServer(object):
c = kwargs.pop('comments')
CFM = self.db.field_metadata
- CKEYS = [key for key in sorted(CFM.get_custom_fields(),
+ CKEYS = [key for key in sorted(custom_fields_to_display(self.db),
cmp=lambda x,y: cmp(CFM[x]['name'].lower(),
CFM[y]['name'].lower()))]
custcols = []
From 67c7555fd0eb22802892ec716ec5564f3d423bc4 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Fri, 24 Sep 2010 11:36:26 +0100
Subject: [PATCH 107/207] Fix content server gui.js bug where it put '...' on
the end of a list even if the list was exactly the right size.
---
resources/content_server/gui.js | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/resources/content_server/gui.js b/resources/content_server/gui.js
index bd0743a854..86cd04289b 100644
--- a/resources/content_server/gui.js
+++ b/resources/content_server/gui.js
@@ -63,8 +63,9 @@ function render_book(book) {
if (tags) {
t = tags.split(':&:', 2);
m = parseInt(t[0]);
+ tall = t[1].split(',');
t = t[1].split(',', m);
- if (t.length == m) t[m] = '...'
+ if (tall.length > m) t[m] = '...'
title += 'Tags=[{0}] '.format(t.join(','));
}
custcols = book.attr("custcols").split(',')
From ad69ef985a04b5485550f83661ee0b56723605f1 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Fri, 24 Sep 2010 12:27:39 +0100
Subject: [PATCH 108/207] Add a 'test' function to templates. Analogous to
lookup, but inserts plain text instead of a template.
---
src/calibre/utils/formatter.py | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/src/calibre/utils/formatter.py b/src/calibre/utils/formatter.py
index a98f0e7f45..5c5893576c 100644
--- a/src/calibre/utils/formatter.py
+++ b/src/calibre/utils/formatter.py
@@ -23,6 +23,12 @@ class TemplateFormatter(string.Formatter):
else:
return self.vformat('{'+field_not_set.strip()+'}', [], self.kwargs)
+ def _test(self, val, value_if_set, value_not_set):
+ if val:
+ return value_if_set
+ else:
+ return value_not_set
+
def _ifempty(self, val, value_if_empty):
if val:
return val
@@ -45,6 +51,7 @@ class TemplateFormatter(string.Formatter):
'ifempty' : (1, _ifempty),
'lookup' : (2, _lookup),
'shorten' : (3, _shorten),
+ 'test' : (2, _lookup),
}
format_string_re = re.compile(r'^(.*)\|(.*)\|(.*)$')
From b2a6ed3af48a47c5cc7bb3372a3b084a004b7fcf Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Fri, 24 Sep 2010 12:43:54 +0100
Subject: [PATCH 109/207] 1) fix bulk edit to not display a tab if library has
only composite columns 2) fix a reference to get_custom_field_metadata that I
somehow missed.
---
src/calibre/gui2/dialogs/metadata_bulk.py | 3 ++-
src/calibre/gui2/preferences/columns.py | 2 +-
2 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py
index a9e45087fd..1e3576e333 100644
--- a/src/calibre/gui2/dialogs/metadata_bulk.py
+++ b/src/calibre/gui2/dialogs/metadata_bulk.py
@@ -167,7 +167,8 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
self.tag_editor_button.clicked.connect(self.tag_editor)
self.autonumber_series.stateChanged[int].connect(self.auto_number_changed)
- if len(db.custom_column_label_map) == 0:
+ if len([k for k in db.custom_field_metadata().values()
+ if k['datatype'] != 'composite']) == 0:
self.central_widget.removeTab(1)
else:
self.create_custom_column_editors()
diff --git a/src/calibre/gui2/preferences/columns.py b/src/calibre/gui2/preferences/columns.py
index c1b9230f42..03a50e6f3a 100644
--- a/src/calibre/gui2/preferences/columns.py
+++ b/src/calibre/gui2/preferences/columns.py
@@ -21,7 +21,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
def genesis(self, gui):
self.gui = gui
db = self.gui.library_view.model().db
- self.custcols = copy.deepcopy(db.field_metadata.get_custom_field_metadata())
+ self.custcols = copy.deepcopy(db.field_metadata.custom_field_metadata())
self.column_up.clicked.connect(self.up_column)
self.column_down.clicked.connect(self.down_column)
From 529756238340e0dd66c3be0cfc1f5f7a8a178cfa Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Fri, 24 Sep 2010 12:57:49 +0100
Subject: [PATCH 110/207] Refactor code to clean interfaces and remove overly
complex loop in bulk edit
---
src/calibre/gui2/dialogs/metadata_bulk.py | 3 +--
src/calibre/library/database2.py | 8 ++++----
src/calibre/library/field_metadata.py | 22 +++++++++++-----------
3 files changed, 16 insertions(+), 17 deletions(-)
diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py
index 1e3576e333..b14390e001 100644
--- a/src/calibre/gui2/dialogs/metadata_bulk.py
+++ b/src/calibre/gui2/dialogs/metadata_bulk.py
@@ -167,8 +167,7 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
self.tag_editor_button.clicked.connect(self.tag_editor)
self.autonumber_series.stateChanged[int].connect(self.auto_number_changed)
- if len([k for k in db.custom_field_metadata().values()
- if k['datatype'] != 'composite']) == 0:
+ if len(db.custom_field_keys(include_composites=False)) == 0:
self.central_widget.removeTab(1)
else:
self.create_custom_column_editors()
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index c7c4926b14..22175d3910 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -539,8 +539,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
def standard_field_keys(self):
return self.field_metadata.standard_field_keys()
- def custom_field_keys(self):
- return self.field_metadata.custom_field_keys()
+ def custom_field_keys(self, include_composites=True):
+ return self.field_metadata.custom_field_keys(include_composites)
def all_field_keys(self):
return self.field_metadata.all_field_keys()
@@ -554,8 +554,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
def search_term_to_field_key(self, term):
return self.field_metadata.search_term_to_key(term)
- def custom_field_metadata(self):
- return self.field_metadata.custom_field_metadata()
+ def custom_field_metadata(self, include_composites=True):
+ return self.field_metadata.custom_field_metadata(include_composites)
def all_metadata(self):
return self.field_metadata.all_metadata()
diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py
index d608dca49d..37393d0d2c 100644
--- a/src/calibre/library/field_metadata.py
+++ b/src/calibre/library/field_metadata.py
@@ -358,10 +358,14 @@ class FieldMetadata(dict):
if self._tb_cats[k]['kind']=='field' and
not self._tb_cats[k]['is_custom']]
- def custom_field_keys(self):
- return [k for k in self._tb_cats.keys()
- if self._tb_cats[k]['kind']=='field' and
- self._tb_cats[k]['is_custom']]
+ def custom_field_keys(self, include_composites=True):
+ res = []
+ for k in self._tb_cats.keys():
+ fm = self._tb_cats[k]
+ if fm['kind']=='field' and fm['is_custom'] and \
+ (fm['datatype'] != 'composite' or include_composites):
+ res.append(k)
+ return res
def all_field_keys(self):
return [k for k in self._tb_cats.keys() if self._tb_cats[k]['kind']=='field']
@@ -402,20 +406,16 @@ class FieldMetadata(dict):
return self.custom_label_to_key_map[label]
raise ValueError('Unknown key [%s]'%(label))
- def get_custom_fields(self):
- return [l for l in self._tb_cats if self._tb_cats[l]['is_custom']]
-
def all_metadata(self):
l = {}
for k in self._tb_cats:
l[k] = self._tb_cats[k]
return l
- def custom_field_metadata(self):
+ def custom_field_metadata(self, include_composites=True):
l = {}
- for k in self._tb_cats:
- if self._tb_cats[k]['is_custom']:
- l[k] = self._tb_cats[k]
+ for k in self.custom_field_keys(include_composites):
+ l[k] = self._tb_cats[k]
return l
def add_custom_field(self, label, table, column, datatype, colnum, name,
From 25905a349c745108abd6290d5835b109359a72de Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Fri, 24 Sep 2010 13:20:26 +0100
Subject: [PATCH 111/207] Test the 'test' function. Add 're' function and test
it.
---
src/calibre/utils/formatter.py | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/src/calibre/utils/formatter.py b/src/calibre/utils/formatter.py
index 5c5893576c..c6bcaa1c3e 100644
--- a/src/calibre/utils/formatter.py
+++ b/src/calibre/utils/formatter.py
@@ -43,6 +43,9 @@ class TemplateFormatter(string.Formatter):
else:
return val
+ def _re(self, val, pattern, replacement):
+ return re.sub(pattern, replacement, val)
+
functions = {
'uppercase' : (0, lambda s,x: x.upper()),
'lowercase' : (0, lambda s,x: x.lower()),
@@ -50,8 +53,9 @@ class TemplateFormatter(string.Formatter):
'capitalize' : (0, lambda s,x: x.capitalize()),
'ifempty' : (1, _ifempty),
'lookup' : (2, _lookup),
+ 're' : (2, _re),
'shorten' : (3, _shorten),
- 'test' : (2, _lookup),
+ 'test' : (2, _test),
}
format_string_re = re.compile(r'^(.*)\|(.*)\|(.*)$')
From 211bb81113865b1cb38c2f8697b33694a4bb38fe Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Fri, 24 Sep 2010 14:43:07 +0100
Subject: [PATCH 112/207] Put back the sanitize after split on slashes.
---
src/calibre/library/save_to_disk.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py
index a58686f709..e479d27121 100644
--- a/src/calibre/library/save_to_disk.py
+++ b/src/calibre/library/save_to_disk.py
@@ -166,6 +166,7 @@ def get_components(template, mi, id, timefmt='%b %Y', length=250,
components = safe_formatter.safe_format(template, format_args, '', mi,
sanitize=sanitize_func)
components = [x.strip() for x in components.split('/') if x.strip()]
+ components = [sanitize_func(x) for x in components if x]
if not components:
components = [str(id)]
components = [x.encode(filesystem_encoding, 'replace') if isinstance(x,
From 93c8836cb6622579323d95f86dfb6fa03dda78cb Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Fri, 24 Sep 2010 15:18:03 +0100
Subject: [PATCH 113/207] Changes to template faq
---
src/calibre/manual/template_lang.rst | 85 +++++++++++++++++++++++-----
1 file changed, 71 insertions(+), 14 deletions(-)
diff --git a/src/calibre/manual/template_lang.rst b/src/calibre/manual/template_lang.rst
index 59e5c1da4c..6d87a90c93 100644
--- a/src/calibre/manual/template_lang.rst
+++ b/src/calibre/manual/template_lang.rst
@@ -7,9 +7,9 @@ The |app| template language
=======================================================
The |app| template language is used in various places. It is used to control the folder structure and file name when saving files from the |app| library to the disk or eBook reader.
-It is used to define "virtual" columns that contain data from other columns and so on.
+It is also used to define "virtual" columns that contain data from other columns and so on.
-In essence, the template language is very simple. The basic idea is that a template consists of names in curly brackets that are then replaced by the corresponding metadata from the book being processed. So, for example, the default template used for saving books to device in |app| is::
+The basi template language is very simple, but has very powerful advanced features. The basic idea is that a template consists of names in curly brackets that are then replaced by the corresponding metadata from the book being processed. So, for example, the default template used for saving books to device in |app| is::
{author_sort}/{title}/{title} - {authors}
@@ -17,7 +17,9 @@ For the book "The Foundation" by "Isaac Asimov" it will become::
Asimov, Isaac/The Foundation/The Foundation - Isaac Asimov
-You can use all the various metadata fields available in calibre in a template, including the custom columns you have created yourself. To find out the template name for a column sinply hover your mouse over the column header. Names for custom fields (columns you have created yourself) are always prefixed by an #. For series type fields, there is always an additional field named ``series_index`` that becomes the series index for that series. So if you have a custom series field named #myseries, there will also be a field named #myseries_index. In addition to the column based fields, you also can use::
+You can use all the various metadata fields available in calibre in a template, including any custom columns you have created yourself. To find out the template name for a column simply hover your mouse over the column header. Names for custom fields (columns you have created yourself) always have a # as the first character. For series type custom fields, there is always an additional field named ``#seriesname_index`` that becomes the series index for that series. So if you have a custom series field named #myseries, there will also be a field named #myseries_index.
+
+In addition to the column based fields, you also can use::
{formats} - A list of formats available in the calibre library for a book
{isbn} - The ISBN number of the book
@@ -26,7 +28,7 @@ If a particular book does not have a particular piece of metadata, the field in
{author_sort}/{series}/{title} {series_index}
-will become::
+If a book has a series, the template will produce::
{Asimov, Isaac}/Foundation/Second Foundation - 3
@@ -40,35 +42,90 @@ and if a book does not have a series::
Advanced formatting
----------------------
-You can do more than just simple substitution with the templates. You can also conditionally include text and control how the substituted data is formatted.
+You can do more than just simple substitution with the templates. You can also conditionally include text and control how the substituted data is formatted.
+
+First, conditionally including text. There are cases where you might want to have text appear in the output only if a field is not empty. A common case is series and series_index, where you want either nothing or the two values with a hyphen between them. Calibre handles this case using a special field syntax.
-Regarding conditionally including text: there are cases where you might want to have text appear in the output only if a field is not empty. A common case is series and series_index, where you want either nothing or the two values with a hyphen between them. Calibre handles this case using a special field syntax.
For example, assume you want to use the template
{series} - {series_index} - {title}
-Unfortunately, if the book has no series, the answer will be '- - title'. Many people would rather it be simply 'title', without the hyphens. To do this, use the extended syntax {some_text|field|other_text}. When you use this syntax, if field has the value SERIES then the result will be some_textSERIESother_text. If field has no value, then the result will be the empty string (nothing). Using this syntax, we can solve the above series problem with the template::
+If the book has no series, the answer will be '- - title'. Many people would rather the result be simply 'title', without the hyphens. To do this, use the extended syntax `{field:|prefix_text|suffix_text}`. When you use this syntax, if field has the value SERIES then the result will be prefix_textSERIESsuffix_text. If field has no value, then the result will be the empty string (nothing). The prefix and suffix can contain blanks.
- {series}{ - |series_index| - }{title}
+Using this syntax, we can solve the above series problem with the template:
-The hyphens will be included only if the book has a series index. Note: you must either use no | characters or both of them. Using one, such as in {field| - }, is not allowed. It is OK to not provide any text for one side or the other, such as in {\|series\| - }. Using {\|title\|} is the same as using {title}.
+ {series}{series_index:| - | - }{title}
-Now to formatting. Suppose you wanted to ensure that the series_index is always formatted as three digits with leading zeros. This would do the trick::
+The hyphens will be included only if the book has a series index.
+
+Notes: you must include the : character if you want to use a prefix or a suffix. You must either use no | characters or both of them; using one, as in `{field:| - }`, is not allowed. It is OK not to provide any text for one side or the other, such as in `{series:|| - }`. Using `{title:||}` is the same as using `{title}`.
+
+Second: formatting. Suppose you wanted to ensure that the series_index is always formatted as three digits with leading zeros. This would do the trick::
{series_index:0>3s} - Three digits with leading zeros
-If instead of leading zeros you want leading spaces, use::
+If instead of leading zeros you want leading spaces, use:
- {series_index:>3s} - Thre digits with leading spaces
+ {series_index:>3s} - Three digits with leading spaces
-For trailing zeros, use::
+For trailing zeros, use:
{series_index:0<3s} - Three digits with trailing zeros
-If you want only the first two letters of the data to be rendered, use::
+If you want only the first two letters of the data, use::
{author_sort:.2} - Only the first two letter of the author sort name
The |app| template language comes from python and for more details on the syntax of these advanced formatting operations, look at the `Python documentation `_.
+Advanced features
+------------------
+
+Using templates in custom columns
+----------------------------------
+
+There are sometimes cases where you want to display metadata that |app| does not normally display, or to display data in a way different from how |app| normally does. For example, you might want to display the ISBN, a field that |app| does not display. You can use custom columns for this. To do so, you create a column with the type 'column built from other columns' (hereafter called composite columns), enter a template, and |app| will display in the column the result of evaluating that template. To display the isbn, create the column and enter `{isbn}` into the template box. To display a column containing the values of two series custom columns separated by a comma, use `{#series1:||,}{#series2}`.
+
+Composite columns can use any template option, including formatting.
+
+You cannot change the data contained in a composite column. If you edit a composite column by double-clicking on any item, you will open the template for editing, not the underlying data. Editing the template on the GUI is a quick way of testing and changing composite columns.
+
+Using functions in templates
+-----------------------------
+
+Suppose you want to display the value of a field in upper case, when that field is normally in title case. You can do this (and many more things) using the functions available for templates. For example, to display the title in upper case, use `{title:uppercase()}`. To display it in title case, use `{title:titlecase()}`.
+
+Function references replace the formatting specification, going after the : and before the first `|` or the closing `}`. Functions must always end with `()`. Some functions take extra values (arguments), and these go inside the `()`.
+
+The syntax for using functions is `{field:function(arguments)}`, or `{field:function(arguments)|prefix|suffix}`. Argument values cannot contain a comma, because it is used to separate arguments. Functions return the value of the field used in the template, suitably modified.
+
+The functions available are:
+
+* `lowercase()` -- return value of the field in lower case.
+* `uppercase()` -- return the value of the field in upper case.
+* `titlecase()` -- return the value of the field in title case.
+* `capitalize()` -- return the value as capitalized.
+* `ifempty(text)` -- if the field is not empty, return the value of the field. Otherwise return `text`.
+* `test(text if not empty, text if empty)` -- return `text if not empty` if the field is not empty, otherwise return `text if empty`.
+* `shorten(left chars, middle text, right chars)` -- Return a shortened version of the field, consisting of `left chars` characters from the beginning of the field, followed by `middle text`, followed by `right chars` characters from the end of the string. `Left chars` and `right chars` must be integers. For example, assume the title of the book is `Ancient English Laws in the Times of Ivanhoe`, and you want it to fit in a space of at most 15 characters. If you use `{title:shorten(9,-,5)}, the result will be `Ancient E-nhoe`. If the field's length is less than `left chars` + `right chars` + the length of `middle text`, then the field will be used intact. For example, the title `The Dome` would not be changed.
+* `lookup(field if not empty, field if empty)` -- like test, except the arguments are field (metadata) names, not text. The value of the appropriate field will be fetched and used. Note that because composite columns are fields, you can use this function in one composite field to use the value of some other composite field. This is extremely useful when constructing variable save paths (more later).
+* `re(pattern, replacement)` -- return the field after applying the regular expression. All instances of `pattern` are replaced with `replacement`. As in all of |app|, these are python-compatible regular expressions.
+
+Special notes for save/send templates
+-------------------------------------
+
+Special processing is applied when a template is used in a `save to disk` or `send to device` template. The values of the fields are cleaned, replacing characters that are special to file systems with underscores, including slashes. This means that field text cannot be used to create folders. However, slashes are not changed in prefix or suffix strings, so slashes in these strings will cause folders to be created. Because of this, you can create variable-depth folder structure.
+
+For example, assume we want the folder structure `series/series_index - title`, with the caveat that if series does not exist, then the title should be in the top folder. The template to do this is
+
+ {series:||/}{series_index:|| - }{title}
+
+The slash and the hyphen appear only if series is not empty.
+
+The lookup function lets us do even fancier processing. For example, assume we want the following: if a book has a series, then we want the folder structure `series/series index - title.fmt`. If the book does not have a series, then we want the folder structure `genre/author_sort/title.fmt`. If the book has no genre, use 'Unknown'. We want two completely different paths, depending on the value of series.
+
+To accomplish this, we:
+1. Create a composite field (call it AA) containing `{series:||}/{series_index} - {title'}`. If the series is not empty, then this template will produce `series/series_index - title`.
+2. Create a composite field (call it BB) containing `{#genre:ifempty(Unknown)}/{author_sort}/{title}`. This template produces `genre/author_sort/title`, where an empty genre is replaced wuth `Unknown`.
+3. Set the save template to `{series:lookup(AA,BB)}`. This template chooses composite field AA if series is not empty, and composite field BB if series is empty. We therefore have two completely different save paths, depending on whether or not `series` is empty.
From 02e9160f3752c179391bfddbb1b3febb9cb3a517 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Fri, 24 Sep 2010 15:23:35 +0100
Subject: [PATCH 114/207] Fix typo
---
src/calibre/manual/template_lang.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/calibre/manual/template_lang.rst b/src/calibre/manual/template_lang.rst
index 6d87a90c93..2c49cfe308 100644
--- a/src/calibre/manual/template_lang.rst
+++ b/src/calibre/manual/template_lang.rst
@@ -9,7 +9,7 @@ The |app| template language
The |app| template language is used in various places. It is used to control the folder structure and file name when saving files from the |app| library to the disk or eBook reader.
It is also used to define "virtual" columns that contain data from other columns and so on.
-The basi template language is very simple, but has very powerful advanced features. The basic idea is that a template consists of names in curly brackets that are then replaced by the corresponding metadata from the book being processed. So, for example, the default template used for saving books to device in |app| is::
+The basic template language is very simple, but has very powerful advanced features. The basic idea is that a template consists of names in curly brackets that are then replaced by the corresponding metadata from the book being processed. So, for example, the default template used for saving books to device in |app| is::
{author_sort}/{title}/{title} - {authors}
From 60e77299062148aa04e325de03b8089d26781234 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Fri, 24 Sep 2010 08:33:42 -0600
Subject: [PATCH 115/207] Don't put duplicates in dirtied_queue
---
src/calibre/library/database2.py | 14 +++++++++-----
1 file changed, 9 insertions(+), 5 deletions(-)
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index 6a0d442927..dc320eb011 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -578,13 +578,17 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
return True
def dirtied(self, book_ids, commit=True):
- self.conn.executemany(
- 'INSERT OR REPLACE INTO metadata_dirtied (book) VALUES (?)',
- [(x,) for x in book_ids])
+ for book in book_ids:
+ try:
+ self.conn.execute(
+ 'INSERT INTO metadata_dirtied (book) VALUES (?)',
+ (book,))
+ self.dirtied_queue.put(book)
+ except IntegrityError:
+ # Already in table
+ continue
if commit:
self.conn.commit()
- for x in book_ids:
- self.dirtied_queue.put(x)
def get_metadata(self, idx, index_is_id=False, get_cover=False):
'''
From 41ebe2bd1443bbf2a88731c6d1919a8207f08275 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Fri, 24 Sep 2010 09:00:46 -0600
Subject: [PATCH 116/207] calibredb now does a backup of changed metadata
---
src/calibre/library/caches.py | 9 ++++++---
src/calibre/library/cli.py | 8 ++++++--
src/calibre/library/database2.py | 6 +++++-
3 files changed, 17 insertions(+), 6 deletions(-)
diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index 0b5a922209..714579ec77 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -36,14 +36,14 @@ class MetadataBackup(Thread): # {{{
def run(self):
while self.keep_running:
try:
- id_ = self.db.dirtied_queue.get(True, 5)
+ id_ = self.db.dirtied_queue.get()
except Empty:
continue
except:
# Happens during interpreter shutdown
break
if self.dump_func([id_]) is None:
- # An exception occured in dump_func, retry once
+ # An exception occurred in dump_func, retry once
prints('Failed to backup metadata for id:', id_, 'once')
time.sleep(2)
if not self.dump_func([id_]):
@@ -84,9 +84,12 @@ class CoverCache(Thread): # {{{
def run(self):
while self.keep_running:
try:
- id_ = self.load_queue.get(True, 1)
+ id_ = self.load_queue.get()
except Empty:
continue
+ except:
+ #Happens during interpreter shutdown
+ break
try:
img = self._image_for_id(id_)
except:
diff --git a/src/calibre/library/cli.py b/src/calibre/library/cli.py
index cd4e472807..6ff17b0781 100644
--- a/src/calibre/library/cli.py
+++ b/src/calibre/library/cli.py
@@ -32,8 +32,9 @@ def send_message(msg=''):
t.conn.send('refreshdb:'+msg)
t.conn.close()
-
-
+def write_dirtied(db):
+ prints('Backing up metadata')
+ db.dump_metadata()
def get_parser(usage):
parser = OptionParser(usage)
@@ -259,6 +260,7 @@ def do_add(db, paths, one_book_per_directory, recurse, add_duplicates):
print >>sys.stderr, '\t', title+':'
print >>sys.stderr, '\t\t ', path
+ write_dirtied(db)
send_message()
finally:
sys.stdout = orig
@@ -299,6 +301,7 @@ def do_add_empty(db, title, authors, isbn):
if isbn:
mi.isbn = isbn
db.import_book(mi, [])
+ write_dirtied()
send_message()
def command_add(args, dbpath):
@@ -452,6 +455,7 @@ def do_set_metadata(db, id, stream):
db.set_metadata(id, mi)
db.clean()
do_show_metadata(db, id, False)
+ write_dirtied()
send_message()
def set_metadata_option_parser():
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index dc320eb011..bdaa643d83 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -557,7 +557,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
def metadata_for_field(self, key):
return self.field_metadata[key]
- def dump_metadata(self, book_ids, remove_from_dirtied=True, commit=True):
+ def dump_metadata(self, book_ids=None, remove_from_dirtied=True, commit=True):
+ 'Write metadata for each record to an individual OPF file'
+ if book_ids is None:
+ book_ids = [x[0] for x in self.conn.get(
+ 'SELECT book FROM metadata_dirtied', all=True)]
for book_id in book_ids:
if not self.data.has_id(book_id):
continue
From c67a9d848745c4fd4280b370583160bad288cbbd Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Fri, 24 Sep 2010 16:38:11 +0100
Subject: [PATCH 117/207] Changes to device editable columns to give fine-grain
control over what columns can be edited.
---
src/calibre/devices/folder_device/driver.py | 2 +-
src/calibre/devices/interface.py | 2 +-
src/calibre/devices/kobo/driver.py | 16 +++++++-------
src/calibre/devices/prs505/driver.py | 2 +-
src/calibre/devices/usbms/driver.py | 2 +-
src/calibre/gui2/library/models.py | 23 ++++++++++++++-------
6 files changed, 27 insertions(+), 20 deletions(-)
diff --git a/src/calibre/devices/folder_device/driver.py b/src/calibre/devices/folder_device/driver.py
index 9cd1280cc9..5919d6d2fb 100644
--- a/src/calibre/devices/folder_device/driver.py
+++ b/src/calibre/devices/folder_device/driver.py
@@ -38,7 +38,7 @@ class FOLDER_DEVICE(USBMS):
THUMBNAIL_HEIGHT = 68 # Height for thumbnails on device
- CAN_SET_METADATA = True
+ CAN_SET_METADATA = ['title', 'authors']
SUPPORTS_SUB_DIRS = True
#: Icon for this device
diff --git a/src/calibre/devices/interface.py b/src/calibre/devices/interface.py
index fc3332a337..2307bf94d6 100644
--- a/src/calibre/devices/interface.py
+++ b/src/calibre/devices/interface.py
@@ -37,7 +37,7 @@ class DevicePlugin(Plugin):
THUMBNAIL_HEIGHT = 68
#: Whether the metadata on books can be set via the GUI.
- CAN_SET_METADATA = True
+ CAN_SET_METADATA = ['title', 'authors', 'collections']
#: Path separator for paths to books on device
path_sep = os.sep
diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py
index f06a804b93..b8516aab4f 100644
--- a/src/calibre/devices/kobo/driver.py
+++ b/src/calibre/devices/kobo/driver.py
@@ -30,7 +30,7 @@ class KOBO(USBMS):
# Ordered list of supported formats
FORMATS = ['epub', 'pdf']
- CAN_SET_METADATA = True
+ CAN_SET_METADATA = ['collections']
VENDOR_ID = [0x2237]
PRODUCT_ID = [0x4161]
@@ -126,7 +126,7 @@ class KOBO(USBMS):
book = self.book_from_path(prefix, lpath, title, authors, mime, date, ContentType, ImageID)
# print 'Update booklist'
book.device_collections = [playlist_map[lpath]] if lpath in playlist_map else []
-
+
if bl.add_book(book, replace_metadata=False):
changed = True
except: # Probably a path encoding error
@@ -250,7 +250,7 @@ class KOBO(USBMS):
# print "Delete file normalized path: " + path
extension = os.path.splitext(path)[1]
ContentType = self.get_content_type_from_extension(extension)
-
+
ContentID = self.contentid_from_path(path, ContentType)
ImageID = self.delete_via_sql(ContentID, ContentType)
@@ -453,7 +453,7 @@ class KOBO(USBMS):
query= 'update content set ReadStatus=0, FirstTimeReading = \'true\' where BookID is Null and ReadStatus = 1 and ContentID like \'file:///mnt/sd/%\''
elif oncard != 'carda' and oncard != 'cardb':
query= 'update content set ReadStatus=0, FirstTimeReading = \'true\' where BookID is Null and ReadStatus = 1 and ContentID not like \'file:///mnt/sd/%\''
-
+
try:
cursor.execute (query)
except:
@@ -489,7 +489,7 @@ class KOBO(USBMS):
query= 'update content set ReadStatus=0, FirstTimeReading = \'true\' where BookID is Null and ReadStatus = 2 and ContentID like \'file:///mnt/sd/%\''
elif oncard != 'carda' and oncard != 'cardb':
query= 'update content set ReadStatus=0, FirstTimeReading = \'true\' where BookID is Null and ReadStatus = 2 and ContentID not like \'file:///mnt/sd/%\''
-
+
try:
cursor.execute (query)
except:
@@ -519,7 +519,7 @@ class KOBO(USBMS):
else:
connection.commit()
# debug_print('Database: Commit set ReadStatus as Finished')
- else: # No collections
+ else: # No collections
# Since no collections exist the ReadStatus needs to be reset to 0 (Unread)
print "Reseting ReadStatus to 0"
# Reset Im_Reading list in the database
@@ -527,7 +527,7 @@ class KOBO(USBMS):
query= 'update content set ReadStatus=0, FirstTimeReading = \'true\' where BookID is Null and ContentID like \'file:///mnt/sd/%\''
elif oncard != 'carda' and oncard != 'cardb':
query= 'update content set ReadStatus=0, FirstTimeReading = \'true\' where BookID is Null and ContentID not like \'file:///mnt/sd/%\''
-
+
try:
cursor.execute (query)
except:
@@ -541,7 +541,7 @@ class KOBO(USBMS):
connection.close()
# debug_print('Finished update_device_database_collections', collections_attributes)
-
+
def sync_booklists(self, booklists, end_session=True):
# debug_print('KOBO: started sync_booklists')
paths = self.get_device_paths()
diff --git a/src/calibre/devices/prs505/driver.py b/src/calibre/devices/prs505/driver.py
index f90a8ab263..7952660c21 100644
--- a/src/calibre/devices/prs505/driver.py
+++ b/src/calibre/devices/prs505/driver.py
@@ -27,7 +27,7 @@ class PRS505(USBMS):
FORMATS = ['epub', 'lrf', 'lrx', 'rtf', 'pdf', 'txt']
- CAN_SET_METADATA = True
+ CAN_SET_METADATA = ['title', 'authors', 'collections']
VENDOR_ID = [0x054c] #: SONY Vendor Id
PRODUCT_ID = [0x031e]
diff --git a/src/calibre/devices/usbms/driver.py b/src/calibre/devices/usbms/driver.py
index a0d1d9dbf8..b4fe5d25fc 100644
--- a/src/calibre/devices/usbms/driver.py
+++ b/src/calibre/devices/usbms/driver.py
@@ -50,7 +50,7 @@ class USBMS(CLI, Device):
book_class = Book
FORMATS = []
- CAN_SET_METADATA = False
+ CAN_SET_METADATA = []
METADATA_CACHE = 'metadata.calibre'
def get_device_information(self, end_session=True):
diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index af1b42bf33..8efd038db8 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -907,7 +907,7 @@ class DeviceBooksModel(BooksModel): # {{{
}
self.marked_for_deletion = {}
self.search_engine = OnDeviceSearch(self)
- self.editable = True
+ self.editable = ['title', 'authors', 'collections']
self.book_in_library = None
def mark_for_deletion(self, job, rows, rows_are_ids=False):
@@ -953,13 +953,13 @@ class DeviceBooksModel(BooksModel): # {{{
if self.map[index.row()] in self.indices_to_be_deleted():
return Qt.ItemIsUserCheckable # Can't figure out how to get the disabled flag in python
flags = QAbstractTableModel.flags(self, index)
- if index.isValid() and self.editable:
+ if index.isValid():
cname = self.column_map[index.column()]
- if cname in ('title', 'authors') or \
- (cname == 'collections' and \
- callable(getattr(self.db, 'supports_collections', None)) and \
- self.db.supports_collections() and \
- prefs['manage_device_metadata']=='manual'):
+ if cname in self.editable and \
+ cname != 'collections' or \
+ (callable(getattr(self.db, 'supports_collections', None)) and \
+ self.db.supports_collections() and \
+ prefs['manage_device_metadata']=='manual'):
flags |= Qt.ItemIsEditable
return flags
@@ -1243,7 +1243,14 @@ class DeviceBooksModel(BooksModel): # {{{
def set_editable(self, editable):
# Cannot edit if metadata is sent on connect. Reason: changes will
# revert to what is in the library on next connect.
- self.editable = editable and prefs['manage_device_metadata']!='on_connect'
+ if isinstance(editable, list):
+ self.editable = editable
+ elif editable:
+ self.editable = ['title', 'authors', 'collections']
+ else:
+ self.editable = []
+ if prefs['manage_device_metadata']=='on_connect':
+ self.editable = []
def set_search_restriction(self, s):
pass
From 993983a70767b56d2ecde432fcea828068d8e7f9 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Fri, 24 Sep 2010 10:05:54 -0600
Subject: [PATCH 118/207] Oops. Restore removed call to commit in set_path and
have set_path call dirtied. Also limit the rate of metadata backups
---
src/calibre/library/caches.py | 1 +
src/calibre/library/database2.py | 2 ++
2 files changed, 3 insertions(+)
diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index 714579ec77..339f1393f5 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -48,6 +48,7 @@ class MetadataBackup(Thread): # {{{
time.sleep(2)
if not self.dump_func([id_]):
prints('Failed to backup metadata for id:', id_, 'again, giving up')
+ time.sleep(0.2) # Limit to five per second
# }}}
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index a34ef9cf89..f62c4ce074 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -455,6 +455,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
self.add_format(id, format, stream, index_is_id=True,
path=tpath, notify=False)
self.conn.execute('UPDATE books SET path=? WHERE id=?', (path, id))
+ self.dirtied([id], commit=False)
+ self.commit()
self.data.set(id, self.FIELD_MAP['path'], path, row_is_id=True)
# Delete not needed directories
if current_path and os.path.exists(spath):
From a11ccd8598d3ba3cf34599647d58666b49038302 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Fri, 24 Sep 2010 17:20:26 +0100
Subject: [PATCH 119/207] Added 'contains' function to templates
---
src/calibre/manual/template_lang.rst | 1 +
src/calibre/utils/formatter.py | 13 ++++++++++---
2 files changed, 11 insertions(+), 3 deletions(-)
diff --git a/src/calibre/manual/template_lang.rst b/src/calibre/manual/template_lang.rst
index 5f672c4989..0c3a87a157 100644
--- a/src/calibre/manual/template_lang.rst
+++ b/src/calibre/manual/template_lang.rst
@@ -108,6 +108,7 @@ The functions available are:
* ``capitalize()`` -- return the value as capitalized.
* ``ifempty(text)`` -- if the field is not empty, return the value of the field. Otherwise return `text`.
* ``test(text if not empty, text if empty)`` -- return `text if not empty` if the field is not empty, otherwise return `text if empty`.
+ * ``contains(pattern, text if match, text if not match`` -- checks if field contains matches for the regular expression `pattern`. Returns `text if match` if matches are found, otherwise it returns `text if no match`.
* ``shorten(left chars, middle text, right chars)`` -- Return a shortened version of the field, consisting of `left chars` characters from the beginning of the field, followed by `middle text`, followed by `right chars` characters from the end of the string. `Left chars` and `right chars` must be integers. For example, assume the title of the book is `Ancient English Laws in the Times of Ivanhoe`, and you want it to fit in a space of at most 15 characters. If you use ``{title:shorten(9,-,5)}``, the result will be `Ancient E-nhoe`. If the field's length is less than ``left chars`` + ``right chars`` + the length of ``middle text``, then the field will be used intact. For example, the title `The Dome` would not be changed.
* ``lookup(field if not empty, field if empty)`` -- like test, except the arguments are field (metadata) names, not text. The value of the appropriate field will be fetched and used. Note that because composite columns are fields, you can use this function in one composite field to use the value of some other composite field. This is extremely useful when constructing variable save paths (more later).
* ``re(pattern, replacement)`` -- return the field after applying the regular expression. All instances of `pattern` are replaced with `replacement`. As in all of |app|, these are python-compatible regular expressions.
diff --git a/src/calibre/utils/formatter.py b/src/calibre/utils/formatter.py
index c6bcaa1c3e..6fed4e157a 100644
--- a/src/calibre/utils/formatter.py
+++ b/src/calibre/utils/formatter.py
@@ -29,6 +29,15 @@ class TemplateFormatter(string.Formatter):
else:
return value_not_set
+ def _contains(self, val, test, value_if_present, value_if_not):
+ if re.search(test, val):
+ return value_if_present
+ else:
+ return value_if_not
+
+ def _re(self, val, pattern, replacement):
+ return re.sub(pattern, replacement, val)
+
def _ifempty(self, val, value_if_empty):
if val:
return val
@@ -43,14 +52,12 @@ class TemplateFormatter(string.Formatter):
else:
return val
- def _re(self, val, pattern, replacement):
- return re.sub(pattern, replacement, val)
-
functions = {
'uppercase' : (0, lambda s,x: x.upper()),
'lowercase' : (0, lambda s,x: x.lower()),
'titlecase' : (0, lambda s,x: x.title()),
'capitalize' : (0, lambda s,x: x.capitalize()),
+ 'contains' : (3, _contains),
'ifempty' : (1, _ifempty),
'lookup' : (2, _lookup),
're' : (2, _re),
From 7d9ca9dda75a03ae6d8b97660e05017193cc8ba3 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Fri, 24 Sep 2010 10:40:53 -0600
Subject: [PATCH 120/207] ...
---
src/calibre/library/caches.py | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index 339f1393f5..1e52350e46 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -97,8 +97,12 @@ class CoverCache(Thread): # {{{
import traceback
traceback.print_exc()
continue
- with self.lock:
- self.cache[id_] = img
+ try:
+ with self.lock:
+ self.cache[id_] = img
+ except:
+ # Happens during interpreter shutdown
+ break
def set_cache(self, ids):
with self.lock:
From f782ef0cb6348cc3deea6fe515ace73b5dd18b92 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Fri, 24 Sep 2010 17:56:27 +0100
Subject: [PATCH 121/207] Make format_field return '' instead of None when the
value really is ''
---
src/calibre/ebooks/metadata/book/base.py | 18 +++++++++---------
src/calibre/gui2/library/models.py | 2 +-
src/calibre/library/server/mobile.py | 3 +--
src/calibre/library/server/opds.py | 2 +-
4 files changed, 12 insertions(+), 13 deletions(-)
diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index 8791d59242..87d034aba8 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -432,14 +432,14 @@ class Metadata(object):
if key in self.user_metadata_keys():
res = self.get(key, None)
cmeta = self.get_user_metadata(key, make_copy=False)
+ name = unicode(cmeta['name'])
if cmeta['datatype'] != 'composite' and (res is None or res == ''):
- return (None, None, None, None)
+ return (name, res, None, None)
orig_res = res
cmeta = self.get_user_metadata(key, make_copy=False)
if res is None or res == '':
- return (None, None, None, None)
+ return (name, res, None, None)
orig_res = res
- name = unicode(cmeta['name'])
datatype = cmeta['datatype']
if datatype == 'text' and cmeta['is_multiple']:
res = u', '.join(res)
@@ -454,11 +454,12 @@ class Metadata(object):
if key in field_metadata and field_metadata[key]['kind'] == 'field':
res = self.get(key, None)
- if res is None or res == '':
- return (None, None, None, None)
- orig_res = res
fmeta = field_metadata[key]
name = unicode(fmeta['name'])
+ if res is None or res == '':
+ return (name, res, None, None)
+ orig_res = res
+ name = unicode(fmeta['name'])
datatype = fmeta['datatype']
if key == 'authors':
res = authors_to_string(res)
@@ -508,9 +509,8 @@ class Metadata(object):
fmt('Rights', unicode(self.rights))
for key in self.user_metadata_keys():
val = self.get(key, None)
- if val is not None:
- (name, val) = self.format_field(key)
- fmt(name, unicode(val))
+ (name, val) = self.format_field(key)
+ fmt(name, unicode(val))
return u'\n'.join(ans)
def to_html(self):
diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index d19bed49fe..fe1701a918 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -327,7 +327,7 @@ class BooksModel(QAbstractTableModel): # {{{
mi = self.db.get_metadata(idx)
for key in mi.user_metadata_keys():
name, val = mi.format_field(key)
- if val is not None:
+ if val:
data[name] = val
return data
diff --git a/src/calibre/library/server/mobile.py b/src/calibre/library/server/mobile.py
index 071c7b1077..c51de90c6d 100644
--- a/src/calibre/library/server/mobile.py
+++ b/src/calibre/library/server/mobile.py
@@ -125,7 +125,6 @@ def build_index(books, num, search, sort, order, start, total, url_base, CKEYS):
series = u'[%s - %s]'%(book['series'], book['series_index']) \
if book['series'] else ''
tags = u'Tags=[%s]'%book['tags'] if book['tags'] else ''
- print tags
ctext = ''
for key in CKEYS:
@@ -231,7 +230,7 @@ class MobileServer(object):
return '%s:#:%s'%(name, unicode(val))
mi = self.db.get_metadata(record[CFM['id']['rec_index']], index_is_id=True)
name, val = mi.format_field(key)
- if val is None:
+ if not val:
continue
datatype = CFM[key]['datatype']
if datatype in ['comments']:
diff --git a/src/calibre/library/server/opds.py b/src/calibre/library/server/opds.py
index 0e6917c504..bd5b2f36b3 100644
--- a/src/calibre/library/server/opds.py
+++ b/src/calibre/library/server/opds.py
@@ -160,7 +160,7 @@ def ACQUISITION_ENTRY(item, version, db, updated, CFM, CKEYS):
for key in CKEYS:
mi = db.get_metadata(item[CFM['id']['rec_index']], index_is_id=True)
name, val = mi.format_field(key)
- if val is not None:
+ if not val:
datatype = CFM[key]['datatype']
if datatype == 'text' and CFM[key]['is_multiple']:
extra.append('%s: %s '%(name, format_tag_string(val, ',',
From fb06e4c72eacb21b6f101d6f7e7d7a1785450a86 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Fri, 24 Sep 2010 11:04:18 -0600
Subject: [PATCH 122/207] ...
---
src/calibre/gui2/add.py | 7 ++-----
src/calibre/library/database2.py | 2 +-
2 files changed, 3 insertions(+), 6 deletions(-)
diff --git a/src/calibre/gui2/add.py b/src/calibre/gui2/add.py
index 9f246aeb93..1d7b5075b4 100644
--- a/src/calibre/gui2/add.py
+++ b/src/calibre/gui2/add.py
@@ -381,11 +381,7 @@ class Adder(QObject): # {{{
# }}}
-###############################################################################
-############################## END ADDER ######################################
-###############################################################################
-
-class Saver(QObject):
+class Saver(QObject): # {{{
def __init__(self, parent, db, callback, rows, path, opts,
spare_server=None):
@@ -446,4 +442,5 @@ class Saver(QObject):
self.pd.set_msg(_('Saved')+' '+title)
if not ok:
self.failures.add((title, tb))
+# }}}
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index a9160f976f..4775e13818 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -1924,7 +1924,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
mi.timestamp = utcnow()
if mi.pubdate is None:
mi.pubdate = utcnow()
- self.set_metadata(id, mi)
+ self.set_metadata(id, mi, ignore_errors=True)
if cover is not None:
try:
self.set_cover(id, cover)
From 4b92c7d68b70c03b3d1b46d17fc643ab7bb00f5d Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Fri, 24 Sep 2010 18:17:15 +0100
Subject: [PATCH 123/207] Don't put '' values into __unicode__ and to_html
---
src/calibre/ebooks/metadata/book/base.py | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index 87d034aba8..df64d16c26 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -509,8 +509,9 @@ class Metadata(object):
fmt('Rights', unicode(self.rights))
for key in self.user_metadata_keys():
val = self.get(key, None)
- (name, val) = self.format_field(key)
- fmt(name, unicode(val))
+ if val:
+ (name, val) = self.format_field(key)
+ fmt(name, unicode(val))
return u'\n'.join(ans)
def to_html(self):
@@ -533,7 +534,7 @@ class Metadata(object):
ans += [(_('Rights'), unicode(self.rights))]
for key in self.user_metadata_keys():
val = self.get(key, None)
- if val is not None:
+ if val:
(name, val) = self.format_field(key)
ans += [(name, val)]
for i, x in enumerate(ans):
From fef5703c1eb74e80c7c5a079f8b5ba3dc473bde8 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Fri, 24 Sep 2010 11:32:21 -0600
Subject: [PATCH 124/207] Conversion pipeline: Fix merging of metadata, broken
by new Metadata class
---
src/calibre/ebooks/metadata/book/base.py | 5 ++++
src/calibre/ebooks/oeb/transforms/metadata.py | 30 +++++++++----------
2 files changed, 20 insertions(+), 15 deletions(-)
diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index 87d034aba8..28a5f21a46 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -76,6 +76,11 @@ class Metadata(object):
self.author = list(authors) if authors else []# Needed for backward compatibility
self.authors = list(authors) if authors else []
+ def is_null(self, field):
+ null_val = NULL_VALUES.get(field, None)
+ val = getattr(self, field, None)
+ return not val or val == null_val
+
def __getattribute__(self, field):
_data = object.__getattribute__(self, '_data')
if field in TOP_LEVEL_CLASSIFIERS:
diff --git a/src/calibre/ebooks/oeb/transforms/metadata.py b/src/calibre/ebooks/oeb/transforms/metadata.py
index 22a89f5a47..4bb25f650e 100644
--- a/src/calibre/ebooks/oeb/transforms/metadata.py
+++ b/src/calibre/ebooks/oeb/transforms/metadata.py
@@ -12,33 +12,33 @@ from calibre import guess_type
def meta_info_to_oeb_metadata(mi, m, log):
from calibre.ebooks.oeb.base import OPF
- if mi.title:
+ if not mi.is_null('title'):
m.clear('title')
m.add('title', mi.title)
if mi.title_sort:
if not m.title:
m.add('title', mi.title_sort)
m.title[0].file_as = mi.title_sort
- if mi.authors:
+ if not mi.is_null('authors'):
m.filter('creator', lambda x : x.role.lower() in ['aut', ''])
for a in mi.authors:
attrib = {'role':'aut'}
if mi.author_sort:
attrib[OPF('file-as')] = mi.author_sort
m.add('creator', a, attrib=attrib)
- if mi.book_producer:
+ if not mi.is_null('book_producer'):
m.filter('contributor', lambda x : x.role.lower() == 'bkp')
m.add('contributor', mi.book_producer, role='bkp')
- if mi.comments:
+ if not mi.is_null('comments'):
m.clear('description')
m.add('description', mi.comments)
- if mi.publisher:
+ if not mi.is_null('publisher'):
m.clear('publisher')
m.add('publisher', mi.publisher)
- if mi.series:
+ if not mi.is_null('series'):
m.clear('series')
m.add('series', mi.series)
- if mi.isbn:
+ if not mi.is_null('isbn'):
has = False
for x in m.identifier:
if x.scheme.lower() == 'isbn':
@@ -46,29 +46,29 @@ def meta_info_to_oeb_metadata(mi, m, log):
has = True
if not has:
m.add('identifier', mi.isbn, scheme='ISBN')
- if mi.language:
+ if not mi.is_null('language'):
m.clear('language')
m.add('language', mi.language)
- if mi.series_index is not None:
+ if not mi.is_null('series_index'):
m.clear('series_index')
m.add('series_index', mi.format_series_index())
- if mi.rating is not None:
+ if not mi.is_null('rating'):
m.clear('rating')
m.add('rating', '%.2f'%mi.rating)
- if mi.tags:
+ if not mi.is_null('tags'):
m.clear('subject')
for t in mi.tags:
m.add('subject', t)
- if mi.pubdate is not None:
+ if not mi.is_null('pubdate'):
m.clear('date')
m.add('date', isoformat(mi.pubdate))
- if mi.timestamp is not None:
+ if not mi.is_null('timestamp'):
m.clear('timestamp')
m.add('timestamp', isoformat(mi.timestamp))
- if mi.rights is not None:
+ if not mi.is_null('rights'):
m.clear('rights')
m.add('rights', mi.rights)
- if mi.publication_type is not None:
+ if not mi.is_null('publication_type'):
m.clear('publication_type')
m.add('publication_type', mi.publication_type)
if not m.timestamp:
From 47ff1ddc42d8d08e145af1ebefaa38b93579b549 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Fri, 24 Sep 2010 18:47:44 +0100
Subject: [PATCH 125/207] Minor updates to the FAQ
---
src/calibre/manual/template_lang.rst | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/src/calibre/manual/template_lang.rst b/src/calibre/manual/template_lang.rst
index 0c3a87a157..1ab004f3f3 100644
--- a/src/calibre/manual/template_lang.rst
+++ b/src/calibre/manual/template_lang.rst
@@ -17,14 +17,14 @@ For the book "The Foundation" by "Isaac Asimov" it will become::
Asimov, Isaac/The Foundation/The Foundation - Isaac Asimov
-You can use all the various metadata fields available in calibre in a template, including any custom columns you have created yourself. To find out the template name for a column simply hover your mouse over the column header. Names for custom fields (columns you have created yourself) always have a # as the first character. For series type custom fields, there is always an additional field named ``#seriesname_index`` that becomes the series index for that series. So if you have a custom series field named #myseries, there will also be a field named #myseries_index.
+You can use all the various metadata fields available in calibre in a template, including any custom columns you have created yourself. To find out the template name for a column simply hover your mouse over the column header. Names for custom fields (columns you have created yourself) always have a # as the first character. For series type custom fields, there is always an additional field named ``#seriesname_index`` that becomes the series index for that series. So if you have a custom series field named ``#myseries``, there will also be a field named ``#myseries_index``.
In addition to the column based fields, you also can use::
{formats} - A list of formats available in the calibre library for a book
{isbn} - The ISBN number of the book
-If a particular book does not have a particular piece of metadata, the field in the template is automatically removed for that book. So for example::
+If a particular book does not have a particular piece of metadata, the field in the template is automatically removed for that book. Consider, for example::
{author_sort}/{series}/{title} {series_index}
@@ -44,19 +44,19 @@ Advanced formatting
You can do more than just simple substitution with the templates. You can also conditionally include text and control how the substituted data is formatted.
-First, conditionally including text. There are cases where you might want to have text appear in the output only if a field is not empty. A common case is series and series_index, where you want either nothing or the two values with a hyphen between them. Calibre handles this case using a special field syntax.
+First, conditionally including text. There are cases where you might want to have text appear in the output only if a field is not empty. A common case is ``series`` and ``series_index``, where you want either nothing or the two values with a hyphen between them. Calibre handles this case using a special field syntax.
For example, assume you want to use the template::
{series} - {series_index} - {title}
-If the book has no series, the answer will be '- - title'. Many people would rather the result be simply 'title', without the hyphens. To do this, use the extended syntax ``{field:|prefix_text|suffix_text}``. When you use this syntax, if field has the value SERIES then the result will be prefix_textSERIESsuffix_text. If field has no value, then the result will be the empty string (nothing). The prefix and suffix can contain blanks.
+If the book has no series, the answer will be ``- - title``. Many people would rather the result be simply ``title``, without the hyphens. To do this, use the extended syntax ``{field:|prefix_text|suffix_text}``. When you use this syntax, if field has the value SERIES then the result will be ``prefix_textSERIESsuffix_text``. If field has no value, then the result will be the empty string (nothing); the prefix and suffix are ignored. The prefix and suffix can contain blanks.
Using this syntax, we can solve the above series problem with the template::
{series}{series_index:| - | - }{title}
-The hyphens will be included only if the book has a series index.
+The hyphens will be included only if the book has a series index, which it will have only if it has a series.
Notes: you must include the : character if you want to use a prefix or a suffix. You must either use no \| characters or both of them; using one, as in ``{field:| - }``, is not allowed. It is OK not to provide any text for one side or the other, such as in ``{series:|| - }``. Using ``{title:||}`` is the same as using ``{title}``.
@@ -85,7 +85,7 @@ Advanced features
Using templates in custom columns
----------------------------------
-There are sometimes cases where you want to display metadata that |app| does not normally display, or to display data in a way different from how |app| normally does. For example, you might want to display the ISBN, a field that |app| does not display. You can use custom columns for this. To do so, you create a column with the type 'column built from other columns' (hereafter called composite columns), enter a template, and |app| will display in the column the result of evaluating that template. To display the isbn, create the column and enter ``{isbn}`` into the template box. To display a column containing the values of two series custom columns separated by a comma, use ``{#series1:||,}{#series2}``.
+There are sometimes cases where you want to display metadata that |app| does not normally display, or to display data in a way different from how |app| normally does. For example, you might want to display the ISBN, a field that |app| does not display. You can use custom columns for this by creating a column with the type 'column built from other columns' (hereafter called composite columns), and entering a template. Result: |app| will display a column showing the result of evaluating that template. To display the ISBN, create the column and enter ``{isbn}`` into the template box. To display a column containing the values of two series custom columns separated by a comma, use ``{#series1:||,}{#series2}``.
Composite columns can use any template option, including formatting.
@@ -98,7 +98,7 @@ Suppose you want to display the value of a field in upper case, when that field
Function references replace the formatting specification, going after the : and before the first ``|`` or the closing ``}``. Functions must always end with ``()``. Some functions take extra values (arguments), and these go inside the ``()``.
-The syntax for using functions is ``{field:function(arguments)}``, or ``{field:function(arguments)|prefix|suffix}``. Argument values cannot contain a comma, because it is used to separate arguments. Functions return the value of the field used in the template, suitably modified.
+The syntax for using functions is ``{field:function(arguments)}``, or ``{field:function(arguments)|prefix|suffix}``. Argument values cannot contain a comma, because it is used to separate arguments. The last (or only) argument cannot contain a closing parenthesis ( ')' ). Functions return the value of the field used in the template, suitably modified.
The functions available are:
From cf6f251b740d8601c80e4e3e4a28ad736ebff7d2 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Fri, 24 Sep 2010 19:41:43 +0100
Subject: [PATCH 126/207] Added dirty bit cache
---
src/calibre/gui2/ui.py | 1 +
src/calibre/library/database2.py | 31 ++++++++++++++++++++++++++++++-
2 files changed, 31 insertions(+), 1 deletion(-)
diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py
index 88a8c68572..6b04f6fa1f 100644
--- a/src/calibre/gui2/ui.py
+++ b/src/calibre/gui2/ui.py
@@ -533,6 +533,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, # {{{
# Save the current field_metadata for applications like calibre2opds
# Goes here, because if cf is valid, db is valid.
db.prefs['field_metadata'] = db.field_metadata.all_metadata()
+ db.commit_dirty_cache()
if DEBUG and db.gm_count > 0:
print 'get_metadata cache: {0:d} calls, {1:4.2f}% misses'.format(
db.gm_count, (db.gm_missed*100.0)/db.gm_count)
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index 4775e13818..c4d2666dd1 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -340,6 +340,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
setattr(self, 'title_sort', functools.partial(self.get_property,
loc=self.FIELD_MAP['sort']))
+ self.dirtied_cache = set()
d = self.conn.get('SELECT book FROM metadata_dirtied', all=True)
for x in d:
self.dirtied_queue.put(x[0])
@@ -585,12 +586,20 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
if remove_from_dirtied:
self.conn.execute('DELETE FROM metadata_dirtied WHERE book=?',
(book_id,))
+ # if a later exception prevents the commit, then the dirtied
+ # table will still have the book. No big deal, because the OPF
+ # is there and correct. We will simply do it again on next
+ # start
+ self.dirtied_cache.discard(book_id)
if commit:
self.conn.commit()
return True
def dirtied(self, book_ids, commit=True):
for book in book_ids:
+ if book in self.dirtied_cache:
+ print 'in dirty cache', book
+ continue
try:
self.conn.execute(
'INSERT INTO metadata_dirtied (book) VALUES (?)',
@@ -598,10 +607,30 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
self.dirtied_queue.put(book)
except IntegrityError:
# Already in table
- continue
+ pass
+ # If the commit doesn't happen, then our cache will be wrong. This
+ # could lead to a problem because we won't put the book back into
+ # the dirtied table. We deal with this by writing the dirty cache
+ # back to the table on GUI exit. Not perfect, but probably OK
+ self.dirtied_cache.add(book)
+ print 'added book', book
if commit:
self.conn.commit()
+ def commit_dirty_cache(self):
+ '''
+ Set the dirty indication for every book in the cache. The vast majority
+ of the time, the indication will already be set. However, sometimes
+ exceptions may have prevented a commit, which may remove some dirty
+ indications from the DB. This call will put them back. Note that there
+ is no problem with setting a dirty indication for a book that isn't in
+ fact dirty. Just wastes a few cycles.
+ '''
+ print 'commit cache'
+ book_ids = list(self.dirtied_cache)
+ self.dirtied_cache = set()
+ self.dirtied(book_ids)
+
def get_metadata(self, idx, index_is_id=False, get_cover=False):
'''
Convenience method to return metadata as a :class:`Metadata` object.
From b2b5e20c8f8c8a48eae23b288be7f347e09c0441 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Fri, 24 Sep 2010 13:51:22 -0600
Subject: [PATCH 127/207] Fourth beta
---
src/calibre/constants.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/calibre/constants.py b/src/calibre/constants.py
index 4c372c63a5..be387d8ca2 100644
--- a/src/calibre/constants.py
+++ b/src/calibre/constants.py
@@ -2,7 +2,7 @@ __license__ = 'GPL v3'
__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
__docformat__ = 'restructuredtext en'
__appname__ = 'calibre'
-__version__ = '0.7.902'
+__version__ = '0.7.903'
__author__ = "Kovid Goyal "
import re
From 02ce96cd688d14ba0b26bd4448d71d22ac704119 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Sat, 25 Sep 2010 20:46:45 -0600
Subject: [PATCH 128/207] Throttle OPF writer thread some more and framework
for restore from OPFs
---
src/calibre/library/caches.py | 2 +-
src/calibre/library/database2.py | 49 ++++---
src/calibre/library/restore.py | 190 +++++++++++++++++++++++++
src/calibre/utils/pyconsole/console.py | 2 +-
4 files changed, 223 insertions(+), 20 deletions(-)
create mode 100644 src/calibre/library/restore.py
diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index 1e52350e46..235584b9f7 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -48,7 +48,7 @@ class MetadataBackup(Thread): # {{{
time.sleep(2)
if not self.dump_func([id_]):
prints('Failed to backup metadata for id:', id_, 'again, giving up')
- time.sleep(0.2) # Limit to five per second
+ time.sleep(0.9) # Limit to one per second
# }}}
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index 94550f2804..ee7c3206bf 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -1198,38 +1198,41 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
else:
raise
if mi.title:
- self.set_title(id, mi.title)
+ self.set_title(id, mi.title, commit=False)
if not mi.authors:
mi.authors = [_('Unknown')]
authors = []
for a in mi.authors:
authors += string_to_authors(a)
- self.set_authors(id, authors, notify=False)
+ self.set_authors(id, authors, notify=False, commit=False)
if mi.author_sort:
- doit(self.set_author_sort, id, mi.author_sort, notify=False)
+ doit(self.set_author_sort, id, mi.author_sort, notify=False,
+ commit=False)
if mi.publisher:
- doit(self.set_publisher, id, mi.publisher, notify=False)
+ doit(self.set_publisher, id, mi.publisher, notify=False,
+ commit=False)
if mi.rating:
- doit(self.set_rating, id, mi.rating, notify=False)
+ doit(self.set_rating, id, mi.rating, notify=False, commit=False)
if mi.series:
- doit(self.set_series, id, mi.series, notify=False)
+ doit(self.set_series, id, mi.series, notify=False, commit=False)
if mi.cover_data[1] is not None:
doit(self.set_cover, id, mi.cover_data[1]) # doesn't use commit
elif mi.cover is not None and os.access(mi.cover, os.R_OK):
doit(self.set_cover, id, open(mi.cover, 'rb'))
if mi.tags:
- doit(self.set_tags, id, mi.tags, notify=False)
+ doit(self.set_tags, id, mi.tags, notify=False, commit=False)
if mi.comments:
- doit(self.set_comment, id, mi.comments, notify=False)
+ doit(self.set_comment, id, mi.comments, notify=False, commit=False)
if mi.isbn and mi.isbn.strip():
- doit(self.set_isbn, id, mi.isbn, notify=False)
+ doit(self.set_isbn, id, mi.isbn, notify=False, commit=False)
if mi.series_index:
- doit(self.set_series_index, id, mi.series_index, notify=False)
+ doit(self.set_series_index, id, mi.series_index, notify=False,
+ commit=False)
if mi.pubdate:
- doit(self.set_pubdate, id, mi.pubdate, notify=False)
+ doit(self.set_pubdate, id, mi.pubdate, notify=False, commit=False)
if getattr(mi, 'timestamp', None) is not None:
- doit(self.set_timestamp, id, mi.timestamp, notify=False)
- self.set_path(id, True)
+ doit(self.set_timestamp, id, mi.timestamp, notify=False,
+ commit=False)
user_mi = mi.get_all_user_metadata(make_copy=False)
for key in user_mi.iterkeys():
@@ -1238,7 +1241,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
doit(self.set_custom, id,
val=mi.get(key),
extra=mi.get_extra(key),
- label=user_mi[key]['label'])
+ label=user_mi[key]['label'], commit=False)
+ self.commit()
self.notify('metadata', [id])
def authors_sort_strings(self, id, index_is_id=False):
@@ -1929,7 +1933,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
else:
mi.tags.append(tag)
- def create_book_entry(self, mi, cover=None, add_duplicates=True):
+ def create_book_entry(self, mi, cover=None, add_duplicates=True,
+ force_id=None):
self._add_newbook_tag(mi)
if not add_duplicates and self.has_book(mi):
return None
@@ -1940,9 +1945,17 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
aus = aus.decode(preferred_encoding, 'replace')
if isbytestring(title):
title = title.decode(preferred_encoding, 'replace')
- obj = self.conn.execute('INSERT INTO books(title, series_index, author_sort) VALUES (?, ?, ?)',
- (title, series_index, aus))
- id = obj.lastrowid
+ if force_id is None:
+ obj = self.conn.execute('INSERT INTO books(title, series_index, author_sort) VALUES (?, ?, ?)',
+ (title, series_index, aus))
+ id = obj.lastrowid
+ else:
+ id = force_id
+ obj = self.conn.execute(
+ 'INSERT INTO books(id, title, series_index, '
+ 'author_sort) VALUES (?, ?, ?, ?)',
+ (id, title, series_index, aus))
+
self.data.books_added([id], self)
self.set_path(id, True)
self.conn.commit()
diff --git a/src/calibre/library/restore.py b/src/calibre/library/restore.py
new file mode 100644
index 0000000000..bdbb5e314a
--- /dev/null
+++ b/src/calibre/library/restore.py
@@ -0,0 +1,190 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+__license__ = 'GPL v3'
+__copyright__ = '2010, Kovid Goyal '
+__docformat__ = 'restructuredtext en'
+
+import re, os, traceback, shutil
+from threading import Thread
+from operator import itemgetter
+from textwrap import TextWrapper
+
+from calibre.ptempfile import TemporaryDirectory
+from calibre.ebooks.metadata.opf2 import OPF
+from calibre.library.database2 import LibraryDatabase2
+from calibre.constants import filesystem_encoding
+from calibre import isbytestring
+
+NON_EBOOK_EXTENSIONS = frozenset([
+ 'jpg', 'jpeg', 'gif', 'png', 'bmp',
+ 'opf', 'swp', 'swo'
+ ])
+
+class RestoreDatabase(LibraryDatabase2):
+
+ def set_path(self, book_id, *args, **kwargs):
+ pass
+
+class Restore(Thread):
+
+ def __init__(self, library_path, progress_callback=None):
+ if isbytestring(library_path):
+ library_path = library_path.decode(filesystem_encoding)
+ self.src_library_path = os.path.abspath(library_path)
+ self.progress_callback = progress_callback
+ self.db_id_regexp = re.compile(r'^.* \((\d+)\)$')
+ self.bad_ext_pat = re.compile(r'[^a-z]+')
+ if not callable(self.progress_callback):
+ self.progress_callback = lambda x, y: x
+ self.dirs = []
+ self.ignored_dirs = []
+ self.failed_dirs = []
+ self.books = []
+ self.conflicting_custom_cols = {}
+ self.failed_restores = []
+
+ @property
+ def errors_occurred(self):
+ return self.failed_dirs or \
+ self.conflicting_custom_cols or self.failed_restores
+
+ @property
+ def report(self):
+ ans = ''
+ failures = list(self.failed_dirs) + [(x['dirpath'], tb) for x, tb in
+ self.failed_restores]
+ if failures:
+ ans += 'Failed to restore the books in the following folders:\n'
+ wrap = TextWrapper(initial_indent='\t\t', width=85)
+ for dirpath, tb in failures:
+ ans += '\t' + dirpath + ' with error:\n'
+ ans += wrap.fill(tb)
+ ans += '\n'
+
+ if self.conflicting_custom_cols:
+ ans += '\n\n'
+ ans += 'The following custom columns were not fully restored:\n'
+ for x in self.conflicting_custom_cols:
+ ans += '\t#'+x+'\n'
+
+ return ans
+
+
+ def run(self):
+ with TemporaryDirectory('_library_restore') as tdir:
+ self.library_path = tdir
+ self.scan_library()
+ self.create_cc_metadata()
+ self.restore_books()
+ self.replace_db()
+
+ def scan_library(self):
+ for dirpath, dirnames, filenames in os.walk(self.src_library_path):
+ leaf = os.path.basename(dirpath)
+ m = self.db_id_regexp.search(leaf)
+ if m is None or 'metadata.opf' not in filenames:
+ self.ignored_dirs.append(dirpath)
+ continue
+ self.dirs.append((dirpath, filenames, m.group(1)))
+
+ self.progress_callback(None, len(self.dirs))
+ for i, x in enumerate(self.dirs):
+ dirpath, filenames, book_id = x
+ try:
+ self.process_dir(dirpath, filenames, book_id)
+ except:
+ self.failed_dirs.append((dirpath, traceback.format_exc()))
+ self.progress_callback(_('Processed') + repr(dirpath), i+1)
+
+ def is_ebook_file(self, filename):
+ ext = os.path.splitext(filename)[1]
+ if not ext:
+ return False
+ ext = ext[1:].lower()
+ if ext in NON_EBOOK_EXTENSIONS or \
+ self.bad_ext_pat.search(ext) is not None:
+ return False
+ return True
+
+ def process_dir(self, dirpath, filenames, book_id):
+ formats = filter(self.is_ebook_file, filenames)
+ fmts = [os.path.splitext(x)[1][1:].upper() for x in formats]
+ sizes = [os.path.getsize(os.path.join(dirpath, x)) for x in formats]
+ names = [os.path.splitext(x)[0] for x in formats]
+ opf = os.path.join(dirpath, 'metadata.opf')
+ mi = OPF(opf).to_book_metadata()
+ timestamp = os.path.getmtime(opf)
+ path = os.path.relpath(dirpath, self.src_library_path).replace(os.sep,
+ '/')
+
+ self.books.append({
+ 'mi': mi,
+ 'timestamp': timestamp,
+ 'formats': list(zip(fmts, sizes, names)),
+ 'id': int(book_id),
+ 'dirpath': dirpath,
+ 'path': path,
+ })
+
+ def create_cc_metadata(self):
+ self.books.sort(key=itemgetter('timestamp'))
+ m = {}
+ fields = ('label', 'name', 'datatype', 'is_multiple', 'editable',
+ 'display')
+ for b in self.books:
+ args = []
+ for x in fields:
+ if x in b:
+ args.append(b[x])
+ if len(args) == len(fields):
+ # TODO: Do series type columns need special handling?
+ label = b['label']
+ if label in m and args != m[label]:
+ if label not in self.conflicting_custom_cols:
+ self.conflicting_custom_cols[label] = set([m[label]])
+ self.conflicting_custom_cols[label].add(args)
+ m[b['label']] = args
+
+ db = LibraryDatabase2(self.library_path)
+ for args in m.values():
+ db.create_custom_column(*args)
+ db.conn.close()
+
+ def restore_books(self):
+ self.progress_callback(None, len(self.books))
+ self.books.sort(key=itemgetter('id'))
+
+ db = RestoreDatabase(self.library_path)
+
+ for i, book in enumerate(self.books):
+ try:
+ self.restore_book(book, db)
+ except:
+ self.failed_restores.append((book, traceback.format_exc()))
+ self.progress_callback(book['mi'].title, i+1)
+
+ db.conn.close()
+
+ def restore_book(self, book, db):
+ db.create_book_entry(book['mi'], add_duplicates=True,
+ force_id=book['id'])
+ db.conn.execute('UPDATE books SET path=? WHERE id=?', (book['path'],
+ book['id']))
+
+ for fmt, size, name in book['formats']:
+ db.conn.execute('''
+ INSERT INTO data (book,format,uncompressed_size,name)
+ VALUES (?,?,?,?)''', (id, fmt, size, name))
+ db.conn.commit()
+
+ def replace_db(self):
+ dbpath = os.path.join(self.src_library_path, 'metadata.db')
+ ndbpath = os.path.join(self.library_path, 'metadata.db')
+
+ save_path = os.path.splitext(dbpath)[0]+'_pre_restore.db'
+ if os.path.exists(save_path):
+ os.remove(save_path)
+ os.rename(dbpath, save_path)
+ shutil.copyfile(ndbpath, dbpath)
+
diff --git a/src/calibre/utils/pyconsole/console.py b/src/calibre/utils/pyconsole/console.py
index 14670fdb59..13c22a928f 100644
--- a/src/calibre/utils/pyconsole/console.py
+++ b/src/calibre/utils/pyconsole/console.py
@@ -171,7 +171,7 @@ class Console(QTextEdit):
def shutdown(self):
dynamic.set('console_history', self.history.serialize())
- self.shutton_down = True
+ self.shutting_down = True
for c in self.controllers:
c.kill()
From 42ec47607c9d4272bf23b122d865e7c2c7c7aad9 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Sat, 25 Sep 2010 21:22:59 -0600
Subject: [PATCH 129/207] Create restore command for calibredb
---
src/calibre/library/cli.py | 48 +++++++++++++++++++++++++++++++++-
src/calibre/library/restore.py | 22 +++++++++++-----
2 files changed, 62 insertions(+), 8 deletions(-)
diff --git a/src/calibre/library/cli.py b/src/calibre/library/cli.py
index 6ff17b0781..1d81ac2bd6 100644
--- a/src/calibre/library/cli.py
+++ b/src/calibre/library/cli.py
@@ -877,8 +877,54 @@ def command_saved_searches(args, dbpath):
COMMANDS = ('list', 'add', 'remove', 'add_format', 'remove_format',
'show_metadata', 'set_metadata', 'export', 'catalog',
'saved_searches', 'add_custom_column', 'custom_columns',
- 'remove_custom_column', 'set_custom')
+ 'remove_custom_column', 'set_custom', 'restore_database')
+def restore_database_option_parser():
+ parser = get_parser(_(
+ '''
+ %prog restore_database [options]
+
+ Restore this database from the metadata stored in OPF
+ files in each directory of the calibre library. This is
+ useful if your metadata.db file has been corrupted.
+
+ WARNING: This completely regenrates your datbase. You will
+ lose stored per-book conversion settings and custom recipes.
+ '''))
+ return parser
+
+def command_restore_database(args, dbpath):
+ from calibre.library.restore import Restore
+ parser = saved_searches_option_parser()
+ opts, args = parser.parse_args(args)
+ if len(args) != 0:
+ parser.print_help()
+ return 1
+
+ class Progress(object):
+ def __init__(self): self.total = 1
+
+ def __call__(self, msg, step):
+ if msg is None:
+ self.total = float(step)
+ else:
+ prints(msg, '...', '%d%%'%int(100*(step/self.total)))
+ r = Restore(dbpath, progress_callback=Progress())
+ r.start()
+ r.join()
+
+ if r.tb is not None:
+ prints('Restoring database failed with error:')
+ prints(r.tb)
+ else:
+ prints('Restoring database succeeded')
+ if r.errors_occurred:
+ name = 'calibre_db_restore_report.txt'
+ open('calibre_db_restore_report.txt',
+ 'wb').write(r.report.encode('utf-8'))
+ prints('Some errors occurred. A detailed report was '
+ 'saved to', name)
+ send_message()
def option_parser():
parser = OptionParser(_(
diff --git a/src/calibre/library/restore.py b/src/calibre/library/restore.py
index bdbb5e314a..0381366810 100644
--- a/src/calibre/library/restore.py
+++ b/src/calibre/library/restore.py
@@ -23,12 +23,16 @@ NON_EBOOK_EXTENSIONS = frozenset([
class RestoreDatabase(LibraryDatabase2):
- def set_path(self, book_id, *args, **kwargs):
+ def set_path(self, *args, **kwargs):
+ pass
+
+ def dirtied(self, *args, **kwargs):
pass
class Restore(Thread):
def __init__(self, library_path, progress_callback=None):
+ super(Restore, self).__init__()
if isbytestring(library_path):
library_path = library_path.decode(filesystem_encoding)
self.src_library_path = os.path.abspath(library_path)
@@ -43,6 +47,7 @@ class Restore(Thread):
self.books = []
self.conflicting_custom_cols = {}
self.failed_restores = []
+ self.tb = None
@property
def errors_occurred(self):
@@ -72,12 +77,15 @@ class Restore(Thread):
def run(self):
- with TemporaryDirectory('_library_restore') as tdir:
- self.library_path = tdir
- self.scan_library()
- self.create_cc_metadata()
- self.restore_books()
- self.replace_db()
+ try:
+ with TemporaryDirectory('_library_restore') as tdir:
+ self.library_path = tdir
+ self.scan_library()
+ self.create_cc_metadata()
+ self.restore_books()
+ self.replace_db()
+ except:
+ self.tb = traceback.format_exc()
def scan_library(self):
for dirpath, dirnames, filenames in os.walk(self.src_library_path):
From 5b8a645050f14de906e7c4a83b4440b7de0a0c99 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Sat, 25 Sep 2010 22:38:19 -0600
Subject: [PATCH 130/207] Move backup I/O into backup thread from GUI thread to
prevent GUI slowdown when the calibre library is on a slow device like a
network share
---
src/calibre/library/caches.py | 30 +++++++++++++++++++++++++-----
src/calibre/library/database2.py | 13 +++++++++----
2 files changed, 34 insertions(+), 9 deletions(-)
diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index 235584b9f7..8261ca40fb 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -28,6 +28,7 @@ class MetadataBackup(Thread): # {{{
self.daemon = True
self.db = db
self.dump_func = dump_func
+ self.dump_queue = Queue()
self.keep_running = True
def stop(self):
@@ -42,13 +43,32 @@ class MetadataBackup(Thread): # {{{
except:
# Happens during interpreter shutdown
break
- if self.dump_func([id_]) is None:
+ if self.dump_func([id_], dump_queue=self.dump_queue) is None:
# An exception occurred in dump_func, retry once
- prints('Failed to backup metadata for id:', id_, 'once')
+ prints('Failed to get backup metadata for id:', id_, 'once')
time.sleep(2)
- if not self.dump_func([id_]):
- prints('Failed to backup metadata for id:', id_, 'again, giving up')
- time.sleep(0.9) # Limit to one per second
+ if not self.dump_func([id_], dump_queue=self.dump_queue):
+ prints('Failed to get backup metadata for id:', id_, 'again, giving up')
+ while True:
+ try:
+ path, raw = self.dump_queue.get_nowait()
+ except:
+ break
+ else:
+ try:
+ with open(path, 'wb') as f:
+ f.write(raw)
+ except:
+ prints('Failed to write backup metadata for id:', id_, 'once')
+ time.sleep(2)
+ try:
+ with open(path, 'wb') as f:
+ f.write(raw)
+ except:
+ prints('Failed to write backup metadata for id:', id_,
+ 'again, giving up')
+
+ time.sleep(0.2) # Limit to five per second
# }}}
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index ee7c3206bf..61c52cf6b7 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -566,7 +566,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
def metadata_for_field(self, key):
return self.field_metadata[key]
- def dump_metadata(self, book_ids=None, remove_from_dirtied=True, commit=True):
+ def dump_metadata(self, book_ids=None, remove_from_dirtied=True,
+ commit=True, dump_queue=None):
'Write metadata for each record to an individual OPF file'
if book_ids is None:
book_ids = [x[0] for x in self.conn.get(
@@ -580,9 +581,13 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
# cover is set/removed
mi.cover = 'cover.jpg'
raw = metadata_to_opf(mi)
- path = self.abspath(book_id, index_is_id=True)
- with open(os.path.join(path, 'metadata.opf'), 'wb') as f:
- f.write(raw)
+ path = os.path.join(self.abspath(book_id, index_is_id=True),
+ 'metadata.opf')
+ if dump_queue is None:
+ with open(path, 'wb') as f:
+ f.write(raw)
+ else:
+ dump_queue.put((path, raw))
if remove_from_dirtied:
self.conn.execute('DELETE FROM metadata_dirtied WHERE book=?',
(book_id,))
From cdc5127fa40201a30a4c3a535cde335c990ac91d Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Sat, 25 Sep 2010 22:50:44 -0600
Subject: [PATCH 131/207] ...
---
src/calibre/library/cli.py | 10 +++++++++-
src/calibre/library/restore.py | 8 ++++----
2 files changed, 13 insertions(+), 5 deletions(-)
diff --git a/src/calibre/library/cli.py b/src/calibre/library/cli.py
index 1d81ac2bd6..7f181edc38 100644
--- a/src/calibre/library/cli.py
+++ b/src/calibre/library/cli.py
@@ -10,7 +10,8 @@ Command line interface to the calibre database.
import sys, os, cStringIO, re
from textwrap import TextWrapper
-from calibre import terminal_controller, preferred_encoding, prints
+from calibre import terminal_controller, preferred_encoding, prints, \
+ isbytestring
from calibre.utils.config import OptionParser, prefs, tweaks
from calibre.ebooks.metadata.meta import get_metadata
from calibre.library.database2 import LibraryDatabase2
@@ -901,6 +902,12 @@ def command_restore_database(args, dbpath):
parser.print_help()
return 1
+ if opts.library_path is not None:
+ dbpath = opts.library_path
+
+ if isbytestring(dbpath):
+ dbpath = dbpath.decode(preferred_encoding)
+
class Progress(object):
def __init__(self): self.total = 1
@@ -918,6 +925,7 @@ def command_restore_database(args, dbpath):
prints(r.tb)
else:
prints('Restoring database succeeded')
+ prints('old database saved as', r.olddb)
if r.errors_occurred:
name = 'calibre_db_restore_report.txt'
open('calibre_db_restore_report.txt',
diff --git a/src/calibre/library/restore.py b/src/calibre/library/restore.py
index 0381366810..63cd152ae9 100644
--- a/src/calibre/library/restore.py
+++ b/src/calibre/library/restore.py
@@ -61,7 +61,7 @@ class Restore(Thread):
self.failed_restores]
if failures:
ans += 'Failed to restore the books in the following folders:\n'
- wrap = TextWrapper(initial_indent='\t\t', width=85)
+ wrap = TextWrapper(initial_indent='\t\t', width=1085)
for dirpath, tb in failures:
ans += '\t' + dirpath + ' with error:\n'
ans += wrap.fill(tb)
@@ -103,7 +103,7 @@ class Restore(Thread):
self.process_dir(dirpath, filenames, book_id)
except:
self.failed_dirs.append((dirpath, traceback.format_exc()))
- self.progress_callback(_('Processed') + repr(dirpath), i+1)
+ self.progress_callback(_('Processed') + ' ' + dirpath, i+1)
def is_ebook_file(self, filename):
ext = os.path.splitext(filename)[1]
@@ -183,14 +183,14 @@ class Restore(Thread):
for fmt, size, name in book['formats']:
db.conn.execute('''
INSERT INTO data (book,format,uncompressed_size,name)
- VALUES (?,?,?,?)''', (id, fmt, size, name))
+ VALUES (?,?,?,?)''', (book['id'], fmt, size, name))
db.conn.commit()
def replace_db(self):
dbpath = os.path.join(self.src_library_path, 'metadata.db')
ndbpath = os.path.join(self.library_path, 'metadata.db')
- save_path = os.path.splitext(dbpath)[0]+'_pre_restore.db'
+ save_path = self.olddb = os.path.splitext(dbpath)[0]+'_pre_restore.db'
if os.path.exists(save_path):
os.remove(save_path)
os.rename(dbpath, save_path)
From f208950bab4c8d483f50a7b36884c262162a9d24 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Sat, 25 Sep 2010 22:53:27 -0600
Subject: [PATCH 132/207] ...
---
src/calibre/library/restore.py | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/src/calibre/library/restore.py b/src/calibre/library/restore.py
index 63cd152ae9..83e6565937 100644
--- a/src/calibre/library/restore.py
+++ b/src/calibre/library/restore.py
@@ -8,7 +8,6 @@ __docformat__ = 'restructuredtext en'
import re, os, traceback, shutil
from threading import Thread
from operator import itemgetter
-from textwrap import TextWrapper
from calibre.ptempfile import TemporaryDirectory
from calibre.ebooks.metadata.opf2 import OPF
@@ -61,11 +60,10 @@ class Restore(Thread):
self.failed_restores]
if failures:
ans += 'Failed to restore the books in the following folders:\n'
- wrap = TextWrapper(initial_indent='\t\t', width=1085)
for dirpath, tb in failures:
ans += '\t' + dirpath + ' with error:\n'
- ans += wrap.fill(tb)
- ans += '\n'
+ ans += '\n'.join('\t\t'+x for x in tb.splitlines())
+ ans += '\n\n'
if self.conflicting_custom_cols:
ans += '\n\n'
From fca7c92ca45d4e62bea2d5f8708e47b71d5c32f6 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Sat, 25 Sep 2010 22:58:29 -0600
Subject: [PATCH 133/207] ...
---
src/calibre/library/cli.py | 1 -
src/calibre/library/restore.py | 4 ++++
2 files changed, 4 insertions(+), 1 deletion(-)
diff --git a/src/calibre/library/cli.py b/src/calibre/library/cli.py
index 7f181edc38..19bd56bf55 100644
--- a/src/calibre/library/cli.py
+++ b/src/calibre/library/cli.py
@@ -932,7 +932,6 @@ def command_restore_database(args, dbpath):
'wb').write(r.report.encode('utf-8'))
prints('Some errors occurred. A detailed report was '
'saved to', name)
- send_message()
def option_parser():
parser = OptionParser(_(
diff --git a/src/calibre/library/restore.py b/src/calibre/library/restore.py
index 83e6565937..48e66e508f 100644
--- a/src/calibre/library/restore.py
+++ b/src/calibre/library/restore.py
@@ -46,6 +46,7 @@ class Restore(Thread):
self.books = []
self.conflicting_custom_cols = {}
self.failed_restores = []
+ self.successes = 0
self.tb = None
@property
@@ -81,6 +82,8 @@ class Restore(Thread):
self.scan_library()
self.create_cc_metadata()
self.restore_books()
+ if self.successes == 0 and len(self.dirs) > 0:
+ raise Exception(('Something bad happened'))
self.replace_db()
except:
self.tb = traceback.format_exc()
@@ -183,6 +186,7 @@ class Restore(Thread):
INSERT INTO data (book,format,uncompressed_size,name)
VALUES (?,?,?,?)''', (book['id'], fmt, size, name))
db.conn.commit()
+ self.successes += 1
def replace_db(self):
dbpath = os.path.join(self.src_library_path, 'metadata.db')
From da949796b0630c049f29f4da0d45037204dc8c54 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Sat, 25 Sep 2010 23:03:29 -0600
Subject: [PATCH 134/207] ...
---
src/calibre/gui2/ui.py | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py
index 6b04f6fa1f..1e7c7550f8 100644
--- a/src/calibre/gui2/ui.py
+++ b/src/calibre/gui2/ui.py
@@ -360,6 +360,10 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, # {{{
def library_moved(self, newloc):
if newloc is None: return
+ try:
+ olddb = self.library_view.model().db
+ except:
+ olddb = None
db = LibraryDatabase2(newloc)
self.library_path = newloc
self.book_on_device(None, reset=True)
@@ -380,6 +384,12 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, # {{{
self.apply_named_search_restriction('') # reset restriction to null
self.saved_searches_changed() # reload the search restrictions combo box
self.apply_named_search_restriction(db.prefs['gui_restriction'])
+ if olddb is not None:
+ try:
+ olddb.conn.close()
+ except:
+ import traceback
+ traceback.print_exc()
def set_window_title(self):
self.setWindowTitle(__appname__ + u' - ||%s||'%self.iactions['Choose Library'].library_name())
From 2d406112a477d3b65e014e548252fefdaa806de5 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Sat, 25 Sep 2010 23:26:28 -0600
Subject: [PATCH 135/207] Grrr keep only the file write in the GUI thread not
the other way around
---
src/calibre/gui2/library/models.py | 2 +-
src/calibre/library/caches.py | 64 ++++++++++++++++++------------
src/calibre/library/database2.py | 2 +-
3 files changed, 40 insertions(+), 28 deletions(-)
diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index fe1701a918..01f597caf4 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -157,7 +157,7 @@ class BooksModel(QAbstractTableModel): # {{{
self.cover_cache = CoverCache(db, FunctionDispatcher(self.db.cover))
self.cover_cache.start()
self.metadata_backup = MetadataBackup(db,
- FunctionDispatcher(self.db.dump_metadata))
+ self.db.dump_metadata)
self.metadata_backup.start()
def refresh_cover(event, ids):
if event == 'cover' and self.cover_cache is not None:
diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index 8261ca40fb..e675c97c76 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -28,8 +28,9 @@ class MetadataBackup(Thread): # {{{
self.daemon = True
self.db = db
self.dump_func = dump_func
- self.dump_queue = Queue()
self.keep_running = True
+ from calibre.gui2 import FunctionDispatcher
+ self.do_write = FunctionDispatcher(self.write)
def stop(self):
self.keep_running = False
@@ -43,32 +44,43 @@ class MetadataBackup(Thread): # {{{
except:
# Happens during interpreter shutdown
break
- if self.dump_func([id_], dump_queue=self.dump_queue) is None:
- # An exception occurred in dump_func, retry once
- prints('Failed to get backup metadata for id:', id_, 'once')
- time.sleep(2)
- if not self.dump_func([id_], dump_queue=self.dump_queue):
- prints('Failed to get backup metadata for id:', id_, 'again, giving up')
- while True:
- try:
- path, raw = self.dump_queue.get_nowait()
- except:
- break
- else:
- try:
- with open(path, 'wb') as f:
- f.write(raw)
- except:
- prints('Failed to write backup metadata for id:', id_, 'once')
- time.sleep(2)
- try:
- with open(path, 'wb') as f:
- f.write(raw)
- except:
- prints('Failed to write backup metadata for id:', id_,
- 'again, giving up')
- time.sleep(0.2) # Limit to five per second
+ dump = []
+ try:
+ self.dump_func([id_], dump_queue=dump)
+ except:
+ prints('Failed to get backup metadata for id:', id_, 'once')
+ import traceback
+ traceback.print_exc()
+ time.sleep(2)
+ dump = []
+ try:
+ self.dump_func([id_], dump_queue=dump)
+ except:
+ prints('Failed to get backup metadata for id:', id_, 'again, giving up')
+ traceback.print_exc()
+ continue
+ try:
+ path, raw = dump[0]
+ except:
+ break
+ try:
+ self.do_write(path, raw)
+ except:
+ prints('Failed to write backup metadata for id:', id_, 'once')
+ time.sleep(2)
+ try:
+ self.do_write(path, raw)
+ except:
+ prints('Failed to write backup metadata for id:', id_,
+ 'again, giving up')
+
+ time.sleep(0.5) # Limit to two per second
+
+ def write(self, path, raw):
+ with open(path, 'wb') as f:
+ f.write(raw)
+
# }}}
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index 61c52cf6b7..8c34510de4 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -587,7 +587,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
with open(path, 'wb') as f:
f.write(raw)
else:
- dump_queue.put((path, raw))
+ dump_queue.append((path, raw))
if remove_from_dirtied:
self.conn.execute('DELETE FROM metadata_dirtied WHERE book=?',
(book_id,))
From 04ccd041e9b0abb5661c333d04e2da9a9e10b5e9 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Sat, 25 Sep 2010 23:35:12 -0600
Subject: [PATCH 136/207] ...
---
src/calibre/gui2/library/models.py | 3 +--
src/calibre/library/caches.py | 13 +++++++++----
src/calibre/library/database2.py | 6 +++---
3 files changed, 13 insertions(+), 9 deletions(-)
diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index 01f597caf4..a2555cfc56 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -156,8 +156,7 @@ class BooksModel(QAbstractTableModel): # {{{
self.cover_cache.stop()
self.cover_cache = CoverCache(db, FunctionDispatcher(self.db.cover))
self.cover_cache.start()
- self.metadata_backup = MetadataBackup(db,
- self.db.dump_metadata)
+ self.metadata_backup = MetadataBackup(db)
self.metadata_backup.start()
def refresh_cover(event, ids):
if event == 'cover' and self.cover_cache is not None:
diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index e675c97c76..bc16681f81 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -22,12 +22,17 @@ from calibre.ebooks.metadata import title_sort
from calibre import fit_image, prints
class MetadataBackup(Thread): # {{{
+ '''
+ Continuously backup changed metadata into OPF files
+ in the book directory. This class runs in its own
+ thread and makes sure that the actual file write happens in the
+ GUI thread to prevent Windows' file locking from causing problems.
+ '''
- def __init__(self, db, dump_func):
+ def __init__(self, db):
Thread.__init__(self)
self.daemon = True
self.db = db
- self.dump_func = dump_func
self.keep_running = True
from calibre.gui2 import FunctionDispatcher
self.do_write = FunctionDispatcher(self.write)
@@ -47,7 +52,7 @@ class MetadataBackup(Thread): # {{{
dump = []
try:
- self.dump_func([id_], dump_queue=dump)
+ self.db.dump_metadata([id_], dump_to=dump)
except:
prints('Failed to get backup metadata for id:', id_, 'once')
import traceback
@@ -55,7 +60,7 @@ class MetadataBackup(Thread): # {{{
time.sleep(2)
dump = []
try:
- self.dump_func([id_], dump_queue=dump)
+ self.db.dump_metadata([id_], dump_to=dump)
except:
prints('Failed to get backup metadata for id:', id_, 'again, giving up')
traceback.print_exc()
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index 8c34510de4..39770068bb 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -567,7 +567,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
return self.field_metadata[key]
def dump_metadata(self, book_ids=None, remove_from_dirtied=True,
- commit=True, dump_queue=None):
+ commit=True, dump_to=None):
'Write metadata for each record to an individual OPF file'
if book_ids is None:
book_ids = [x[0] for x in self.conn.get(
@@ -583,11 +583,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
raw = metadata_to_opf(mi)
path = os.path.join(self.abspath(book_id, index_is_id=True),
'metadata.opf')
- if dump_queue is None:
+ if dump_to is None:
with open(path, 'wb') as f:
f.write(raw)
else:
- dump_queue.append((path, raw))
+ dump_to.append((path, raw))
if remove_from_dirtied:
self.conn.execute('DELETE FROM metadata_dirtied WHERE book=?',
(book_id,))
From 37eadd1c10c316ddb41cce51a7efc8b30487cb22 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Sat, 25 Sep 2010 23:37:21 -0600
Subject: [PATCH 137/207] ...
---
src/calibre/library/database2.py | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index 39770068bb..aeaed9b46e 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -568,7 +568,12 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
def dump_metadata(self, book_ids=None, remove_from_dirtied=True,
commit=True, dump_to=None):
- 'Write metadata for each record to an individual OPF file'
+ '''
+ Write metadata for each record to an individual OPF file
+
+ :param dump_to: None or list. If list then instead of writing to file,
+ data is append to list
+ '''
if book_ids is None:
book_ids = [x[0] for x in self.conn.get(
'SELECT book FROM metadata_dirtied', all=True)]
From 14537228158d25572f1e9ca7aa67922cf31e44f2 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sun, 26 Sep 2010 11:00:32 +0100
Subject: [PATCH 138/207] 1) Fix problem with editing bool custom column
metadata 2) make on-send and manual metadata managment work with sonys
---
src/calibre/devices/usbms/books.py | 31 +++++++++++++++++--
.../gui2/preferences/create_custom_column.py | 2 +-
2 files changed, 29 insertions(+), 4 deletions(-)
diff --git a/src/calibre/devices/usbms/books.py b/src/calibre/devices/usbms/books.py
index 13fcb90b49..915d937379 100644
--- a/src/calibre/devices/usbms/books.py
+++ b/src/calibre/devices/usbms/books.py
@@ -111,6 +111,12 @@ class CollectionsBookList(BookList):
from calibre.devices.usbms.driver import debug_print
debug_print('Starting get_collections:', prefs['manage_device_metadata'])
debug_print('Renaming rules:', tweaks['sony_collection_renaming_rules'])
+
+ # Complexity: we can use renaming rules only when using automatic
+ # management. Otherwise we don't always have the metadata to make the
+ # right decisions
+ use_renaming_rules = prefs['manage_device_metadata'] == 'on_connect'
+
collections = {}
# This map of sets is used to avoid linear searches when testing for
# book equality
@@ -139,7 +145,16 @@ class CollectionsBookList(BookList):
attrs = collection_attributes
for attr in attrs:
attr = attr.strip()
- ign, val, orig_val, fm = book.format_field_extended(attr)
+ # If attr is device_collections, then we cannot use
+ # format_field, because we don't know the fields where the
+ # values came from.
+ if attr == 'device_collections':
+ doing_dc = True
+ val = book.device_collections # is a list
+ else:
+ doing_dc = False
+ ign, val, orig_val, fm = book.format_field_extended(attr)
+
if not val: continue
if isbytestring(val):
val = val.decode(preferred_encoding, 'replace')
@@ -151,9 +166,15 @@ class CollectionsBookList(BookList):
val = orig_val
else:
val = [val]
+
for category in val:
is_series = False
- if fm['is_custom']: # is a custom field
+ if doing_dc:
+ # Attempt to determine if this value is a series by
+ # comparing it to the series name.
+ if category == book.series:
+ is_series = True
+ elif fm['is_custom']: # is a custom field
if fm['datatype'] == 'text' and len(category) > 1 and \
category[0] == '[' and category[-1] == ']':
continue
@@ -167,7 +188,11 @@ class CollectionsBookList(BookList):
('series' in collection_attributes and
book.get('series', None) == category):
is_series = True
- cat_name = self.compute_category_name(attr, category, fm)
+ if use_renaming_rules:
+ cat_name = self.compute_category_name(attr, category, fm)
+ else:
+ cat_name = category
+
if cat_name not in collections:
collections[cat_name] = []
collections_lpaths[cat_name] = set()
diff --git a/src/calibre/gui2/preferences/create_custom_column.py b/src/calibre/gui2/preferences/create_custom_column.py
index bec21270df..ebf33784d4 100644
--- a/src/calibre/gui2/preferences/create_custom_column.py
+++ b/src/calibre/gui2/preferences/create_custom_column.py
@@ -38,7 +38,7 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
'is_multiple':False},
8:{'datatype':'bool',
'text':_('Yes/No'), 'is_multiple':False},
- 8:{'datatype':'composite',
+ 9:{'datatype':'composite',
'text':_('Column built from other columns'), 'is_multiple':False},
}
From e3781d0f15e4b7fa9dec8f55348dae129172c52f Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sun, 26 Sep 2010 12:38:34 +0100
Subject: [PATCH 139/207] 1) add force renumber series to custom series bulk
editing 2) add clear series to both standard and custom series bulk editing
---
src/calibre/gui2/custom_column_widgets.py | 44 +++++--
src/calibre/gui2/dialogs/metadata_bulk.py | 8 +-
src/calibre/gui2/dialogs/metadata_bulk.ui | 137 +++++++++++++---------
3 files changed, 124 insertions(+), 65 deletions(-)
diff --git a/src/calibre/gui2/custom_column_widgets.py b/src/calibre/gui2/custom_column_widgets.py
index 90abfc2474..1d265fea1e 100644
--- a/src/calibre/gui2/custom_column_widgets.py
+++ b/src/calibre/gui2/custom_column_widgets.py
@@ -452,9 +452,25 @@ class BulkSeries(BulkBase):
self.name_widget = w
self.widgets = [QLabel('&'+self.col_metadata['name']+':', parent), w]
- self.widgets.append(QLabel(_('Automatically number books in this series'), parent))
- self.idx_widget=QCheckBox(parent)
- self.widgets.append(self.idx_widget)
+ self.widgets.append(QLabel('', parent))
+ w = QWidget(parent)
+ layout = QHBoxLayout(w)
+ layout.setContentsMargins(0, 0, 0, 0)
+ self.remove_series = QCheckBox(parent)
+ self.remove_series.setText(_('Remove series'))
+ layout.addWidget(self.remove_series)
+ self.idx_widget = QCheckBox(parent)
+ self.idx_widget.setText(_('Automatically number books'))
+ layout.addWidget(self.idx_widget)
+ self.force_number = QCheckBox(parent)
+ self.force_number.setText(_('Force numbers to start with '))
+ layout.addWidget(self.force_number)
+ self.series_start_number = QSpinBox(parent)
+ self.series_start_number.setMinimum(1)
+ self.series_start_number.setProperty("value", 1)
+ layout.addWidget(self.series_start_number)
+ layout.addItem(QSpacerItem(20, 10, QSizePolicy.Expanding, QSizePolicy.Minimum))
+ self.widgets.append(w)
def initialize(self, book_id):
self.idx_widget.setChecked(False)
@@ -465,17 +481,27 @@ class BulkSeries(BulkBase):
def getter(self):
n = unicode(self.name_widget.currentText()).strip()
i = self.idx_widget.checkState()
- return n, i
+ f = self.force_number.checkState()
+ s = self.series_start_number.value()
+ r = self.remove_series.checkState()
+ return n, i, f, s, r
def commit(self, book_ids, notify=False):
- val, update_indices = self.gui_val
- val = self.normalize_ui_val(val)
- if val != '':
+ val, update_indices, force_start, at_value, clear = self.gui_val
+ val = '' if clear else self.normalize_ui_val(val)
+ if clear or val != '':
extras = []
next_index = self.db.get_next_cc_series_num_for(val, num=self.col_id)
+ print 'cc commit next index', next_index
for book_id in book_ids:
+ if clear:
+ extras.append(None)
+ continue
if update_indices:
- if tweaks['series_index_auto_increment'] == 'next':
+ if force_start:
+ s_index = at_value
+ at_value += 1
+ elif tweaks['series_index_auto_increment'] == 'next':
s_index = next_index
next_index += 1
else:
@@ -483,6 +509,8 @@ class BulkSeries(BulkBase):
else:
s_index = self.db.get_custom_extra(book_id, num=self.col_id,
index_is_id=True)
+ if s_index is None:
+ s_index = 1.0
extras.append(s_index)
self.db.set_custom_bulk(book_ids, val, extras=extras,
num=self.col_id, notify=notify)
diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py
index b14390e001..9c83b3aee5 100644
--- a/src/calibre/gui2/dialogs/metadata_bulk.py
+++ b/src/calibre/gui2/dialogs/metadata_bulk.py
@@ -32,7 +32,7 @@ class Worker(Thread):
remove, add, au, aus, do_aus, rating, pub, do_series, \
do_autonumber, do_remove_format, remove_format, do_swap_ta, \
do_remove_conv, do_auto_author, series, do_series_restart, \
- series_start_value, do_title_case = self.args
+ series_start_value, do_title_case, clear_series = self.args
# first loop: do author and title. These will commit at the end of each
# operation, because each operation modifies the file system. We want to
@@ -75,6 +75,9 @@ class Worker(Thread):
if pub:
self.db.set_publisher(id, pub, notify=False, commit=False)
+ if clear_series:
+ self.db.set_series(id, '', notify=False, commit=False)
+
if do_series:
if do_series_restart:
next = series_start_value
@@ -592,6 +595,7 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
rating = self.rating.value()
pub = unicode(self.publisher.text())
do_series = self.write_series
+ clear_series = self.clear_series.isChecked()
series = unicode(self.series.currentText()).strip()
do_autonumber = self.autonumber_series.isChecked()
do_series_restart = self.series_numbering_restarts.isChecked()
@@ -606,7 +610,7 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
args = (remove, add, au, aus, do_aus, rating, pub, do_series,
do_autonumber, do_remove_format, remove_format, do_swap_ta,
do_remove_conv, do_auto_author, series, do_series_restart,
- series_start_value, do_title_case)
+ series_start_value, do_title_case, clear_series)
bb = BlockingBusy(_('Applying changes to %d books. This may take a while.')
%len(self.ids), parent=self)
diff --git a/src/calibre/gui2/dialogs/metadata_bulk.ui b/src/calibre/gui2/dialogs/metadata_bulk.ui
index e03a59b7ea..60e24dbceb 100644
--- a/src/calibre/gui2/dialogs/metadata_bulk.ui
+++ b/src/calibre/gui2/dialogs/metadata_bulk.ui
@@ -225,61 +225,50 @@
-
-
- List of known series. You can add new series.
-
-
- List of known series. You can add new series.
-
-
- true
-
-
- QComboBox::InsertAlphabetically
-
-
- QComboBox::AdjustToContents
-
-
-
-
-
-
- Remove &format:
-
-
- remove_format
-
-
-
-
-
-
-
-
-
- true
-
-
-
-
-
-
- &Swap title and author
-
-
-
-
-
-
- Change title to title case
-
-
- Force the title to be in title case. If both this and swap authors are checked,
-title and author are swapped before the title case is set
-
-
+
+
+
+
+ List of known series. You can add new series.
+
+
+ List of known series. You can add new series.
+
+
+ true
+
+
+ QComboBox::InsertAlphabetically
+
+
+ QComboBox::AdjustToContents
+
+
+
+
+
+
+ If checked, the series will be cleared
+
+
+ Clear series
+
+
+
+
+
+
+ Qt::Horizontal
+
+
+
+ 20
+ 00
+
+
+
+
+
@@ -339,6 +328,44 @@ from the value in the box
+
+
+
+ Remove &format:
+
+
+ remove_format
+
+
+
+
+
+
+
+
+
+ true
+
+
+
+
+
+
+ &Swap title and author
+
+
+
+
+
+
+ Change title to title case
+
+
+ Force the title to be in title case. If both this and swap authors are checked,
+title and author are swapped before the title case is set
+
+
+
From 3e1cb3b5e08c98105f7ce1636f454619fdde3879 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sun, 26 Sep 2010 13:07:22 +0100
Subject: [PATCH 140/207] Make covercache and backup stoppable.
---
src/calibre/gui2/library/models.py | 2 ++
src/calibre/library/caches.py | 4 ++--
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index a2555cfc56..ef251a884a 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -156,6 +156,8 @@ class BooksModel(QAbstractTableModel): # {{{
self.cover_cache.stop()
self.cover_cache = CoverCache(db, FunctionDispatcher(self.db.cover))
self.cover_cache.start()
+ if self.metadata_backup is not None:
+ self.metadata_backup.stop()
self.metadata_backup = MetadataBackup(db)
self.metadata_backup.start()
def refresh_cover(event, ids):
diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index bc16681f81..8d449974a5 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -43,7 +43,7 @@ class MetadataBackup(Thread): # {{{
def run(self):
while self.keep_running:
try:
- id_ = self.db.dirtied_queue.get()
+ id_ = self.db.dirtied_queue.get(True, 2)
except Empty:
continue
except:
@@ -122,7 +122,7 @@ class CoverCache(Thread): # {{{
def run(self):
while self.keep_running:
try:
- id_ = self.load_queue.get()
+ id_ = self.load_queue.get(True, 2)
except Empty:
continue
except:
From c8dbd705467ed6d15bc443939c06997fcee5b91b Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sun, 26 Sep 2010 13:48:30 +0100
Subject: [PATCH 141/207] metadata backup that gets metadata on the GUI thread,
computes the OPF on a separate thread, then writes the file on the GUI
thread.
---
src/calibre/gui2/library/models.py | 1 +
src/calibre/library/caches.py | 26 ++++++++++++++++++--------
src/calibre/library/database2.py | 23 +++++++++++++++++++++++
3 files changed, 42 insertions(+), 8 deletions(-)
diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index ef251a884a..b908019bcb 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -89,6 +89,7 @@ class BooksModel(QAbstractTableModel): # {{{
self.alignment_map = {}
self.buffer_size = buffer
self.cover_cache = None
+ self.metadata_backup = None
self.bool_yes_icon = QIcon(I('ok.png'))
self.bool_no_icon = QIcon(I('list_remove.png'))
self.bool_blank_icon = QIcon(I('blank.png'))
diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index 8d449974a5..7d8a8624a9 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -19,6 +19,7 @@ from calibre.utils.date import parse_date, now, UNDEFINED_DATE
from calibre.utils.search_query_parser import SearchQueryParser
from calibre.utils.pyparsing import ParseException
from calibre.ebooks.metadata import title_sort
+from calibre.ebooks.metadata.opf2 import metadata_to_opf
from calibre import fit_image, prints
class MetadataBackup(Thread): # {{{
@@ -36,6 +37,8 @@ class MetadataBackup(Thread): # {{{
self.keep_running = True
from calibre.gui2 import FunctionDispatcher
self.do_write = FunctionDispatcher(self.write)
+ self.get_metadata_for_dump = FunctionDispatcher(db.get_metadata_for_dump)
+ self.clear_dirtied = FunctionDispatcher(db.clear_dirtied)
def stop(self):
self.keep_running = False
@@ -43,6 +46,7 @@ class MetadataBackup(Thread): # {{{
def run(self):
while self.keep_running:
try:
+ time.sleep(0.5) # Limit to two per second
id_ = self.db.dirtied_queue.get(True, 2)
except Empty:
continue
@@ -50,25 +54,27 @@ class MetadataBackup(Thread): # {{{
# Happens during interpreter shutdown
break
- dump = []
try:
- self.db.dump_metadata([id_], dump_to=dump)
+ path, mi = self.get_metadata_for_dump(id_)
except:
prints('Failed to get backup metadata for id:', id_, 'once')
import traceback
traceback.print_exc()
time.sleep(2)
- dump = []
try:
- self.db.dump_metadata([id_], dump_to=dump)
+ path, mi = self.get_metadata_for_dump(id_)
except:
prints('Failed to get backup metadata for id:', id_, 'again, giving up')
traceback.print_exc()
continue
try:
- path, raw = dump[0]
+ print 'now do metadata'
+ raw = metadata_to_opf(mi)
except:
- break
+ prints('Failed to convert to opf for id:', id_)
+ traceback.print_exc()
+ continue
+
try:
self.do_write(path, raw)
except:
@@ -79,8 +85,12 @@ class MetadataBackup(Thread): # {{{
except:
prints('Failed to write backup metadata for id:', id_,
'again, giving up')
+ continue
- time.sleep(0.5) # Limit to two per second
+ try:
+ self.clear_dirtied([id_])
+ except:
+ prints('Failed to clear dirtied for id:', id_)
def write(self, path, raw):
with open(path, 'wb') as f:
@@ -106,7 +116,6 @@ class CoverCache(Thread): # {{{
self.keep_running = False
def _image_for_id(self, id_):
- time.sleep(0.050) # Limit 20/second to not overwhelm the GUI
img = self.cover_func(id_, index_is_id=True, as_image=True)
if img is None:
img = QImage()
@@ -122,6 +131,7 @@ class CoverCache(Thread): # {{{
def run(self):
while self.keep_running:
try:
+ time.sleep(0.050) # Limit 20/second to not overwhelm the GUI
id_ = self.load_queue.get(True, 2)
except Empty:
continue
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index a6f3f286bc..e6587f06a2 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -566,6 +566,24 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
def metadata_for_field(self, key):
return self.field_metadata[key]
+ def clear_dirtied(self, book_ids=None):
+ '''
+ Clear the dirtied indicator for the books. This is used when fetching
+ metadata, creating an OPF, and writing a file are separated into steps.
+ The last step is clearing the indicator
+ '''
+ for book_id in book_ids:
+ if not self.data.has_id(book_id):
+ continue
+ self.conn.execute('DELETE FROM metadata_dirtied WHERE book=?',
+ (book_id,))
+ # if a later exception prevents the commit, then the dirtied
+ # table will still have the book. No big deal, because the OPF
+ # is there and correct. We will simply do it again on next
+ # start
+ self.dirtied_cache.discard(book_id)
+ self.conn.commit()
+
def dump_metadata(self, book_ids=None, remove_from_dirtied=True,
commit=True, dump_to=None):
'''
@@ -638,6 +656,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
self.dirtied_cache = set()
self.dirtied(book_ids)
+ def get_metadata_for_dump(self, idx):
+ path = os.path.join(self.abspath(idx, index_is_id=True), 'metadata.opf')
+ mi = self.get_metadata(idx, index_is_id=True)
+ return ((path, mi))
+
def get_metadata(self, idx, index_is_id=False, get_cover=False):
'''
Convenience method to return metadata as a :class:`Metadata` object.
From 12f75ddaf86e89c03013c75531aa1b5f25f7409a Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sun, 26 Sep 2010 13:56:31 +0100
Subject: [PATCH 142/207] Note why we don't do a join where we should.
---
src/calibre/gui2/library/models.py | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index b908019bcb..ff6d8b70f0 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -155,10 +155,14 @@ class BooksModel(QAbstractTableModel): # {{{
self.database_changed.emit(db)
if self.cover_cache is not None:
self.cover_cache.stop()
+ # Would like to to a join here, but the thread might be waiting to
+ # do something on the GUI thread. Deadlock.
self.cover_cache = CoverCache(db, FunctionDispatcher(self.db.cover))
self.cover_cache.start()
if self.metadata_backup is not None:
self.metadata_backup.stop()
+ # Would like to to a join here, but the thread might be waiting to
+ # do something on the GUI thread. Deadlock.
self.metadata_backup = MetadataBackup(db)
self.metadata_backup.start()
def refresh_cover(event, ids):
From f6870bd14b6cdf9041401a7f7c2ac02385bb4ae1 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sun, 26 Sep 2010 14:06:56 +0100
Subject: [PATCH 143/207] Introduce scheduling opportunities into the backup
thread
---
src/calibre/library/caches.py | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index 7d8a8624a9..a9ef6a4281 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -67,6 +67,11 @@ class MetadataBackup(Thread): # {{{
prints('Failed to get backup metadata for id:', id_, 'again, giving up')
traceback.print_exc()
continue
+
+ # Give the GUI thread a chance to do something. Python threads don't
+ # have priorities, so this thread would naturally keep the processor
+ # until some scheduling event happens. The sleep makes such an event
+ time.sleep(0.010)
try:
print 'now do metadata'
raw = metadata_to_opf(mi)
@@ -75,6 +80,7 @@ class MetadataBackup(Thread): # {{{
traceback.print_exc()
continue
+ time.sleep(0.010) # Give the GUI thread a chance to do something
try:
self.do_write(path, raw)
except:
@@ -87,6 +93,7 @@ class MetadataBackup(Thread): # {{{
'again, giving up')
continue
+ time.sleep(0.010) # Give the GUI thread a chance to do something
try:
self.clear_dirtied([id_])
except:
From 66ed343b1185f4a42976be447cfe0313e5223803 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sun, 26 Sep 2010 14:51:25 +0100
Subject: [PATCH 144/207] 1) Remove a print statement 2) fix series formatting
---
src/calibre/ebooks/metadata/book/base.py | 4 ++++
src/calibre/library/caches.py | 9 ++++-----
2 files changed, 8 insertions(+), 5 deletions(-)
diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index fd7ce8a6c3..0526de96a0 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -455,6 +455,8 @@ class Metadata(object):
res = format_date(res, cmeta['display'].get('date_format','dd MMM yyyy'))
elif datatype == 'bool':
res = _('Yes') if res else _('No')
+ elif datatype == 'float' and key.endswith('_index'):
+ res = self.format_series_index(res)
return (name, unicode(res), orig_res, cmeta)
if key in field_metadata and field_metadata[key]['kind'] == 'field':
@@ -468,6 +470,8 @@ class Metadata(object):
datatype = fmeta['datatype']
if key == 'authors':
res = authors_to_string(res)
+ elif key == 'series_index':
+ res = self.format_series_index(res)
elif datatype == 'text' and fmeta['is_multiple']:
res = u', '.join(res)
elif datatype == 'series' and series_with_index:
diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index a9ef6a4281..74be3cd594 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -53,7 +53,7 @@ class MetadataBackup(Thread): # {{{
except:
# Happens during interpreter shutdown
break
-
+ print 'doing id', id_
try:
path, mi = self.get_metadata_for_dump(id_)
except:
@@ -71,16 +71,15 @@ class MetadataBackup(Thread): # {{{
# Give the GUI thread a chance to do something. Python threads don't
# have priorities, so this thread would naturally keep the processor
# until some scheduling event happens. The sleep makes such an event
- time.sleep(0.010)
+ time.sleep(0.1)
try:
- print 'now do metadata'
raw = metadata_to_opf(mi)
except:
prints('Failed to convert to opf for id:', id_)
traceback.print_exc()
continue
- time.sleep(0.010) # Give the GUI thread a chance to do something
+ time.sleep(0.1) # Give the GUI thread a chance to do something
try:
self.do_write(path, raw)
except:
@@ -93,7 +92,7 @@ class MetadataBackup(Thread): # {{{
'again, giving up')
continue
- time.sleep(0.010) # Give the GUI thread a chance to do something
+ time.sleep(0.1) # Give the GUI thread a chance to do something
try:
self.clear_dirtied([id_])
except:
From 87020e38be2441b7352a9c04dc8587547fdc6f53 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sun, 26 Sep 2010 15:49:15 +0100
Subject: [PATCH 145/207] Ensure that cached Metadata copies contain valid
cover info when get_metadata is called with get_cover = True
---
src/calibre/library/database2.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index e6587f06a2..0943c86e43 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -670,6 +670,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
mi = self.data.get(idx, self.FIELD_MAP['all_metadata'],
row_is_id = index_is_id)
if mi is not None:
+ if get_cover and mi.cover is None:
+ mi.cover = self.cover(idx, index_is_id=index_is_id, as_path=True)
return mi
self.gm_missed += 1
From 88cb23d952bdb2578418808feb6cc823da000e16 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sun, 26 Sep 2010 15:54:29 +0100
Subject: [PATCH 146/207] Take out print statement
---
src/calibre/library/caches.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index 74be3cd594..cdf0c1fce6 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -53,7 +53,6 @@ class MetadataBackup(Thread): # {{{
except:
# Happens during interpreter shutdown
break
- print 'doing id', id_
try:
path, mi = self.get_metadata_for_dump(id_)
except:
From 04ca51333b6e5ad821dc8d2e9c9ec8122ff10dcc Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Sun, 26 Sep 2010 09:10:05 -0600
Subject: [PATCH 147/207] Fix path too long message when restoring db
---
src/calibre/gui2/main.py | 3 +--
src/calibre/library/restore.py | 2 ++
2 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/src/calibre/gui2/main.py b/src/calibre/gui2/main.py
index 24ba7ef47c..d736835fd6 100644
--- a/src/calibre/gui2/main.py
+++ b/src/calibre/gui2/main.py
@@ -233,8 +233,7 @@ class GuiRunner(QObject):
def show_splash_screen(self):
self.splash_pixmap = QPixmap()
self.splash_pixmap.load(I('library.png'))
- self.splash_screen = QSplashScreen(self.splash_pixmap,
- Qt.SplashScreen)
+ self.splash_screen = QSplashScreen(self.splash_pixmap)
self.splash_screen.showMessage(_('Starting %s: Loading books...') %
__appname__)
self.splash_screen.show()
diff --git a/src/calibre/library/restore.py b/src/calibre/library/restore.py
index 48e66e508f..ec50a39aa0 100644
--- a/src/calibre/library/restore.py
+++ b/src/calibre/library/restore.py
@@ -22,6 +22,8 @@ NON_EBOOK_EXTENSIONS = frozenset([
class RestoreDatabase(LibraryDatabase2):
+ PATH_LIMIT = 10
+
def set_path(self, *args, **kwargs):
pass
From 1169eaef9959fab39d0631bbb77c6f86c3c23cd8 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sun, 26 Sep 2010 16:26:45 +0100
Subject: [PATCH 148/207] Fix problem trying to back up metadata for a deleted
book Add a 'backup all' button
---
src/calibre/gui2/preferences/misc.py | 6 ++++++
src/calibre/gui2/preferences/misc.ui | 11 +++++++++--
src/calibre/library/caches.py | 7 ++++++-
src/calibre/library/database2.py | 7 +++++--
4 files changed, 26 insertions(+), 5 deletions(-)
diff --git a/src/calibre/gui2/preferences/misc.py b/src/calibre/gui2/preferences/misc.py
index eae79fdfc0..e749a6fb98 100644
--- a/src/calibre/gui2/preferences/misc.py
+++ b/src/calibre/gui2/preferences/misc.py
@@ -88,10 +88,16 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
r('enforce_cpu_limit', config, restart_required=True)
self.device_detection_button.clicked.connect(self.debug_device_detection)
self.compact_button.clicked.connect(self.compact)
+ self.button_all_books_dirty.clicked.connect(self.mark_dirty)
self.button_open_config_dir.clicked.connect(self.open_config_dir)
self.button_osx_symlinks.clicked.connect(self.create_symlinks)
self.button_osx_symlinks.setVisible(isosx)
+ def mark_dirty(self):
+ db = self.gui.library_view.model().db
+ ids = [id for id in db.data.iterallids()]
+ db.dirtied(ids)
+
def debug_device_detection(self, *args):
from calibre.gui2.preferences.device_debug import DebugDevice
d = DebugDevice(self)
diff --git a/src/calibre/gui2/preferences/misc.ui b/src/calibre/gui2/preferences/misc.ui
index f8582a3675..573c61aba5 100644
--- a/src/calibre/gui2/preferences/misc.ui
+++ b/src/calibre/gui2/preferences/misc.ui
@@ -124,7 +124,14 @@
-
+
+
+
+ Force saving metadata of all books
+
+
+
+ Qt::Vertical
@@ -132,7 +139,7 @@
20
- 18
+ 1000
diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index cdf0c1fce6..9d6f87324f 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -44,6 +44,7 @@ class MetadataBackup(Thread): # {{{
self.keep_running = False
def run(self):
+ import traceback
while self.keep_running:
try:
time.sleep(0.5) # Limit to two per second
@@ -53,11 +54,11 @@ class MetadataBackup(Thread): # {{{
except:
# Happens during interpreter shutdown
break
+
try:
path, mi = self.get_metadata_for_dump(id_)
except:
prints('Failed to get backup metadata for id:', id_, 'once')
- import traceback
traceback.print_exc()
time.sleep(2)
try:
@@ -67,6 +68,10 @@ class MetadataBackup(Thread): # {{{
traceback.print_exc()
continue
+ if mi is None:
+ self.clear_dirtied([id_])
+ continue
+
# Give the GUI thread a chance to do something. Python threads don't
# have priorities, so this thread would naturally keep the processor
# until some scheduling event happens. The sleep makes such an event
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index 0943c86e43..6f628d8454 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -657,8 +657,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
self.dirtied(book_ids)
def get_metadata_for_dump(self, idx):
- path = os.path.join(self.abspath(idx, index_is_id=True), 'metadata.opf')
- mi = self.get_metadata(idx, index_is_id=True)
+ try:
+ path = os.path.join(self.abspath(idx, index_is_id=True), 'metadata.opf')
+ mi = self.get_metadata(idx, index_is_id=True)
+ except:
+ return ((None, None))
return ((path, mi))
def get_metadata(self, idx, index_is_id=False, get_cover=False):
From 482990dd031ae247e433f503cee50a044d7c6180 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Sun, 26 Sep 2010 09:40:34 -0600
Subject: [PATCH 149/207] Ensure application_id from OPF matched id in dir name
and actually fix path too long error
---
src/calibre/library/restore.py | 22 +++++++++++++---------
1 file changed, 13 insertions(+), 9 deletions(-)
diff --git a/src/calibre/library/restore.py b/src/calibre/library/restore.py
index ec50a39aa0..89c7fe8395 100644
--- a/src/calibre/library/restore.py
+++ b/src/calibre/library/restore.py
@@ -119,6 +119,7 @@ class Restore(Thread):
return True
def process_dir(self, dirpath, filenames, book_id):
+ book_id = int(book_id)
formats = filter(self.is_ebook_file, filenames)
fmts = [os.path.splitext(x)[1][1:].upper() for x in formats]
sizes = [os.path.getsize(os.path.join(dirpath, x)) for x in formats]
@@ -129,14 +130,17 @@ class Restore(Thread):
path = os.path.relpath(dirpath, self.src_library_path).replace(os.sep,
'/')
- self.books.append({
- 'mi': mi,
- 'timestamp': timestamp,
- 'formats': list(zip(fmts, sizes, names)),
- 'id': int(book_id),
- 'dirpath': dirpath,
- 'path': path,
- })
+ if int(mi.application_id) == book_id:
+ self.books.append({
+ 'mi': mi,
+ 'timestamp': timestamp,
+ 'formats': list(zip(fmts, sizes, names)),
+ 'id': book_id,
+ 'dirpath': dirpath,
+ 'path': path,
+ })
+ else:
+ self.ignored_dirs.append(dirpath)
def create_cc_metadata(self):
self.books.sort(key=itemgetter('timestamp'))
@@ -157,7 +161,7 @@ class Restore(Thread):
self.conflicting_custom_cols[label].add(args)
m[b['label']] = args
- db = LibraryDatabase2(self.library_path)
+ db = RestoreDatabase(self.library_path)
for args in m.values():
db.create_custom_column(*args)
db.conn.close()
From 2ea859906ded2ef2980502cccf07df1bb5d1f80c Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sun, 26 Sep 2010 16:40:46 +0100
Subject: [PATCH 150/207] Change text on backup button
---
src/calibre/gui2/preferences/misc.ui | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/calibre/gui2/preferences/misc.ui b/src/calibre/gui2/preferences/misc.ui
index 573c61aba5..492540901d 100644
--- a/src/calibre/gui2/preferences/misc.ui
+++ b/src/calibre/gui2/preferences/misc.ui
@@ -127,7 +127,7 @@
- Force saving metadata of all books
+ Back up metadata of all books (while you are working)
From 1cd78a56f29c110fb8601ab9ba63f3283f6828e7 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sun, 26 Sep 2010 17:31:39 +0100
Subject: [PATCH 151/207] Fix problem where :0>3s produced a string of all
zeros for null fields. Add a confirmation dialog to the backup pushbutton
---
src/calibre/gui2/preferences/misc.py | 3 +++
src/calibre/gui2/preferences/misc.ui | 2 +-
src/calibre/library/caches.py | 1 +
src/calibre/utils/formatter.py | 2 +-
4 files changed, 6 insertions(+), 2 deletions(-)
diff --git a/src/calibre/gui2/preferences/misc.py b/src/calibre/gui2/preferences/misc.py
index e749a6fb98..e72a1921ef 100644
--- a/src/calibre/gui2/preferences/misc.py
+++ b/src/calibre/gui2/preferences/misc.py
@@ -97,6 +97,9 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
db = self.gui.library_view.model().db
ids = [id for id in db.data.iterallids()]
db.dirtied(ids)
+ info_dialog(self, _('Backup metadata'),
+ _('Metadata will be backed up while calibre is running, at the '
+ 'rate of 30 books per minute.'), show=True)
def debug_device_detection(self, *args):
from calibre.gui2.preferences.device_debug import DebugDevice
diff --git a/src/calibre/gui2/preferences/misc.ui b/src/calibre/gui2/preferences/misc.ui
index 492540901d..adf2a15c16 100644
--- a/src/calibre/gui2/preferences/misc.ui
+++ b/src/calibre/gui2/preferences/misc.ui
@@ -127,7 +127,7 @@
- Back up metadata of all books (while you are working)
+ Back up metadata of all books
diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index 9d6f87324f..3e7f4d85ee 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -75,6 +75,7 @@ class MetadataBackup(Thread): # {{{
# Give the GUI thread a chance to do something. Python threads don't
# have priorities, so this thread would naturally keep the processor
# until some scheduling event happens. The sleep makes such an event
+ print 'do one'
time.sleep(0.1)
try:
raw = metadata_to_opf(mi)
diff --git a/src/calibre/utils/formatter.py b/src/calibre/utils/formatter.py
index 6fed4e157a..f95a6deee5 100644
--- a/src/calibre/utils/formatter.py
+++ b/src/calibre/utils/formatter.py
@@ -100,7 +100,7 @@ class TemplateFormatter(string.Formatter):
val = func[1](self, val)
else:
val = func[1](self, val, *args)
- else:
+ elif val:
val = string.Formatter.format_field(self, val, fmt)
if not val:
return ''
From a05448f7844d9f6f0730a4354b6c6e70dc8020ed Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sun, 26 Sep 2010 17:49:52 +0100
Subject: [PATCH 152/207] Fix stupid mistake in method naming in base.py
---
src/calibre/ebooks/metadata/book/base.py | 21 +++++++--------------
src/calibre/gui2/library/models.py | 2 +-
2 files changed, 8 insertions(+), 15 deletions(-)
diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index 0526de96a0..bdf11ad4ba 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -181,17 +181,10 @@ class Metadata(object):
'''
return metadata describing a standard or custom field.
'''
- if key in self.user_metadata_keys():
+ if key not in self.custom_field_keys():
return self.get_standard_metadata(self, key, make_copy=False)
return self.get_user_metadata(key, make_copy=False)
- def user_metadata_keys(self):
- '''
- Return the standard keys actually in this book.
- '''
- _data = object.__getattribute__(self, '_data')
- return frozenset(_data['user_metadata'].iterkeys())
-
def all_non_none_fields(self):
'''
Return a dictionary containing all non-None metadata fields, including
@@ -305,7 +298,7 @@ class Metadata(object):
def print_all_attributes(self):
for x in STANDARD_METADATA_FIELDS:
prints('%s:'%x, getattr(self, x, 'None'))
- for x in self.user_metadata_keys():
+ for x in self.custom_field_keys():
meta = self.get_user_metadata(x, make_copy=False)
if meta is not None:
prints(x, meta)
@@ -370,8 +363,8 @@ class Metadata(object):
if len(other_cover) > len(self_cover):
self.cover_data = other.cover_data
- if getattr(other, 'user_metadata_keys', None):
- for x in other.user_metadata_keys():
+ if getattr(other, 'custom_field_keys', None):
+ for x in other.custom_field_keys():
meta = other.get_user_metadata(x, make_copy=True)
if meta is not None:
self_tags = self.get(x, [])
@@ -434,7 +427,7 @@ class Metadata(object):
'''
returns the tuple (field_name, formatted_value)
'''
- if key in self.user_metadata_keys():
+ if key in self.custom_field_keys():
res = self.get(key, None)
cmeta = self.get_user_metadata(key, make_copy=False)
name = unicode(cmeta['name'])
@@ -516,7 +509,7 @@ class Metadata(object):
fmt('Published', isoformat(self.pubdate))
if self.rights is not None:
fmt('Rights', unicode(self.rights))
- for key in self.user_metadata_keys():
+ for key in self.custom_field_keys():
val = self.get(key, None)
if val:
(name, val) = self.format_field(key)
@@ -541,7 +534,7 @@ class Metadata(object):
ans += [(_('Published'), unicode(self.pubdate.isoformat(' ')))]
if self.rights is not None:
ans += [(_('Rights'), unicode(self.rights))]
- for key in self.user_metadata_keys():
+ for key in self.custom_field_keys():
val = self.get(key, None)
if val:
(name, val) = self.format_field(key)
diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index ff6d8b70f0..6725989ee5 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -331,7 +331,7 @@ class BooksModel(QAbstractTableModel): # {{{
_('Book %s of %s.')%\
(sidx, prepare_string_for_xml(series))
mi = self.db.get_metadata(idx)
- for key in mi.user_metadata_keys():
+ for key in mi.custom_field_keys():
name, val = mi.format_field(key)
if val:
data[name] = val
From 10f9af93a9e2ec28938a2d855fc01247ab129960 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sun, 26 Sep 2010 18:18:45 +0100
Subject: [PATCH 153/207] Fix restoring custom column definitions
---
src/calibre/library/caches.py | 1 -
src/calibre/library/restore.py | 38 ++++++++++++++++++++--------------
2 files changed, 23 insertions(+), 16 deletions(-)
diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index 3e7f4d85ee..9d6f87324f 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -75,7 +75,6 @@ class MetadataBackup(Thread): # {{{
# Give the GUI thread a chance to do something. Python threads don't
# have priorities, so this thread would naturally keep the processor
# until some scheduling event happens. The sleep makes such an event
- print 'do one'
time.sleep(0.1)
try:
raw = metadata_to_opf(mi)
diff --git a/src/calibre/library/restore.py b/src/calibre/library/restore.py
index 89c7fe8395..d34e831fc7 100644
--- a/src/calibre/library/restore.py
+++ b/src/calibre/library/restore.py
@@ -145,25 +145,33 @@ class Restore(Thread):
def create_cc_metadata(self):
self.books.sort(key=itemgetter('timestamp'))
m = {}
- fields = ('label', 'name', 'datatype', 'is_multiple', 'editable',
+ fields = ('label', 'name', 'datatype', 'is_multiple', 'is_editable',
'display')
for b in self.books:
- args = []
- for x in fields:
- if x in b:
- args.append(b[x])
- if len(args) == len(fields):
- # TODO: Do series type columns need special handling?
- label = b['label']
- if label in m and args != m[label]:
- if label not in self.conflicting_custom_cols:
- self.conflicting_custom_cols[label] = set([m[label]])
- self.conflicting_custom_cols[label].add(args)
- m[b['label']] = args
+ for key in b['mi'].custom_field_keys():
+ cfm = b['mi'].metadata_for_field(key)
+ args = []
+ for x in fields:
+ if x in cfm:
+ if x == 'is_multiple':
+ args.append(cfm[x] is not None)
+ else:
+ args.append(cfm[x])
+ if len(args) == len(fields):
+ # TODO: Do series type columns need special handling?
+ label = cfm['label']
+ if label in m and args != m[label]:
+ if label not in self.conflicting_custom_cols:
+ self.conflicting_custom_cols[label] = set([m[label]])
+ self.conflicting_custom_cols[label].add(args)
+ m[cfm['label']] = args
db = RestoreDatabase(self.library_path)
- for args in m.values():
- db.create_custom_column(*args)
+ self.progress_callback(None, len(m))
+ if len(m):
+ for i,args in enumerate(m.values()):
+ db.create_custom_column(*args)
+ self.progress_callback(_('creating custom column ')+args[0], i+1)
db.conn.close()
def restore_books(self):
From 4f522bde37cffedc184f4ff1f70b8b56aed1962d Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sun, 26 Sep 2010 18:25:45 +0100
Subject: [PATCH 154/207] Add dialog box back to the backup button
---
src/calibre/gui2/preferences/misc.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/src/calibre/gui2/preferences/misc.py b/src/calibre/gui2/preferences/misc.py
index 37887d56dc..99080c63bc 100644
--- a/src/calibre/gui2/preferences/misc.py
+++ b/src/calibre/gui2/preferences/misc.py
@@ -96,6 +96,9 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
def mark_dirty(self):
db = self.gui.library_view.model().db
db.dirtied(list(db.data.iterallids()))
+ info_dialog(self, _('Backup metadata'),
+ _('Metadata will be backed up while calibre is running, at the '
+ 'rate of 30 books per minute.'), show=True)
def debug_device_detection(self, *args):
from calibre.gui2.preferences.device_debug import DebugDevice
From 49561cd26c5f880a91e05025013dc8c8a7338d47 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sun, 26 Sep 2010 18:53:05 +0100
Subject: [PATCH 155/207] Change dirtied clear/set model in backup thread
---
src/calibre/gui2/ui.py | 3 +++
src/calibre/library/caches.py | 12 +++++------
src/calibre/library/database2.py | 36 +++++++++++++++++++-------------
3 files changed, 30 insertions(+), 21 deletions(-)
diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py
index 1e7c7550f8..2082a3b90a 100644
--- a/src/calibre/gui2/ui.py
+++ b/src/calibre/gui2/ui.py
@@ -565,6 +565,9 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, # {{{
mb = self.library_view.model().metadata_backup
if mb is not None:
mb.stop()
+ # give the thread time to stop so all operations complete
+ # otherwise the exit could kill the thread mid-write
+ time.sleep(2)
self.hide_windows()
self.emailer.stop()
diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index 7a15cb3ce1..09adc4a9fd 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -39,6 +39,7 @@ class MetadataBackup(Thread): # {{{
self.do_write = FunctionDispatcher(self.write)
self.get_metadata_for_dump = FunctionDispatcher(db.get_metadata_for_dump)
self.clear_dirtied = FunctionDispatcher(db.clear_dirtied)
+ self.set_dirtied = FunctionDispatcher(db.dirtied)
def stop(self):
self.keep_running = False
@@ -68,8 +69,9 @@ class MetadataBackup(Thread): # {{{
traceback.print_exc()
continue
+ # at this point the dirty indication is off
+
if mi is None:
- self.clear_dirtied([id_])
continue
# Give the GUI thread a chance to do something. Python threads don't
@@ -79,6 +81,7 @@ class MetadataBackup(Thread): # {{{
try:
raw = metadata_to_opf(mi)
except:
+ self.set_dirtied([id_])
prints('Failed to convert to opf for id:', id_)
traceback.print_exc()
continue
@@ -92,16 +95,11 @@ class MetadataBackup(Thread): # {{{
try:
self.do_write(path, raw)
except:
+ self.set_dirtied([id_])
prints('Failed to write backup metadata for id:', id_,
'again, giving up')
continue
- time.sleep(0.1) # Give the GUI thread a chance to do something
- try:
- self.clear_dirtied([id_])
- except:
- prints('Failed to clear dirtied for id:', id_)
-
def write(self, path, raw):
with open(path, 'wb') as f:
f.write(raw)
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index ce72b473e1..281f8cdc78 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -598,20 +598,17 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
for book_id in book_ids:
if not self.data.has_id(book_id):
continue
- path, mi = self.get_metadata_for_dump(book_id)
+ path, mi = self.get_metadata_for_dump(book_id,
+ remove_from_dirtied=remove_from_dirtied)
if path is None:
continue
- raw = metadata_to_opf(mi)
- with open(path, 'wb') as f:
- f.write(raw)
- if remove_from_dirtied:
- self.conn.execute('DELETE FROM metadata_dirtied WHERE book=?',
- (book_id,))
- # if a later exception prevents the commit, then the dirtied
- # table will still have the book. No big deal, because the OPF
- # is there and correct. We will simply do it again on next
- # start
- self.dirtied_cache.discard(book_id)
+ try:
+ raw = metadata_to_opf(mi)
+ with open(path, 'wb') as f:
+ f.write(raw)
+ except:
+ # Something went wrong. Put the book back on the dirty list
+ self.dirtied([book_id])
if commit:
self.conn.commit()
return True
@@ -649,7 +646,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
self.dirtied_cache = set()
self.dirtied(book_ids)
- def get_metadata_for_dump(self, idx):
+ def get_metadata_for_dump(self, idx, remove_from_dirtied=True):
try:
path = os.path.join(self.abspath(idx, index_is_id=True), 'metadata.opf')
mi = self.get_metadata(idx, index_is_id=True)
@@ -658,7 +655,18 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
# cover is set/removed
mi.cover = 'cover.jpg'
except:
- return (None, None)
+ # This almost certainly means that the book has been deleted while
+ # the backup operation sat in the queue.
+ path,mi = (None, None)
+
+ try:
+ # clear the dirtied indicator. The user must put it back if
+ # something goes wrong with writing the OPF
+ if remove_from_dirtied:
+ self.clear_dirtied([idx])
+ except:
+ # No real problem. We will just do it again.
+ pass
return (path, mi)
def get_metadata(self, idx, index_is_id=False, get_cover=False):
From bfdcb4250d600e47a14e52001a4b0bb344a88547 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sun, 26 Sep 2010 19:07:08 +0100
Subject: [PATCH 156/207] Have get_metadata put id into Metadata. It should be
there, because it is in field_metadata as a field type. It is not in
STANDARD_FIELDS, so it won't be serialized.
---
src/calibre/library/database2.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index 281f8cdc78..a9d372bed1 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -719,6 +719,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
mi.isbn = self.isbn(idx, index_is_id=index_is_id)
id = idx if index_is_id else self.id(idx)
mi.application_id = id
+ mi.id = id
for key,meta in self.field_metadata.iteritems():
if meta['is_custom']:
mi.set_user_metadata(key, meta)
From 0f341f5cd57ed6c9346ec11f812057f7f399251e Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Sun, 26 Sep 2010 12:28:35 -0600
Subject: [PATCH 157/207] Fifth beta
---
src/calibre/constants.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/calibre/constants.py b/src/calibre/constants.py
index be387d8ca2..6cab1d32e7 100644
--- a/src/calibre/constants.py
+++ b/src/calibre/constants.py
@@ -2,7 +2,7 @@ __license__ = 'GPL v3'
__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
__docformat__ = 'restructuredtext en'
__appname__ = 'calibre'
-__version__ = '0.7.903'
+__version__ = '0.7.904'
__author__ = "Kovid Goyal "
import re
From 8c2b672e6c086db3003f3f1e7ce2283d7c57da1d Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Sun, 26 Sep 2010 12:30:19 -0600
Subject: [PATCH 158/207] ...
---
resources/recipes/boortz.recipe | 15 +++++++--------
resources/recipes/popscience.recipe | 26 +++++++++++++-------------
2 files changed, 20 insertions(+), 21 deletions(-)
diff --git a/resources/recipes/boortz.recipe b/resources/recipes/boortz.recipe
index 0b52e0b9ca..8fb8041411 100644
--- a/resources/recipes/boortz.recipe
+++ b/resources/recipes/boortz.recipe
@@ -1,5 +1,4 @@
from calibre.web.feeds.news import BasicNewsRecipe
-from calibre.ebooks.BeautifulSoup import BeautifulSoup, re
class AdvancedUserRecipe1282101454(BasicNewsRecipe):
title = 'Nealz Nuze'
language = 'en'
@@ -12,11 +11,11 @@ class AdvancedUserRecipe1282101454(BasicNewsRecipe):
linearize_tables = True
no_stylesheets = True
remove_javascript = True
-
+
masthead_url = 'http://boortz.com/images/nuze_logo.gif'
keep_only_tags = [
dict(name='td', attrs={'id':['contentWellCell']})
-
+
]
remove_tags = [
dict(name='a', attrs={'class':['blogPermalink']}),
@@ -26,13 +25,13 @@ class AdvancedUserRecipe1282101454(BasicNewsRecipe):
remove_tags_after = [dict(name='div', attrs={'class':'blogEntryBody'}),]
feeds = [
('NUZE', 'http://boortz.com/nealz_nuze_rss/rss.xml')
-
+
]
-
-
-
-
+
+
+
+
diff --git a/resources/recipes/popscience.recipe b/resources/recipes/popscience.recipe
index 2bef7e4807..1527a1bb71 100644
--- a/resources/recipes/popscience.recipe
+++ b/resources/recipes/popscience.recipe
@@ -1,5 +1,5 @@
from calibre.web.feeds.news import BasicNewsRecipe
-from calibre.ebooks.BeautifulSoup import BeautifulSoup, re
+from calibre.ebooks.BeautifulSoup import re
class AdvancedUserRecipe1282101454(BasicNewsRecipe):
title = 'Popular Science'
@@ -13,35 +13,35 @@ class AdvancedUserRecipe1282101454(BasicNewsRecipe):
no_stylesheets = True
remove_javascript = True
use_embedded_content = True
-
+
masthead_url = 'http://www.raytheon.com/newsroom/rtnwcm/groups/Public/documents/masthead/rtn08_popscidec_masthead.jpg'
-
-
+
+
feeds = [
-
+
('Gadgets', 'http://www.popsci.com/full-feed/gadgets'),
('Cars', 'http://www.popsci.com/full-feed/cars'),
('Science', 'http://www.popsci.com/full-feed/science'),
('Technology', 'http://www.popsci.com/full-feed/technology'),
('DIY', 'http://www.popsci.com/full-feed/diy'),
-
+
]
-
- #The following will get read of the Gallery: links when found
-
+
+ #The following will get read of the Gallery: links when found
+
def preprocess_html(self, soup) :
print 'SOUP IS: ', soup
weblinks = soup.findAll(['head','h2'])
if weblinks is not None:
for link in weblinks:
if re.search('(Gallery)(:)',str(link)):
-
+
link.parent.extract()
return soup
- #-----------------------------------------------------------------
-
-
+ #-----------------------------------------------------------------
+
+
From f74530210fdde5a632c88d28f15b7beed9c8ef34 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sun, 26 Sep 2010 21:01:55 +0100
Subject: [PATCH 159/207] Make composite columns sort case-insensitive.
---
src/calibre/library/caches.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index 09adc4a9fd..6cd0c227dd 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -780,7 +780,7 @@ class SortKeyGenerator(object):
sidx = record[sidx_fm['rec_index']]
val = (val, sidx)
- elif dt in ('text', 'comments'):
+ elif dt in ('text', 'comments', 'composite'):
if val is None:
val = ''
val = val.lower()
From c63b55115079ce06d516f2f21c8e75f580ecc118 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Sun, 26 Sep 2010 15:25:33 -0600
Subject: [PATCH 160/207] Shutdown the metadata backup thread before running a
db integrity check
---
src/calibre/gui2/library/models.py | 2 +-
src/calibre/gui2/preferences/misc.py | 8 +++++++-
2 files changed, 8 insertions(+), 2 deletions(-)
diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index 6725989ee5..b2a7f08055 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -72,7 +72,7 @@ class BooksModel(QAbstractTableModel): # {{{
'publisher' : _("Publisher"),
'tags' : _("Tags"),
'series' : _("Series"),
- }
+ }
def __init__(self, parent=None, buffer=40):
QAbstractTableModel.__init__(self, parent)
diff --git a/src/calibre/gui2/preferences/misc.py b/src/calibre/gui2/preferences/misc.py
index 99080c63bc..865115c2ed 100644
--- a/src/calibre/gui2/preferences/misc.py
+++ b/src/calibre/gui2/preferences/misc.py
@@ -106,8 +106,14 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
d.exec_()
def compact(self, *args):
- d = CheckIntegrity(self.gui.library_view.model().db, self)
+ from calibre.library.caches import MetadataBackup
+ m = self.gui.library_view.model()
+ if m.metadata_backup is not None:
+ m.metadata_backup.stop()
+ d = CheckIntegrity(m.db, self)
d.exec_()
+ m.metadata_backup = MetadataBackup(m.db)
+ m.metadata_backup.start()
def open_config_dir(self, *args):
from calibre.utils.config import config_dir
From 1dee223f14d413f5969e1ff7b261882b5779be0d Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Mon, 27 Sep 2010 15:12:25 +0100
Subject: [PATCH 161/207] First implementation of plugboards
---
src/calibre/customize/builtins.py | 15 +-
src/calibre/ebooks/metadata/book/base.py | 29 ++-
src/calibre/gui2/device.py | 32 ++-
src/calibre/gui2/preferences/plugboard.py | 257 ++++++++++++++++++++++
src/calibre/gui2/preferences/plugboard.ui | 138 ++++++++++++
src/calibre/library/save_to_disk.py | 22 +-
6 files changed, 484 insertions(+), 9 deletions(-)
create mode 100644 src/calibre/gui2/preferences/plugboard.py
create mode 100644 src/calibre/gui2/preferences/plugboard.ui
diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py
index 4e47c70bb0..89c800afb2 100644
--- a/src/calibre/customize/builtins.py
+++ b/src/calibre/customize/builtins.py
@@ -796,6 +796,17 @@ class Sending(PreferencesPlugin):
description = _('Control how calibre transfers files to your '
'ebook reader')
+class Plugboard(PreferencesPlugin):
+ name = 'Plugboard'
+ icon = I('plugboard.png')
+ gui_name = _('Metadata plugboard')
+ category = 'Import/Export'
+ gui_category = _('Import/Export')
+ category_order = 3
+ name_order = 4
+ config_widget = 'calibre.gui2.preferences.plugboard'
+ description = _('Change metadata fields before saving/sending')
+
class Email(PreferencesPlugin):
name = 'Email'
icon = I('mail.png')
@@ -856,8 +867,8 @@ class Misc(PreferencesPlugin):
description = _('Miscellaneous advanced configuration')
plugins += [LookAndFeel, Behavior, Columns, Toolbar, InputOptions,
- CommonOptions, OutputOptions, Adding, Saving, Sending, Email, Server,
- Plugins, Tweaks, Misc]
+ CommonOptions, OutputOptions, Adding, Saving, Sending, Plugboard,
+ Email, Server, Plugins, Tweaks, Misc]
#}}}
diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index bf95e989e8..aaa7c78e9a 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -182,7 +182,7 @@ class Metadata(object):
return metadata describing a standard or custom field.
'''
if key not in self.custom_field_keys():
- return self.get_standard_metadata(self, key, make_copy=False)
+ return self.get_standard_metadata(key, make_copy=False)
return self.get_user_metadata(key, make_copy=False)
def all_non_none_fields(self):
@@ -294,6 +294,33 @@ class Metadata(object):
_data = object.__getattribute__(self, '_data')
_data['user_metadata'][field] = metadata
+ def copy_specific_attributes(self, other, attrs):
+ '''
+ Takes a dict {src:dest, src:dest} and copys other[src] to self[dest].
+ This is on a best-efforts basis. Some assignments can make no sense.
+ '''
+ if not attrs:
+ return
+ for src in attrs:
+ try:
+ print src
+ sfm = other.metadata_for_field(src)
+ dfm = self.metadata_for_field(attrs[src])
+ if dfm['is_multiple']:
+ if sfm['is_multiple']:
+ self.set(attrs[src], other.get(src))
+ else:
+ self.set(attrs[src],
+ [f.strip() for f in other.get(src).split(',')
+ if f.strip()])
+ elif sfm['is_multiple']:
+ self.set(attrs[src], ','.join(other.get(src)))
+ else:
+ self.set(attrs[src], other.get(src))
+ except:
+ traceback.print_exc()
+ pass
+
# Old Metadata API {{{
def print_all_attributes(self):
for x in STANDARD_METADATA_FIELDS:
diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py
index 58c5e5d9ad..eb1716f782 100644
--- a/src/calibre/gui2/device.py
+++ b/src/calibre/gui2/device.py
@@ -317,19 +317,40 @@ class DeviceManager(Thread): # {{{
args=[booklist, on_card],
description=_('Send collections to device'))
- def _upload_books(self, files, names, on_card=None, metadata=None):
+ def _upload_books(self, files, names, on_card=None, metadata=None, plugboards=None):
'''Upload books to device: '''
if metadata and files and len(metadata) == len(files):
for f, mi in zip(files, metadata):
if isinstance(f, unicode):
ext = f.rpartition('.')[-1].lower()
+ dev_name = self.connected_device.name
+ cpb = None
+ if ext in plugboards:
+ cpb = plugboards[ext]
+ elif ' any' in plugboards:
+ cpb = plugboards[' any']
+ if cpb is not None:
+ if dev_name in cpb:
+ cpb = cpb[dev_name]
+ elif ' any' in plugboards[ext]:
+ cpb = cpb[' any']
+ else:
+ cpb = None
+
+ if DEBUG:
+ prints('Using plugboard', cpb)
if ext:
try:
if DEBUG:
prints('Setting metadata in:', mi.title, 'at:',
f, file=sys.__stdout__)
with open(f, 'r+b') as stream:
- set_metadata(stream, mi, stream_type=ext)
+ if cpb:
+ newmi = mi.deepcopy()
+ newmi.copy_specific_attributes(mi, cpb)
+ else:
+ newmi = mi
+ set_metadata(stream, newmi, stream_type=ext)
except:
if DEBUG:
prints(traceback.format_exc(), file=sys.__stdout__)
@@ -338,12 +359,12 @@ class DeviceManager(Thread): # {{{
metadata=metadata, end_session=False)
def upload_books(self, done, files, names, on_card=None, titles=None,
- metadata=None):
+ metadata=None, plugboards=None):
desc = _('Upload %d books to device')%len(names)
if titles:
desc += u':' + u', '.join(titles)
return self.create_job(self._upload_books, done, args=[files, names],
- kwargs={'on_card':on_card,'metadata':metadata}, description=desc)
+ kwargs={'on_card':on_card,'metadata':metadata,'plugboards':plugboards}, description=desc)
def add_books_to_metadata(self, locations, metadata, booklists):
self.device.add_books_to_metadata(locations, metadata, booklists)
@@ -1257,10 +1278,11 @@ class DeviceMixin(object): # {{{
:param files: List of either paths to files or file like objects
'''
titles = [i.title for i in metadata]
+ plugboards = self.library_view.model().db.prefs.get('plugboards', None)
job = self.device_manager.upload_books(
Dispatcher(self.books_uploaded),
files, names, on_card=on_card,
- metadata=metadata, titles=titles
+ metadata=metadata, titles=titles, plugboards=plugboards
)
self.upload_memory[job] = (metadata, on_card, memory, files)
diff --git a/src/calibre/gui2/preferences/plugboard.py b/src/calibre/gui2/preferences/plugboard.py
new file mode 100644
index 0000000000..5691120cef
--- /dev/null
+++ b/src/calibre/gui2/preferences/plugboard.py
@@ -0,0 +1,257 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+__license__ = 'GPL v3'
+__copyright__ = '2010, Kovid Goyal '
+__docformat__ = 'restructuredtext en'
+
+from PyQt4 import QtGui
+
+from calibre.gui2 import error_dialog
+from calibre.gui2.preferences import ConfigWidgetBase, test_widget, \
+ AbortCommit
+from calibre.gui2.preferences.plugboard_ui import Ui_Form
+from calibre.customize.ui import metadata_writers, device_plugins
+
+
+class ConfigWidget(ConfigWidgetBase, Ui_Form):
+
+ def genesis(self, gui):
+ self.gui = gui
+ self.db = gui.library_view.model().db
+ self.current_plugboards = self.db.prefs.get('plugboards', {'epub': {' any': {'title':'authors', 'authors':'tags'}}})
+ self.current_device = None
+ self.current_format = None
+# self.proxy = ConfigProxy(config())
+#
+# r = self.register
+#
+# for x in ('asciiize', 'update_metadata', 'save_cover', 'write_opf',
+# 'replace_whitespace', 'to_lowercase', 'formats', 'timefmt'):
+# r(x, self.proxy)
+#
+# self.save_template.changed_signal.connect(self.changed_signal.emit)
+
+ def clear_fields(self, edit_boxes=False, new_boxes=False):
+ self.ok_button.setEnabled(False)
+ for w in self.source_widgets:
+ w.clear()
+ for w in self.dest_widgets:
+ w.clear()
+ if edit_boxes:
+ self.edit_device.setCurrentIndex(0)
+ self.edit_format.setCurrentIndex(0)
+ if new_boxes:
+ self.new_device.setCurrentIndex(0)
+ self.new_format.setCurrentIndex(0)
+
+ def set_fields(self):
+ self.ok_button.setEnabled(True)
+ for w in self.source_widgets:
+ w.addItems(self.fields)
+ for w in self.dest_widgets:
+ w.addItems(self.fields)
+
+ def set_field(self, i, src, dst):
+ print i, src, dst
+ idx = self.fields.index(src)
+ self.source_widgets[i].setCurrentIndex(idx)
+ idx = self.fields.index(dst)
+ self.dest_widgets[i].setCurrentIndex(idx)
+
+ def edit_device_changed(self, txt):
+ if txt == '':
+ self.current_device = None
+ return
+ print 'edit device changed'
+ self.clear_fields(new_boxes=True)
+ self.current_device = unicode(txt)
+ fpb = self.current_plugboards.get(self.current_format, None)
+ if fpb is None:
+ print 'None format!'
+ return
+ dpb = fpb.get(self.current_device, None)
+ if dpb is None:
+ print 'none device!'
+ return
+ self.set_fields()
+ for i,src in enumerate(dpb):
+ self.set_field(i, src, dpb[src])
+ self.ok_button.setEnabled(True)
+
+ def edit_format_changed(self, txt):
+ if txt == '':
+ self.edit_device.setCurrentIndex(0)
+ self.current_format = None
+ self.current_device = None
+ return
+ print 'edit_format_changed'
+ self.clear_fields(new_boxes=True)
+ txt = unicode(txt)
+ fpb = self.current_plugboards.get(txt, None)
+ if fpb is None:
+ print 'None editable format!'
+ return
+ self.current_format = txt
+ devices = ['']
+ for d in fpb:
+ devices.append(d)
+ self.edit_device.clear()
+ self.edit_device.addItems(devices)
+ self.edit_device.setCurrentIndex(0)
+
+ def new_device_changed(self, txt):
+ if txt == '':
+ self.current_device = None
+ return
+ print 'new_device_changed'
+ self.clear_fields(edit_boxes=True)
+ self.current_device = unicode(txt)
+ error = False
+ if self.current_format == ' any':
+ for f in self.current_plugboards:
+ if self.current_device == ' any' and len(self.current_plugboards[f]):
+ error = True
+ break
+ if self.current_device in self.current_plugboards[f]:
+ error = True
+ break
+ if ' any' in self.current_plugboards[f]:
+ error = True
+ break
+ else:
+ fpb = self.current_plugboards.get(self.current_format, None)
+ if fpb is not None:
+ if ' any' in fpb:
+ error = True
+ else:
+ dpb = fpb.get(self.current_device, None)
+ if dpb is not None:
+ error = True
+
+ if error:
+ error_dialog(self, '',
+ _('That format and device already has a plugboard'),
+ show=True)
+ self.new_device.setCurrentIndex(0)
+ return
+ self.set_fields()
+
+ def new_format_changed(self, txt):
+ if txt == '':
+ self.current_format = None
+ self.current_device = None
+ return
+ print 'new_format_changed'
+ self.clear_fields(edit_boxes=True)
+ self.current_format = unicode(txt)
+ self.new_device.setCurrentIndex(0)
+
+ def ok_clicked(self):
+ pb = {}
+ print self.current_format, self.current_device
+ for i in range(0, len(self.source_widgets)):
+ s = self.source_widgets[i].currentIndex()
+ if s != 0:
+ d = self.dest_widgets[i].currentIndex()
+ if d != 0:
+ pb[self.fields[s]] = self.fields[d]
+ if len(pb) == 0:
+ if self.current_format in self.current_plugboards:
+ fpb = self.current_plugboards[self.current_format]
+ if self.current_device in fpb:
+ del fpb[self.current_device]
+ if len(fpb) == 0:
+ del self.current_plugboards[self.current_format]
+ else:
+ if self.current_format not in self.current_plugboards:
+ self.current_plugboards[self.current_format] = {}
+ fpb = self.current_plugboards[self.current_format]
+ fpb[self.current_device] = pb
+ self.changed_signal.emit()
+ self.refill_all_boxes()
+
+ def refill_all_boxes(self):
+ self.current_device = None
+ self.current_format = None
+ self.clear_fields(new_boxes=True)
+ self.edit_format.clear()
+ self.edit_format.addItem('')
+ for format in self.current_plugboards:
+ self.edit_format.addItem(format)
+ self.edit_format.setCurrentIndex(0)
+ self.edit_device.clear()
+ self.ok_button.setEnabled(False)
+
+ def initialize(self):
+ def field_cmp(x, y):
+ if x.startswith('#'):
+ if y.startswith('#'):
+ return cmp(x.lower(), y.lower())
+ else:
+ return 1
+ elif y.startswith('#'):
+ return -1
+ else:
+ return cmp(x.lower(), y.lower())
+
+ ConfigWidgetBase.initialize(self)
+
+ self.devices = ['', ' any', 'save to disk']
+ for device in device_plugins():
+ self.devices.append(device.name)
+ self.devices.sort(cmp=lambda x, y: cmp(x.lower(), y.lower()))
+ self.new_device.addItems(self.devices)
+
+ self.formats = ['', ' any']
+ for w in metadata_writers():
+ for f in w.file_types:
+ self.formats.append(f)
+ self.formats.sort()
+ self.new_format.addItems(self.formats)
+
+ self.fields = ['']
+ for f in self.db.all_field_keys():
+ if self.db.field_metadata[f].get('rec_index', None) is not None and\
+ self.db.field_metadata[f]['datatype'] is not None and \
+ self.db.field_metadata[f]['search_terms']:
+ self.fields.append(f)
+ self.fields.sort(cmp=field_cmp)
+
+ self.source_widgets = []
+ self.dest_widgets = []
+ for i in range(0, 10):
+ w = QtGui.QComboBox(self)
+ self.source_widgets.append(w)
+ self.fields_layout.addWidget(w, 5+i, 0, 1, 1)
+ w = QtGui.QComboBox(self)
+ self.dest_widgets.append(w)
+ self.fields_layout.addWidget(w, 5+i, 1, 1, 1)
+
+ self.edit_device.currentIndexChanged[str].connect(self.edit_device_changed)
+ self.edit_format.currentIndexChanged[str].connect(self.edit_format_changed)
+ self.new_device.currentIndexChanged[str].connect(self.new_device_changed)
+ self.new_format.currentIndexChanged[str].connect(self.new_format_changed)
+ self.ok_button.clicked.connect(self.ok_clicked)
+
+ self.refill_all_boxes()
+
+ def restore_defaults(self):
+ ConfigWidgetBase.restore_defaults(self)
+ self.current_plugboards = {}
+ self.refill_all_boxes()
+ self.changed_signal.emit()
+
+ def commit(self):
+ self.db.prefs.set('plugboards', self.current_plugboards)
+ return ConfigWidgetBase.commit(self)
+
+ def refresh_gui(self, gui):
+ pass
+
+
+if __name__ == '__main__':
+ from PyQt4.Qt import QApplication
+ app = QApplication([])
+ test_widget('Import/Export', 'plugboards')
+
diff --git a/src/calibre/gui2/preferences/plugboard.ui b/src/calibre/gui2/preferences/plugboard.ui
new file mode 100644
index 0000000000..ad72ec359f
--- /dev/null
+++ b/src/calibre/gui2/preferences/plugboard.ui
@@ -0,0 +1,138 @@
+
+
+ Form
+
+
+
+ 0
+ 0
+ 707
+ 340
+
+
+
+ Form
+
+
+
+
+
+ Here you can control what metadata calibre uses when saving or sending books:
+
+
+ true
+
+
+
+
+
+
+
+
+ Add new plugboard
+
+
+
+
+
+
+ Edit existing plugboard
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Format (choose first)
+
+
+ Qt::AlignCenter
+
+
+
+
+
+
+ Device (choose second)
+
+
+ Qt::AlignCenter
+
+
+
+
+
+
+ Qt::Vertical
+
+
+
+ 20
+ 40
+
+
+
+
+
+
+
+
+
+
+
+ Source field
+
+
+ Qt::AlignCenter
+
+
+
+
+
+
+ Destination field
+
+
+ Qt::AlignCenter
+
+
+
+
+
+
+ Qt::Vertical
+
+
+
+ 20
+ 40
+
+
+
+
+
+
+
+ Done
+
+
+
+
+
+
+
+
+
+
diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py
index e479d27121..54671da4b4 100644
--- a/src/calibre/library/save_to_disk.py
+++ b/src/calibre/library/save_to_disk.py
@@ -232,6 +232,21 @@ def save_book_to_disk(id, db, root, opts, length):
written = False
for fmt in formats:
+ dev_name = 'save to disk'
+ plugboards = db.prefs.get('plugboards', None)
+ cpb = None
+ if fmt in plugboards:
+ cpb = plugboards[fmt]
+ elif ' any' in plugboards:
+ cpb = plugboards[' any']
+ if cpb is not None:
+ if dev_name in cpb:
+ cpb = cpb[dev_name]
+ elif ' any' in plugboards[fmt]:
+ cpb = cpb[' any']
+ else:
+ cpb = None
+
data = db.format(id, fmt, index_is_id=True)
if data is None:
continue
@@ -242,7 +257,12 @@ def save_book_to_disk(id, db, root, opts, length):
stream.write(data)
stream.seek(0)
try:
- set_metadata(stream, mi, fmt)
+ if cpb:
+ newmi = mi.deepcopy()
+ newmi.copy_specific_attributes(mi, cpb)
+ else:
+ newmi = mi
+ set_metadata(stream, newmi, fmt)
except:
traceback.print_exc()
stream.seek(0)
From d6fc61d1b706dffe4063aa63f78245ee48401a4a Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Mon, 27 Sep 2010 15:14:55 +0100
Subject: [PATCH 162/207] Add the icon
---
resources/images/plugboard.png | Bin 0 -> 31806 bytes
1 file changed, 0 insertions(+), 0 deletions(-)
create mode 100644 resources/images/plugboard.png
diff --git a/resources/images/plugboard.png b/resources/images/plugboard.png
new file mode 100644
index 0000000000000000000000000000000000000000..88f0869b8d9de4552184308b934889c5c83ec85f
GIT binary patch
literal 31806
zcmXtIIzd;WG02p<&QJ8;w)c+PO=-*Y}
z=2!8*jm}rwDgXeu#`3=f1QZr?0sv5e4oc1JQL$K`&UEl(5Km`OgYpXL^~L;3+G|F1
zQss>XKURtiMw2MdN$q0l*1Ho>q&+k$gEze_-Q-fuo>PZf%AWI}
zWLRiu=;3_-KB>5-AvkMjGFC>hEmm~)v&Vji`5JG;(F?o-t%(N{Cyd5dFofd%`?t~2
z6U$_}Xdu4ct?*vE4w7dA&=h6%Q2Jw2KrjWqBC0{Dpf@cJDFtd1WtNVaw^h8w6
zZZ1eiee>>SK4y{?+5GLd?~{PwH-G+%J_A8Y(t>!1`iub?su;rC0BMB(G7w5EP`r_uQ6B8f-~|rt)r3#j{k;g
zmvEK;SW`44i8Z7NP>&c$1b>=7atQP@HLCY>(y)1z?I>8}0Zq-QpWX3Y=J|Cl$l;Re
zrNNmSdgIx{f;Q}J^cmTj=-x6ZYp`xf)>u`;JK7}^)>VwIfeI-QiRn#eehS~}vqi^Q
z`*QiaCy4ti5vnm#4~L~hAH7uOIqhKeMJ03J54m@1Gn4@MzWK#0B|7oA8y*$DBzg4u
zp!bc>j2wa{`GDaIz7*7FUQfjh)6$&kS-!hm+ajYuDUCG&j)*Kj0&G&l^o?@qa4#xfjt11gJJiX%~C98bv-S9B*
zDGWW)OQ3@0S0+c_jL8YTcW~+qm&;o}Fb)7x%GuT^InGp0c6Tc)vcFt!uk!8=f7IPG
zz(FG=i-ymQN$_%5#g$ZjdPR%bJ=>R^G#lZk-_8!mS{z*4Jd2n@6Nq8Uyq$ZD`2E9Q
zllfaO^&j549>8+`qcyt;w*e-*7J9%fZ`VgN
zXCZO*a+5ds*3mE@-dJ4z>7lLNW$nPfd(HSt+?C#Z{r&ob$6F;=zq=xfW<9O6vHqpE
zf+K$uSFctEe%|qlCHPCY6E5Bvb0QPPjf+WCsD0I<*4+)5)dTwSc5
z`>g8K2FiF#Gv4@l@q6M9@6|`)2fWu87z=s+5WQMS&tmsea|3^yvwU#-zZW?(sT*zz+aUxoep
zGmUDD4hmoG8a5fByZZg?%BN8ij$vUOigrJVygGcdpGh%R!7KXHPqksZ?C+B5#1S+8
zLw1Ewupa=X{xsuZ^2Lllbn^><+|Z|BQ05poFWTVU@HqicSy{<+kX8eMITi)cXH@0(
z5Oz<>E>j|Nby^=9L3_-zcrSlz?Gbgcz}CHlABw!S-yZxei#$F%V;Aw)j>}MJ%sQHE
z{M3OBEXqIF&g$jC36Td@QdZJ@meVL15OML#8e7zSIFS06;Y&pW($0c&
z*CL=*N$e;rGQi%9z+U%D
zPj=Vo`KJ~FXNjr4Hfs@Boqi+;9ujg&!k90y7!n^IsodIH8Gd=w+(4m@PaVq86>_I)
zw!N|Det_#+lFK1uWAp8f{r@(0?wp@fz!F(4h@$E?AvZt&_li|(xyk6P#$_z1>lf)!
ze(i`s7KQ^9-F(hQD;^*fe=_05av!nWVpRFiiXQ5KhtAeSC(PKpU<-;(T`^mDqO$^z
zu6p{fT>@5ZuFknw
zMTV##^vE?kEI7!XgxR)eEOn&wbFQp6;FFcA9+ZoB;50)?jnwR^GpzmG)o~I~
z7oeYQK4%GpfdKYnEj#loH=;+1VKH<-27{%Vwc}&4U`DO%S5fbSKY1pygv_{e@>A_%
z2IGVqgrY|)*D8fyP
z@MZ4kadM+1%!g+euCOX8X7V*m)a*%g-ESTxAl{|a8hQJ)7@Rq)jI`)@z5XDI(X-08
zLZMUD5pwmqQi#eTzL+}*^v0L8f2|u(o|+9xjIgD2*K|S1tD8FdApuaN5T#K)V?8#2
z4N04t3eh0JGR+u;8`B^LY@Fs^q!EGY-o64#CfoVye4CtyfgxS$)BsB|98Ej^-9Y6<
zs2@u1p%e1$ARmH)IX`(MeS}-2>Qfy&z&92Qf&d*!o#;`*QVZ5ZMy~!~u9Go}-_d@~
z=B$%SMnMNI3i8XZB}9*!q2`8@Z$)m^0hl&xfE82Ei*iMmCXXEYE`+bnRqx)~O0Rql
zw4O@olR1*+2vSf`pg;e(Nlq
zpjbI4nx0lX%~-@6de3w;o)6!v@$I#u@-=a_XuR+%-Q3yTRrwoxwRtF4PlwvXpoe&X
z;9}ZzI{Z&eZwm#Kk^;;@6)$-09TG@KcE{E;88?m>zPAk>tKL^9SLe0o=kw-6DoXYY2B}i-EIkBaQWQe;YJ`kWK=3~W
z7*y!8X{<}70ZKNwlN(MP6g10`vp{R@Q91y6+{Z<%)aX}zE(
zvPV69bmx(!Pr^Ma>ax5ZvCvy%(p!?@|5fASk6taW9@e-8-+%6|TU2AvyHOZI@Cy;+
z6VftJE}UKWvmkGU)>qqpVs#F+yOXPo@^+H-soOfUAf6T%8I#J
z_@Qd9Y4XQ`;xm`L>0D=7Dw2G$XH;x(e|s|*w47g2IgXQPQW7mv&Aq;1jS*m)`aZ&h
zY~-5@owY7V;%_tyL0JRz(WdibaUFHL>UGgU+=sNv^>S63x}C2SJ4GK(smD;eaX*vO
zBUUGK*WPy&?MPeqnizeo%a^*oA{#1EJtytl=q{j!ZVIjuoPK_%inZGCdLNhmu5=L`
z^NH7+-xOB2aX{_ln28xiF}c!$@!Y^-@(SDFLwmSf9Fnk^uD9RI`p~yMTRoZLc?ZNh
z*r(=63cP+|(@H2^?uHA|JC5#g_
z2k4t&*|_ofBKaROgRF9ATXp2?DZfV@ct+_>QQBhyCZcHVncf!zxay_sk*ts`T|%f8
zEo2B3>#C+%Pi+V=c{)3>0_15gWEa2$(SN6{q-U~ksct#{j9$^VWfk`j$MOpYB1|%h
z^BZzmpVw_Vs~uzf@haj$PyE$B@wn8EFU#v%+VTq0Z*DWCAqnLPH4<`Q2s8i9SJpaG
zHA0H63{wzKeEAw;>uUYk)ye6b^X4v}79f1fxo<_6tevD`Myf2Vh6N-u=F=P(PQIF*
z3B=~aP`LgsyZWFS5@znbNz*e5yN}Zb!nohn^5K=R^Wsb
zGmREo0FFY&Bt*_Qh(fT+9#9Np1}U->y4j){dd@B`jwP?YmF0^yl-xNP{&up@6ve|j
zL8#j6#dvXVak}SaYOA%*$|kzW2Lv51CJbDXUyS0`Kf&9jnoZk=6i0C9$A9v+CKAb1
z0W`~g)?)`%OE%7~(Bt}*cQ~FRxlwF%i@y<3=T7nI-lTUp{Ihuy85W=Wv3!tJSz%>s
z(;u0SbTjt6JyMhiDRKzEbTaomWAl|sS()y6q}U(kPq3D<^gL|~JNrr8{QGKeo~go2
zraJGRc)rE7wExXrk!LKG6&_p`@#EJo#NE)#77gQXC%?X)oD4sUJbO3n`{TI8-Hh-Q
z&)?_4;Mo)Gr7NXe8C^gD$nPuU`>=l7y#XiGp*!*MbSczCCY@ei+@BEh;zfwh!8^&w
z-zj*5X4K2RySG-CmtEprBtK7+AAZ+xz4eQXY-stKkPWcc1t?dAtKw|DIdv0xv*g|R
z(u;@1U(ey&40nj$YD?#msf^jf#BQ_5(D2i%@bhyHBRht@HrB*h2JD~9eAUPR65~yE
zYWqQiWN>)@)r66qUGZI?w+&BkzH4D$-rwBY8y+1^3C#dMF^qi_7SMLSv*lx66p;Ao
z`jTk*t$XKx&lJfgn>OO!l6IEcciSI?{n+$;;uq`>>3OJcT^y~W0@Gw_cpvp@_76fL
z)Fgy{2$*ap6%f??RTpMnbpr&8eP}Q8ONqRZ0ywWF0IdH__=5LG-^CkrLd(NZ+eVlvafz&*%U_%w=>?i0NGu~OgSn@5LkyBeMv?mIheM~#g~r>jqD_p*8k
zK{cqxEb`9ij%4^nUamCX9rWX~zoM_NhVDGL*ws?~yLO!?u%D}QsiU&A*!zLXlgsBT
z*U#SUCss7vicr3MFB!Ri!7)8>)qkjE{3=tO-oQy#h_;H>Dc3ZL46eGh6&iehx&NPF
zH6M+P4F4m>ZrsDdOvNTj_uFDm8I@5MTH6`0Mw;{>NAS*K3B={44Oa94>nY
zC4R+k&T@=`uPg9vN)r*N*d_$q+^t?vd=g6zkCx`bgE<*-*
zIhoG;jP5;-eEc{x!e8EKY;R9NlnU-T=cyzGD$kdLV@&}1UPTxUzPkFyY65HzoqWYQ
z8fZcJ=0{p;laBO>ZjpX|KfjLpl$MroiCs_Z;wA@1oQ{$wBqQ1n5+&^-yn8n?!%OZx
zx(ZbYTfV+>J>vK98=r;^CeWqP)%w*J`_;3#t{T6Erunk4zig>tL4JOp_9KojROhYz
zoAV?;E8Xeu?hW$~2tRN33BM!BMM!wFUsxrY1+0=VrZlD`PNJd&UHmAj$Y&cRA**-q
z{6mhi$jd*&!xCGmx1p}k$E`vN!?fiq+O5DG8gZdB3?mg+fHAhl>
z(-YD6(zj1e%1Hk@y?t~nxk&37HCC?V&s7dv`B%;^==6UV+nrM18cpN*{#Uqo1zD5I(F
zH$ailYo6g{ig7E7$#gd>tyY6-7OQZPoNTS`RCy7c4E1L4N}Rd7_Y(EhP~=6~AyeY;
zB`v<(w|AjMo+4`h;9&PvRsK6S!{ObHzxpe;ym_7c;-a|0NdQ(N+?XWqWrV_2Z%(x9
zil>OggK7I_WxX0}Cfi-HFm)KswP<%XgsdD4B&!hwX0xF1rw0G`Q9e2<1s^?c(=jpzk5J6?O~s_OHe$mCLhAufz@9D>1P&1Mx|*
zv3)lo0C1c=)AslmSEdADESI#F;~eZ+)n~q_00!|oDHc8iBvI(LGoPB-pqM&er`P@5
zKO4san$!S@c*kZYkr7!ii2m^JrOb2-Y8KL=@1(Yx8tCKVq8RY+X;
zFbLW)S#3yf!WEpZf%pDYlUDqDZ{R;2*S>s5-3@2I`g%9V`RY!eUUISC6cs~(U9iQ7F}sEIov=cIa_@L&l_^9ho=on@)`L6
zka9)Ax953q<{Ehr3tz>YG!H)?EP)l1Z1BZx;5Nx-v$CORU+A4SKJmi)p5eMf>N3Od
z4Ia>1*s>(44lNR>>t!NBkL_saHC??f8Y&kU4O!;#g?DVc=L0{6Qp$h=O-;!SJ?~c>
zDrNOSJ-u+mzeh|bw$Qy8_^omu8#7b0;=t@@+CR__)bz%<7VUM-TWoEOC9Z^VpV!b(
zP`;m3Ust5Y1#pik-`ZTixI8Kg>+Ksb@{-x=nR67Hl(roR3+fJaRSGf~kX8UcZ0ih7z^mY1e5eeB`y*pR-Neb|Sg;V!>#@5~l`e
zQ`z@f?NM2(>V9&B$%B{I6S{ot22MBJ7w#^<39Hvju&z$+QkXyNKdmlW>YMkm%j;#e
z`Zc!Fu0L=-f4Qh);Pd&l`E6|O%JBY7=fGLYgULjeE$6?$bagrQ;`D^v}
za8TEq@PPYm;wgjx+ZTpc2azYIiNSK8d4qip1E=hq`|QffO4@B>0r-4$0Qp~=+iBxi
zf2}`>LH#?k+M5!IeUhJ_njG^n=B%k(5@Ak07KOhQk`
z_HESl^AdavQezoh;yDMGrbt(d`V@}>z;ZZw{z6{qLl};1{4=a8Sj+sydm88~EiCAV
zSSQ7V>>J={-+_pL3$3ep-W(gwp2ImIXuzKz5B_eSubu;uNPQ_Li8%7reB|-K-`|O<
zAR2W{Ean@zNa63-s-F+5*+nOZGT&6_}+po6F`
z?;o`ItcLw?x7u{rPVA&27ar|RKS=Ali?Q*Vjv}exxF#Eo2J+Ja@ES`#Qtgvs*RTJp
z9wfVS*ox!@l|7O&O5FKJI4c*IzrT;BM0C`XYupO1PfkwTfo6iAB-^TT@BKU9hB`UOhLUsQ-B#pe0O{b16G
zZEAF}P5I`>Dn$k!=0Yxj1WEch!>jK{MgbGpDqSi{^(wIUArl%JbUOec2SWYFsh*m&lu)P%ZhTBN
zCS*dgCskn^DO0$pLF&46j#V`UW>0Cs5O{G}ajY7{kACv
z+xMoQALS%F=4RF4pMqkALVLR_SWN6`d^HtbP#ab;rNeHTaZt$8r?@%6kpC$raXl|$
zF~KKAo~*x67GJey{GVU7|#+x7>^P$$hVFhQy4WH-OY?}i63cR-Yr_JQj3J0ONsGvkXN0VaJgDMCg=
zHDT?XZXN&Z9W6%%XLa>w(HMGds5PIxXTJVw_|_kv@b&pB{l1Hmd?iXb!MvKJ?b5c*
z>IUu9zkjyM($+1j8$e(kHkf+V%Gx%oIw_~Y-5g6MFR$3}fOUYBW-GJYvlPrx4enSO
z1sV-U6g5;!)X8qkePa4C4W13b%KJG8s_B)w_Yy^WmCU
zBLjPCnd_X6sQ^dorGL$d=}Qf;WEoQ7rOcl&Sg2>87R|Yqkw6FXMcycdm;
z|3-Nes^?q(PNOq}hb=WQM`}y7S$w`foXHxkq#2;#E66$s7>U~WkJBEt^_mk>6l#J;
zenEYSa%%AgwYVqz<&;1GKs14x3XVo($u|aE7y(~lC|eLKz!@fI94i!uEXU}=4d(e0|BWLG6W(aW
zWp1%LmU2)rc-_pW1#
zspfy_oP-tG1h5h`qYmV+P1jH-3RX$}ME|8>1P=ftcxR*|@(Xp)IHnCii+zpZmjk^0
z?yjTz+MkpNQI-TGD<&%~@nQlO9~IY4SeA_%O4>QtG)}?aaMb7X#s6c#RLb$Oa)bA_
zED$2l+sEgfn?6Jd!(vM8b+bsdwD;EOvm3Fk9rk-&y*A+(r0}`Ss}s;;C}T}(Nkr+4
zSkTE%y=tgkc@z+4@Yq)__x@ZE-XK+WeM$ba?_)FO*CSHD7DKFyW*B7#~4ot66zshNsGXZP?iXtkd^?0#;OkPC4(QA3j4ZlI(n0&i5;8s
zFO=S`N3^2}vMKg(x3=8e$a=e3+w)^_y1<+H}xOzG}d|DOdoD8;Nt8S-tp+p&p)
z5rW3S59_~rkC-(xgYqq#TLxKy#`}=L*8Dzr^qo3>AapB?cFQvk@}J0R$EB#ZKA9x1C>i`qusX?#IQz`sTnaU`<@T
zs5P*JQ^Uz$BM?hdDbbPCWA1mPV#wHOfZvo5a+hbgt!W0K3;+lSII6?UWKvN~BAT>u
z+yAN`os>?2ruNVIuXgo9f0m(yc2Jwl$)ec962%uC%bX0ask4^1J&Xvnat%z?41^hmmk9r*t~TEP?`eST%csosSsix~
z|0x<)XLr|EgP%V{JOr3poa|cb+5JmMyw+~6cZ2@9XiupS
zET{DTSLH!%;KYjT^hdeoBtB+)&RVfbsSZrshvHiuw_iA?QEqVN1HZr28h*$+Wk4LU
zVVnUnfZ{znvTw{bBo_fq{-FXkNC6dyqCm(9F=Hu_>O$-$0=^asg6aXjAplx(?WiXRUknDuX0j@<=wiWfh#r6GJXjEtu+s@{E=p+Qv
zQV1~x@Ke?cj?M<{ANU%}Fu4_OMWuQkxE=^~65?YM$Ks{$-AnlIQ#b>FmIA?iH`6Qx
zF|s=sZ>gJ;sYD6TqBIHDHvxcRir~GC6hqIalB<)8s#&okJiH+fmhn0blpo($>4P-O
zM?U5EX7Lp!Lt1eF0hq9F-7OWc02WF~$=?V|;2awIyCvxM)x4RYSHIvTuVwmh6Ysj~
zT4_yg%t>NYGvHe0=}~*(NM9}|iH|8C%6M4U@rzI1pfK$mX>40EaVz%Q=5{3$0PhM5
z@WhDAO=s2lCBu|7)fGZyu{4axjQsCgLDvYngqM6K91sDy#4(vvO8YNyoJ+yth5=J}
zTJ7lA2G)9X1And!g&$Pdzxi`;Y(22j99|7`EZyRCELxlms!XxAeK7~S^;(~P6xx&F
z!K_gZ7zdy>xz2hE-0WA0}urxR!`!)2UZ7T1LRC7v~U#AO^mzj?do^qE$`yuz5uj0ynib;D3x2C?3RCFU$AIw
z@j6dD@2fSM(wpi!?J_E}3;(g(;m{O%Fu$|mbS=SRi=!{GZxF1McJykMH~9R*V&IIe
zIjH~hJ#Ozj=P>`!p4F$a0;|Ur9#X-GJ&E%=GpdH?H?%+0|27MfM0CbpzyG#%q!VLG
z&;7Kq+w4Wct|jqWTc0?`md5<6*pQV5uX%T;G=Ny;GlM7!>R&`rF9ok2lpR+)cja}gS4}&UalFaTW
zG$Z1NPxyt0t!~JCSL%qA3=@xm~WH==+nHtT+KFo2d<7u)AB|CpsS2G
zSWAdHFo343P|t3&;@WgsDf6GbE}wtGO!e&ddgzq5HlFr(z>KV5dt-`>KiT^5mN#*K
z*yzpA{}7AMAMXu5D^_C#zyhFvW)7F7!E6sdglS`?V;biSWSK$=RQV9@AJBE}rObw7
z#8vAO57Q(@H|PA~hj&P^p2OP~9nlkVqr$%oo6P`E?fU-3zo3TF*VlxukdS|9>C@t~
zS^fI)5iT;{?oHS*Z-0PJOJd@=k?O^mH2m9-zD!j<6O#$K+%Uk3lM%KH62aZiOe;$&~&x9=5{OtiJ3dTlK(STtKfM~YB=
zdPF=~k&M{quv3W)j=X|mlcJJ{9obFRbB*%p5}~44>~UAbURR#rma~Zo3y=b!MeVq~
z$jbFW%|P;n#PqZTz_r+gLJMUo6C!mjqfb8bT2XO{w?>sa7KJABfZ@i0QocT>6gVL}
z^7XM%5izXYo5wwbloXE~x8lQtTO+I!M6VlLOGQr*Du!)!;^<9uZo~R(yjp!sG2v}Q
zq11>H#}+5(r(dLRhS$Jjzmwx%2lnXO^-6S#2x%z*i{DbmK}J4fYG)a
z%7t)d=5#Uobbol|x8uNzqJbYCE=wfdF%iu+QN`zXl$2Yu(TyXly`2{)53atgUKm~O
zmW7``;X|Z@QEXDC0q9CULSC(g{hhlD5r01I
zhDF%I;j4WEl>B~%p?oUNRlJF@>Aw={QlPxVTs|K_b4fRuc9Q
z3X>Jmtg?}o&Umi8k=cix?|8XKN-PTtdlF)B+$}%_cpT`;Ez(-&IUjrf(dGBAo^3r(
zo|NRTj1ndwqZ43O{cyXQwyd=Y^1{OC4o6w!um5}>poxPk;1wb*L^1cOtX9D*^V{nw
zin6xX1yNXYSBn(=R5!iyu%8rhHIL#<*H
zRnVE`mk+*3^jlzYZw{^g@=0!aXlYplZ59bnhjx;7Cw9mu>#;Y-XheULUqtjS5kKD$
zlo7N~BCt{xj9PkyhjriR{jd1`oq-#7Gs=avX{3jTvS}+RojcfFDXnKh1*AWUTMiYY
z7+Jq8dh)h9tSEdGz)@MEL|`3aCj)M9SmuGvNI1|w6Ip$`A?1Bc{@rpN&(Co~Z_j=^
zKG}L_;NU5rco_bli9(a(8+xcTzs!oXDN5(qe_++7FTnR=@j0ojaV38wmoctYXhb_%
z`-6AOk+&*Sf9m;;uESZ-nY~sdXbUkO_?T4`5@XmTr(^nL{R=
z7w;_KLim1__uybGxs=mXl+O`PLT(R=Qu|&WIxBRI%NnKF&%^*Fwx20VBMEaV++N;6
zEpA-mC|0CD&RWyJk)HO0V+jQd?5yG;bDZRJ8Nb^1&DAbAfjE#-lQ}Eh^qk7wc!6Pj
zffcRqr=cOCyB)kOMET^5)OYBXUu^|bDUYv1N2D2U
z#{aV+fPP;ADj!)&<0$h*fH*ckqnPw#T#gk*Hj$p9H2lNMDO@MM7R(a|r_+QZeB9o}
zl~XDbSj@7kL};(gQDM65g#-$B4K?%cyu`#T!(Y=(x`!usA<Tr|G%%Fg*F@a4}Sg>McO-RTFQ8=rTvF0RC29nB4|5#uHbdV
z-90LrS?yk!mo(952`PdAaT#uN0iGof+by!OY>jW9CfmN3U&?DOXy3gv}dgx7)JgGz~oomLWEjqXL5
zUoLs2U&v~T_oZJ_J9necY_?pj$NaFpSy_ooDUG#d{aUj_TtnGS@fcGPm^;o=4uXRK
zXdfn5%m-FU&zNXZb_X(Ks;|px-jESzN|B#NeWz4^bX3h~CE8MJ>QGcUZ;0LMyzgrb
z;9`QM%(-#s+#=|Ng@!Ji^!MD2`1||&chAx-fsGhRoNo-j*YaU!UagDp-g63(GFv5x~aNo*p$$`u?(wOl1zBwSyrPtVBGXe6EXlNv<$93=OkHXZ{
z=`$mVm`~EN3waPR{$)kdyH3@<-q66m@3!nF7rgFIlwWJq6-Wb#Thr%VeDnVNXTNBg
zr^(w>5~r@PozO-DPwp*8UrUbIKC7T`=2B*dU_EjnZ<`4H!6n(Ses?WG^=-2%Rzapp
z(Txu(+}br_tnun3Y3nEcN&MgfuIg;8Dy@uQeq9}Xv~)tHwoA>Mz4#P68=K3CvWTl)
z_tVI~iC0hz6Vr7oa4De&{KTPPd|}aPf7HTGX<)ImWMDPuu2en;*P!~kGpNB4*W7vX
z*j9JGfzJU4g$aO5GuD8u49zB%D#}Y~Y{MT!EvreX0J>{>^xbXlOl~uz#+~73yb4({
z5u8j9*Sypg-VO^|ojLR_my{F_6!9K(_Prg(1${Ad_t1ENM9>@F%D&^Q>3Mp7{L0R2
znS;+BC}ko+o8_9(Lg759HiSuN5xMh;(dg9xH>Hu&U=jGEDVOHjLA>*#k8QRL?5;T7
zc76)2)HLoEpBN$iq+y_Y*(+rI8+r7fLD=$Oz|JxsFA|;VS-QD)-cxFO&`DZWNzQ$K
zQd9T-XB{+S=kszQ!f3sFU^(LWGSR}KFV-V*-rI&FXPl5aXYD2Tu>ezLyc=tD>t5J@
zb=e1>Ywbj1b%7s~fc_cZu~f3z4yEt1vPh)1YrAJBCq=Ioe0Pv=25|y`K>}S+^)#rk
z5WM<2^l1C+CBL-B6G26eK*1+$K#{LMzstVub8sc9$mE7;eBJfMQP9|Py^$5~*x&l|
z-?nSpLQ`)GhlGA3Hl%fzrn=+jP>sdg$K+F6INDs}0jd|K@j%%Dd*kuGSzNca!zLzB
zE*tyccy7P1OhsjMxXq#ejo*1%?ma>QR6k>iAYknjuCFZr`N8)zCLgA~j=+9At`q4*
z39QD8-uAfq4z*h}BQJgULHyW^nSszQY+A*m610sqdo<(^@1G10$9iaKq*!Oq-F0cO
zlHdqz%akur4Z444l>4<(RjdYuw)z+1O||4)W5c!}VFe%Cn;}_sEzjgF`b>R=KDorO
z5)6H(Wv`PBClMVxAJ(&Ig+X}O+868oy*X4RgyC7fagkMPOW~9Gnzw`!IF$}!=3@s-
z&a1ESfASjLJiUtyr9mmX2~?CgUks%-Z5*CG`?z`a&)F#Xlh_;vX0{e~DD
z>TOTp_1;ehnUg~west`OWiKQ-S;qC;p|F-Et#C6n-h0NDv7D3M?#o6m(-?qjaBtMy
z*
It had only been two years since Addison v. Clark.
+ The court case gave us a revised version of what life was
+
+(shamelessly ripped out of `this thread `_). You'd have to remove some of the tags as well. In this example, I'd recommend beginning with the tag ````, now you have to end with the corresponding closing tag (opening tags are ````, closing tags are ````), which is simply the next ```` in this case. (Refer to a good HTML manual or ask in the forum if you are unclear on this point.) The opening tag can be described using ````, the closing tag using ````, thus we could remove everything between those tags using ``.*?``. But using this expression would be a bad idea, because it removes everything enclosed by - tags (which, by the way, render the enclosed text in bold print), and it's a fair bet that we'll remove portions of the book in this way. Instead, include the beginning of the enclosed string as well, making the regular expression ``\s*Generated\s+by\s+ABC\s+Amber\s+LIT.*?`` The ``\s`` with quantifiers are included here instead of explicitly using the spaces as seen in the string to catch any variations of the string that might occur. Remember to check what |app| will remove to make sure you don't remove any portions you want to keep if you test a new expression. If you only check one occurrence, you might miss a mismatch somewhere else in the text. Also note that should you accidentally remove more or fewer tags than you actually wanted to, |app| tries to repair the damaged code after doing the header/footer removal.
+
+Adding books
+^^^^^^^^^^^^^^^^
+
+Another thing you can use regular expressions for is to extract metadata from filenames. You can find this feature in the "Adding books" part of the settings. There's a special feature here: You can use field names for metadata fields, for example ``(?P)`` would indicate that calibre uses this part of the string as book title. The allowed field names are listed in the windows, together with another nice test field. An example: Say you want to import a whole bunch of files named like ``Classical Texts: The Divine Comedy by Dante Alighieri.mobi``.
+(Obviously, this is already in your library, since we all love classical italian poetry) or ``Science Fiction epics: The Foundation Trilogy by Isaac Asimov.epub``. This is obviously a naming scheme that |app| won't extract any meaningful data out of - its standard expression for extracting metadata is ``(?P.+) - (?P[^_]+)``. A regular expression that works here would be ``[a-zA-Z]+: (?P.+) by (?P.+)``. Please note that, inside the group for the metadata field, you need to use expressions to describe what the field actually matches. And also note that, when using the test field |app| provides, you need to add the file extension to your testing filename, otherwise you won't get any matches at all, despite using a working expression.
+
+Bulk editing metadata
+^^^^^^^^^^^^^^^^^^^^^^^
+
+The last part is regular expression search and replace in metadata fields. You can access this by selecting multiple books in the library and using bulk metadata edit. Be very careful when using this last feature, as it can do **Very Bad Things** to your library! Doublecheck that your expressions do what you want them to using the test fields, and only mark the books you really want to change! In the regular expression search mode, you can search in one field, replace the text with something and even write the result into another field. A practical example: Say your library contained the books of Frank Herbert's Dune series, named after the fashion ``Dune 1 - Dune``, ``Dune 2 - Dune Messiah`` and so on. Now you want to get ``Dune`` into the series field. You can do that by searching for ``(.*?) \d+ - .*`` in the title field and replacing it with ``\1`` in the series field. See what I did there? That's a reference to the first group you're replacing the series field with. Now that you have the series all set, you only need to do another search for ``.*? -`` in the title field and replace it with ``""`` (an empty string), again in the title field, and your metadata is all neat and tidy. Isn't that great? By the way, instead of replacing the entire field, you can also append or prepend to the field, so, if you *wanted* the book title to be prepended with series info, you could do that as well. As you by now have undoubtedly noticed, there's a checkbox labeled :guilabel:`Case sensitive`, so you won't have to use flags to select behaviour here.
+
+Well, that just about concludes the very short introduction to regular expressions. Hopefully I'll have shown you enough to at least get you started and to enable you to continue learning by yourself- a good starting point would be the `Python documentation for regexps `_.
+
+One last word of warning, though: Regexps are powerful, but also really easy to get wrong. |app| provides really great testing possibilities to see if your expressions behave as you expect them to. Use them. Try not to shoot yourself in the foot. (God, I love that expression...) But should you, despite the warning, injure your foot (or any other body parts), try to learn from it.
+
+Credits
+-------------
+
+Thanks for helping with tips, corrections and such:
+
+ * ldolse
+ * kovidgoyal
+ * chaley
+ * dwanthny
+ * kacir
+ * Starson17
+
+
diff --git a/src/calibre/manual/tutorials.rst b/src/calibre/manual/tutorials.rst
index 1e4cab8493..084c44ff64 100644
--- a/src/calibre/manual/tutorials.rst
+++ b/src/calibre/manual/tutorials.rst
@@ -14,5 +14,6 @@ Here you will find tutorials to get you started using |app|'s more advanced feat
news
xpath
template_lang
+ regexp
portable
From c3dadbf3041d5eb828fb3b3a7bd91b5f3c69a6e2 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Mon, 27 Sep 2010 11:43:06 -0600
Subject: [PATCH 166/207] ...
---
src/calibre/manual/regexp.rst | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/src/calibre/manual/regexp.rst b/src/calibre/manual/regexp.rst
index 5927cfc1a3..5cd9a8b097 100644
--- a/src/calibre/manual/regexp.rst
+++ b/src/calibre/manual/regexp.rst
@@ -8,8 +8,9 @@ All about using regular expressions in |app|
Regular expressions are features used in many places in |app| to perform sophisticated manipulation of ebook content and metadata. This tutorial is a gentle introduction to getting you started with using regular expressions in |app|.
-.. toctree::
- :maxdepth: 2
+.. contents:: Contents
+ :depth: 2
+ :local:
First, a word of warning and a word of courage
@@ -80,7 +81,7 @@ You missed...
... wait just a minute, there's one last, really neat thing you can do with groups. If you have a group that you previously matched, you can use references to that group later in the expression: Groups are numbered starting with 1, and you reference them by escaping the number of the group you want to reference, thus, the fifth group would be referenced as ``\5``. So, if you searched for ``([^ ]+) \1`` in the string "Test Test", you'd match the whole string!
-You missed something. In the beginning, you said there was a way to make a regular expression case insensitive?
+In the beginning, you said there was a way to make a regular expression case insensitive?
------------------------------------------------------------------------------------------------------------------
Yes, I did, thanks for paying attention and reminding me. You can tell |app| how you want certain things handled by using something called flags. You include flags in your expression by using the special construct ``(?flags go here)`` where, obviously, you'd replace "flags go here" with the specific flags you want. For ignoring case, the flag is ``i``, thus you include ``(?i)`` in your expression. Thus, ``test(?i)`` would match "Test", "tEst", "TEst" and any case variation you could think of.
From 62f1edd84879022bcd89d75fcc70bdfb0d33fed1 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Tue, 28 Sep 2010 11:41:35 +0100
Subject: [PATCH 167/207] Cleanups of plugboard code. Improvements to the gui.
---
src/calibre/ebooks/metadata/book/base.py | 1 -
src/calibre/gui2/device.py | 14 +-
src/calibre/gui2/preferences/plugboard.py | 196 +++++++++++++---------
src/calibre/gui2/preferences/plugboard.ui | 80 +++++----
src/calibre/library/save_to_disk.py | 19 ++-
5 files changed, 191 insertions(+), 119 deletions(-)
diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index aaa7c78e9a..951a55da10 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -303,7 +303,6 @@ class Metadata(object):
return
for src in attrs:
try:
- print src
sfm = other.metadata_for_field(src)
dfm = self.metadata_for_field(attrs[src])
if dfm['is_multiple']:
diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py
index 72ad8b1890..4c866b1855 100644
--- a/src/calibre/gui2/device.py
+++ b/src/calibre/gui2/device.py
@@ -34,6 +34,8 @@ from calibre.ebooks.metadata.meta import set_metadata
from calibre.constants import DEBUG
from calibre.utils.config import prefs, tweaks
from calibre.utils.magick.draw import thumbnail
+from calibre.library.save_to_disk import plugboard_any_device_value, \
+ plugboard_any_format_value
# }}}
class DeviceJob(BaseJob): # {{{
@@ -323,22 +325,22 @@ class DeviceManager(Thread): # {{{
for f, mi in zip(files, metadata):
if isinstance(f, unicode):
ext = f.rpartition('.')[-1].lower()
- dev_name = self.connected_device.name
+ dev_name = self.connected_device.__class__.__name__
cpb = None
if ext in plugboards:
cpb = plugboards[ext]
- elif ' any' in plugboards:
- cpb = plugboards[' any']
+ elif plugboard_any_format_value in plugboards:
+ cpb = plugboards[plugboard_any_format_value]
if cpb is not None:
if dev_name in cpb:
cpb = cpb[dev_name]
- elif ' any' in plugboards[ext]:
- cpb = cpb[' any']
+ elif plugboard_any_device_value in plugboards[ext]:
+ cpb = cpb[plugboard_any_device_value]
else:
cpb = None
if DEBUG:
- prints('Using plugboard', cpb)
+ prints('Using plugboard', ext, dev_name, cpb)
if ext:
try:
if DEBUG:
diff --git a/src/calibre/gui2/preferences/plugboard.py b/src/calibre/gui2/preferences/plugboard.py
index 7fdd093dc1..b723fb938c 100644
--- a/src/calibre/gui2/preferences/plugboard.py
+++ b/src/calibre/gui2/preferences/plugboard.py
@@ -8,32 +8,84 @@ __docformat__ = 'restructuredtext en'
from PyQt4 import QtGui
from calibre.gui2 import error_dialog
-from calibre.gui2.preferences import ConfigWidgetBase, test_widget, \
- AbortCommit
+from calibre.gui2.preferences import ConfigWidgetBase, test_widget
from calibre.gui2.preferences.plugboard_ui import Ui_Form
from calibre.customize.ui import metadata_writers, device_plugins
-
+from calibre.library.save_to_disk import plugboard_any_format_value, \
+ plugboard_any_device_value, plugboard_save_to_disk_value
class ConfigWidget(ConfigWidgetBase, Ui_Form):
def genesis(self, gui):
self.gui = gui
self.db = gui.library_view.model().db
- self.current_plugboards = self.db.prefs.get('plugboards', {'epub': {' any': {'title':'authors', 'authors':'tags'}}})
+ self.current_plugboards = self.db.prefs.get('plugboards',{})
self.current_device = None
self.current_format = None
-# self.proxy = ConfigProxy(config())
-#
-# r = self.register
-#
-# for x in ('asciiize', 'update_metadata', 'save_cover', 'write_opf',
-# 'replace_whitespace', 'to_lowercase', 'formats', 'timefmt'):
-# r(x, self.proxy)
-#
-# self.save_template.changed_signal.connect(self.changed_signal.emit)
+
+ def initialize(self):
+ def field_cmp(x, y):
+ if x.startswith('#'):
+ if y.startswith('#'):
+ return cmp(x.lower(), y.lower())
+ else:
+ return 1
+ elif y.startswith('#'):
+ return -1
+ else:
+ return cmp(x.lower(), y.lower())
+
+ ConfigWidgetBase.initialize(self)
+
+ self.devices = ['']
+ for device in device_plugins():
+ n = device.__class__.__name__
+ if n.startswith('FOLDER_DEVICE'):
+ n = 'FOLDER_DEVICE'
+ self.devices.append(n)
+ self.devices.sort(cmp=lambda x, y: cmp(x.lower(), y.lower()))
+ self.devices.insert(1, plugboard_save_to_disk_value)
+ self.devices.insert(2, plugboard_any_device_value)
+ self.new_device.addItems(self.devices)
+
+ self.formats = ['']
+ for w in metadata_writers():
+ for f in w.file_types:
+ self.formats.append(f)
+ self.formats.sort()
+ self.formats.insert(1, plugboard_any_format_value)
+ self.new_format.addItems(self.formats)
+
+ self.fields = ['']
+ for f in self.db.all_field_keys():
+ if self.db.field_metadata[f].get('rec_index', None) is not None and\
+ self.db.field_metadata[f]['datatype'] is not None and \
+ self.db.field_metadata[f]['search_terms']:
+ self.fields.append(f)
+ self.fields.sort(cmp=field_cmp)
+
+ self.source_widgets = []
+ self.dest_widgets = []
+ for i in range(0, 10):
+ w = QtGui.QComboBox(self)
+ self.source_widgets.append(w)
+ self.fields_layout.addWidget(w, 5+i, 0, 1, 1)
+ w = QtGui.QComboBox(self)
+ self.dest_widgets.append(w)
+ self.fields_layout.addWidget(w, 5+i, 1, 1, 1)
+
+ self.edit_device.currentIndexChanged[str].connect(self.edit_device_changed)
+ self.edit_format.currentIndexChanged[str].connect(self.edit_format_changed)
+ self.new_device.currentIndexChanged[str].connect(self.new_device_changed)
+ self.new_format.currentIndexChanged[str].connect(self.new_format_changed)
+ self.ok_button.clicked.connect(self.ok_clicked)
+ self.del_button.clicked.connect(self.del_clicked)
+
+ self.refill_all_boxes()
def clear_fields(self, edit_boxes=False, new_boxes=False):
self.ok_button.setEnabled(False)
+ self.del_button.setEnabled(False)
for w in self.source_widgets:
w.clear()
for w in self.dest_widgets:
@@ -47,6 +99,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
def set_fields(self):
self.ok_button.setEnabled(True)
+ self.del_button.setEnabled(True)
for w in self.source_widgets:
w.addItems(self.fields)
for w in self.dest_widgets:
@@ -76,6 +129,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
for i,src in enumerate(dpb):
self.set_field(i, src, dpb[src])
self.ok_button.setEnabled(True)
+ self.del_button.setEnabled(True)
def edit_format_changed(self, txt):
if txt == '':
@@ -104,26 +158,42 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
self.clear_fields(edit_boxes=True)
self.current_device = unicode(txt)
error = False
- if self.current_format == ' any':
+ if self.current_format == plugboard_any_format_value:
+ # user specified any format.
for f in self.current_plugboards:
- if self.current_device == ' any' and len(self.current_plugboards[f]):
+ devs = set(self.current_plugboards[f])
+ print 'check', self.current_format, devs
+ if self.current_device != plugboard_save_to_disk_value and \
+ plugboard_any_device_value in devs:
+ # specific format/any device in list. conflict.
+ # note: any device does not match save_to_disk
error = True
break
- if self.current_device in self.current_plugboards[f]:
+ if self.current_device in devs:
+ # specific format/current device in list. conflict
error = True
break
- if ' any' in self.current_plugboards[f]:
+ if self.current_device == plugboard_any_device_value:
+ # any device and a specific device already there. conflict
error = True
break
else:
- fpb = self.current_plugboards.get(self.current_format, None)
- if fpb is not None:
- if ' any' in fpb:
+ # user specified specific format.
+ for f in self.current_plugboards:
+ devs = set(self.current_plugboards[f])
+ if f == plugboard_any_format_value and \
+ self.current_device in devs:
+ # any format/same device in list. conflict.
error = True
- else:
- dpb = fpb.get(self.current_device, None)
- if dpb is not None:
- error = True
+ break
+ if f == self.current_format and self.current_device in devs:
+ # current format/current device in list. conflict
+ error = True
+ break
+ if f == self.current_format and plugboard_any_device_value in devs:
+ # current format/any device in list. conflict
+ error = True
+ break
if error:
error_dialog(self, '',
@@ -165,6 +235,16 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
self.changed_signal.emit()
self.refill_all_boxes()
+ def del_clicked(self):
+ if self.current_format in self.current_plugboards:
+ fpb = self.current_plugboards[self.current_format]
+ if self.current_device in fpb:
+ del fpb[self.current_device]
+ if len(fpb) == 0:
+ del self.current_plugboards[self.current_format]
+ self.changed_signal.emit()
+ self.refill_all_boxes()
+
def refill_all_boxes(self):
self.current_device = None
self.current_format = None
@@ -176,59 +256,21 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
self.edit_format.setCurrentIndex(0)
self.edit_device.clear()
self.ok_button.setEnabled(False)
-
- def initialize(self):
- def field_cmp(x, y):
- if x.startswith('#'):
- if y.startswith('#'):
- return cmp(x.lower(), y.lower())
- else:
- return 1
- elif y.startswith('#'):
- return -1
- else:
- return cmp(x.lower(), y.lower())
-
- ConfigWidgetBase.initialize(self)
-
- self.devices = ['', ' any', 'save to disk']
- for device in device_plugins():
- self.devices.append(device.name)
- self.devices.sort(cmp=lambda x, y: cmp(x.lower(), y.lower()))
- self.new_device.addItems(self.devices)
-
- self.formats = ['', ' any']
- for w in metadata_writers():
- for f in w.file_types:
- self.formats.append(f)
- self.formats.sort()
- self.new_format.addItems(self.formats)
-
- self.fields = ['']
- for f in self.db.all_field_keys():
- if self.db.field_metadata[f].get('rec_index', None) is not None and\
- self.db.field_metadata[f]['datatype'] is not None and \
- self.db.field_metadata[f]['search_terms']:
- self.fields.append(f)
- self.fields.sort(cmp=field_cmp)
-
- self.source_widgets = []
- self.dest_widgets = []
- for i in range(0, 10):
- w = QtGui.QComboBox(self)
- self.source_widgets.append(w)
- self.fields_layout.addWidget(w, 5+i, 0, 1, 1)
- w = QtGui.QComboBox(self)
- self.dest_widgets.append(w)
- self.fields_layout.addWidget(w, 5+i, 1, 1, 1)
-
- self.edit_device.currentIndexChanged[str].connect(self.edit_device_changed)
- self.edit_format.currentIndexChanged[str].connect(self.edit_format_changed)
- self.new_device.currentIndexChanged[str].connect(self.new_device_changed)
- self.new_format.currentIndexChanged[str].connect(self.new_format_changed)
- self.ok_button.clicked.connect(self.ok_clicked)
-
- self.refill_all_boxes()
+ self.del_button.setEnabled(False)
+ txt = ''
+ for f in self.formats:
+ if f not in self.current_plugboards:
+ continue
+ for d in self.devices:
+ if d not in self.current_plugboards[f]:
+ continue
+ ops = []
+ for op in self.fields:
+ if op not in self.current_plugboards[f][d]:
+ continue
+ ops.append(op + '->' + self.current_plugboards[f][d][op])
+ txt += '%s:%s [%s]\n'%(f, d, ', '.join(ops))
+ self.existing_plugboards.setPlainText(txt)
def restore_defaults(self):
ConfigWidgetBase.restore_defaults(self)
diff --git a/src/calibre/gui2/preferences/plugboard.ui b/src/calibre/gui2/preferences/plugboard.ui
index ad72ec359f..f88af8ff50 100644
--- a/src/calibre/gui2/preferences/plugboard.ui
+++ b/src/calibre/gui2/preferences/plugboard.ui
@@ -26,32 +26,6 @@
-
-
-
- Add new plugboard
-
-
-
-
-
-
- Edit existing plugboard
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -72,7 +46,50 @@
+
+
+
+ Add new plugboard
+
+
+
+
+
+
+
+
+
+
+
+
+ Edit existing plugboard
+
+
+
+
+
+
+
+
+
+
+
+ Existing plugboards
+
+
+ Qt::AlignLeading|Qt::AlignLeft|Qt::AlignTop
+
+
+
+
+
+
+ QPlainTextEdit::NoWrap
+
+
+
+ Qt::Vertical
@@ -122,10 +139,17 @@
-
+
- Done
+ Save
+
+
+
+
+
+
+ Delete
diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py
index 2504832df7..5465150797 100644
--- a/src/calibre/library/save_to_disk.py
+++ b/src/calibre/library/save_to_disk.py
@@ -17,7 +17,12 @@ from calibre.ebooks.metadata.meta import set_metadata
from calibre.constants import preferred_encoding, filesystem_encoding
from calibre.ebooks.metadata import fmt_sidx
from calibre.ebooks.metadata import title_sort
-from calibre import strftime
+from calibre import strftime, prints
+
+plugboard_any_device_value = 'any device'
+plugboard_any_format_value = 'any format'
+plugboard_save_to_disk_value = 'save_to_disk'
+
DEFAULT_TEMPLATE = '{author_sort}/{title}/{title} - {authors}'
DEFAULT_SEND_TEMPLATE = '{author_sort}/{title} - {authors}'
@@ -232,21 +237,21 @@ def save_book_to_disk(id, db, root, opts, length):
written = False
for fmt in formats:
- dev_name = 'save to disk'
+ global plugboard_save_to_disk_value, plugboard_any_format_value
+ dev_name = plugboard_save_to_disk_value
plugboards = db.prefs.get('plugboards', {})
cpb = None
if fmt in plugboards:
cpb = plugboards[fmt]
- elif ' any' in plugboards:
- cpb = plugboards[' any']
+ elif plugboard_any_format_value in plugboards:
+ cpb = plugboards[plugboard_any_format_value]
+ # must find a save_to_disk entry for this format
if cpb is not None:
if dev_name in cpb:
cpb = cpb[dev_name]
- elif ' any' in plugboards[fmt]:
- cpb = cpb[' any']
else:
cpb = None
-
+ prints('Using plugboard:', fmt, cpb)
data = db.format(id, fmt, index_is_id=True)
if data is None:
continue
From e08da942ec7a1c12db488c0e49bd6cb0c61aef6b Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Tue, 28 Sep 2010 11:46:53 +0100
Subject: [PATCH 168/207] Fix typo in faq.rst (too -> to)
---
src/calibre/manual/faq.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/calibre/manual/faq.rst b/src/calibre/manual/faq.rst
index c9f6abe2c0..3cf171bc1b 100644
--- a/src/calibre/manual/faq.rst
+++ b/src/calibre/manual/faq.rst
@@ -289,7 +289,7 @@ Yes, you can. Follow the instructions in the answer above for adding custom colu
How do I move my |app| library from one computer to another?
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Simply copy the |app| library folder from the old to the new computer. You can find out what the library folder is by clicking the calibre icon in the toolbar. The very first item is the path to the library folder. Now on the new computer, start |app| for the first time. It will run the Welcome Wizard asking you for the location of the |app| library. Point it to the previously copied folder. If the computer you are transferring too already has a calibre installation, then the Welcome wizard wont run. In that case, click the calibre icon in the tooolbar and point it to the newly copied directory. You will now have two calibre libraries on your computer and you can switch between them by clicking the calibre icon on the toolbar.
+Simply copy the |app| library folder from the old to the new computer. You can find out what the library folder is by clicking the calibre icon in the toolbar. The very first item is the path to the library folder. Now on the new computer, start |app| for the first time. It will run the Welcome Wizard asking you for the location of the |app| library. Point it to the previously copied folder. If the computer you are transferring to already has a calibre installation, then the Welcome wizard wont run. In that case, click the calibre icon in the tooolbar and point it to the newly copied directory. You will now have two calibre libraries on your computer and you can switch between them by clicking the calibre icon on the toolbar.
Note that if you are transferring between different types of computers (for example Windows to OS X) then after doing the above you should also go to :guilabel:`Preferences->Advanced->Miscellaneous` and click the "Check database integrity button". It will warn you about missing files, if any, which you should then transfer by hand.
From cca39d2e730a2877a753747ba2b4fd93a9b6384f Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Tue, 28 Sep 2010 13:11:55 +0100
Subject: [PATCH 169/207] Small cleanups for messages and name
---
src/calibre/customize/builtins.py | 2 +-
src/calibre/gui2/preferences/plugboard.py | 3 ++-
2 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py
index 89c800afb2..cf6995d3bb 100644
--- a/src/calibre/customize/builtins.py
+++ b/src/calibre/customize/builtins.py
@@ -799,7 +799,7 @@ class Sending(PreferencesPlugin):
class Plugboard(PreferencesPlugin):
name = 'Plugboard'
icon = I('plugboard.png')
- gui_name = _('Metadata plugboard')
+ gui_name = _('Metadata plugboards')
category = 'Import/Export'
gui_category = _('Import/Export')
category_order = 3
diff --git a/src/calibre/gui2/preferences/plugboard.py b/src/calibre/gui2/preferences/plugboard.py
index b723fb938c..124654b643 100644
--- a/src/calibre/gui2/preferences/plugboard.py
+++ b/src/calibre/gui2/preferences/plugboard.py
@@ -197,7 +197,8 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
if error:
error_dialog(self, '',
- _('That format and device already has a plugboard'),
+ _('That format and device already has a plugboard or '
+ 'conflicts with another plugboard.'),
show=True)
self.new_device.setCurrentIndex(0)
return
From 8a94c2194eada809060d3918d501f7029c0947ff Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Tue, 28 Sep 2010 15:13:30 +0100
Subject: [PATCH 170/207] Fix mutually recursive fields in save_to_disk. Fix
mistake in any_format template handling in save_to_disk
---
src/calibre/library/save_to_disk.py | 23 +++++++++++++++++++----
1 file changed, 19 insertions(+), 4 deletions(-)
diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py
index 5465150797..a2c8a62694 100644
--- a/src/calibre/library/save_to_disk.py
+++ b/src/calibre/library/save_to_disk.py
@@ -111,18 +111,31 @@ class SafeFormat(TemplateFormatter):
'''
Provides a format function that substitutes '' for any missing value
'''
+
+ composite_values = {}
+
def get_value(self, key, args, kwargs):
try:
b = self.book.get_user_metadata(key, False)
key = key.lower()
if b is not None and b['datatype'] == 'composite':
- return self.vformat(b['display']['composite_template'], [], kwargs)
+ if key in self.composite_values:
+ return self.composite_values[key]
+ self.composite_values[key] = 'RECURSIVE_COMPOSITE FIELD (S2D) ' + key
+ self.composite_values[key] = \
+ self.vformat(b['display']['composite_template'], [], kwargs)
+ return self.composite_values[key]
if kwargs[key]:
return self.sanitize(kwargs[key.lower()])
return ''
except:
return ''
+ def safe_format(self, fmt, kwargs, error_value, book, sanitize=None):
+ self.composite_values = {}
+ return TemplateFormatter.safe_format(self, fmt, kwargs, error_value,
+ book, sanitize)
+
safe_formatter = SafeFormat()
def get_components(template, mi, id, timefmt='%b %Y', length=250,
@@ -243,10 +256,12 @@ def save_book_to_disk(id, db, root, opts, length):
cpb = None
if fmt in plugboards:
cpb = plugboards[fmt]
- elif plugboard_any_format_value in plugboards:
+ if dev_name in cpb:
+ cpb = cpb[dev_name]
+ else:
+ cpb = None
+ if cpb is None and plugboard_any_format_value in plugboards:
cpb = plugboards[plugboard_any_format_value]
- # must find a save_to_disk entry for this format
- if cpb is not None:
if dev_name in cpb:
cpb = cpb[dev_name]
else:
From 96bc9f6bec337c2d551a0171f42ca9759d715326 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Tue, 28 Sep 2010 16:55:22 -0600
Subject: [PATCH 171/207] Stop metadata backup thread before bulk metadata
edits to improve performance
---
src/calibre/gui2/actions/edit_metadata.py | 2 +-
src/calibre/gui2/dialogs/metadata_bulk.py | 20 ++++++++++++++------
src/calibre/gui2/library/models.py | 19 +++++++++++++------
src/calibre/gui2/preferences/misc.py | 13 ++++++-------
4 files changed, 34 insertions(+), 20 deletions(-)
diff --git a/src/calibre/gui2/actions/edit_metadata.py b/src/calibre/gui2/actions/edit_metadata.py
index bd9728989b..cc74b3c515 100644
--- a/src/calibre/gui2/actions/edit_metadata.py
+++ b/src/calibre/gui2/actions/edit_metadata.py
@@ -184,7 +184,7 @@ class EditMetadataAction(InterfaceAction):
self.gui.tags_view.blockSignals(True)
try:
changed = MetadataBulkDialog(self.gui, rows,
- self.gui.library_view.model().db).changed
+ self.gui.library_view.model()).changed
finally:
self.gui.tags_view.blockSignals(False)
if changed:
diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py
index 9c83b3aee5..b0ce0a1e6d 100644
--- a/src/calibre/gui2/dialogs/metadata_bulk.py
+++ b/src/calibre/gui2/dialogs/metadata_bulk.py
@@ -142,12 +142,13 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
_('Append to field'),
]
- def __init__(self, window, rows, db):
+ def __init__(self, window, rows, model):
QDialog.__init__(self, window)
Ui_MetadataBulkDialog.__init__(self)
self.setupUi(self)
- self.db = db
- self.ids = [db.id(r) for r in rows]
+ self.model = model
+ self.db = model.db
+ self.ids = [self.db.id(r) for r in rows]
self.box_title.setText('
' +
_('Editing meta information for %d books') %
len(rows))
@@ -170,7 +171,7 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
self.tag_editor_button.clicked.connect(self.tag_editor)
self.autonumber_series.stateChanged[int].connect(self.auto_number_changed)
- if len(db.custom_field_keys(include_composites=False)) == 0:
+ if len(self.db.custom_field_keys(include_composites=False)) == 0:
self.central_widget.removeTab(1)
else:
self.create_custom_column_editors()
@@ -617,8 +618,15 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
self.worker = Worker(args, self.db, self.ids,
getattr(self, 'custom_column_widgets', []),
Dispatcher(bb.accept, parent=bb))
- self.worker.start()
- bb.exec_()
+
+ # The metadata backup thread causes database commits
+ # which can slow down bulk editing of large numbers of books
+ self.model.stop_metadata_backup()
+ try:
+ self.worker.start()
+ bb.exec_()
+ finally:
+ self.model.start_metadata_backup()
if self.worker.error is not None:
return error_dialog(self, _('Failed'),
diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index b2a7f08055..9da5420681 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -159,17 +159,24 @@ class BooksModel(QAbstractTableModel): # {{{
# do something on the GUI thread. Deadlock.
self.cover_cache = CoverCache(db, FunctionDispatcher(self.db.cover))
self.cover_cache.start()
- if self.metadata_backup is not None:
- self.metadata_backup.stop()
- # Would like to to a join here, but the thread might be waiting to
- # do something on the GUI thread. Deadlock.
- self.metadata_backup = MetadataBackup(db)
- self.metadata_backup.start()
+ self.stop_metadata_backup()
+ self.start_metadata_backup()
def refresh_cover(event, ids):
if event == 'cover' and self.cover_cache is not None:
self.cover_cache.refresh(ids)
db.add_listener(refresh_cover)
+ def start_metadata_backup(self):
+ self.metadata_backup = MetadataBackup(self.db)
+ self.metadata_backup.start()
+
+ def stop_metadata_backup(self):
+ if getattr(self, 'metadata_backup', None) is not None:
+ self.metadata_backup.stop()
+ # Would like to to a join here, but the thread might be waiting to
+ # do something on the GUI thread. Deadlock.
+
+
def refresh_ids(self, ids, current_row=-1):
rows = self.db.refresh_ids(ids)
if rows:
diff --git a/src/calibre/gui2/preferences/misc.py b/src/calibre/gui2/preferences/misc.py
index 865115c2ed..582d110c6c 100644
--- a/src/calibre/gui2/preferences/misc.py
+++ b/src/calibre/gui2/preferences/misc.py
@@ -106,14 +106,13 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
d.exec_()
def compact(self, *args):
- from calibre.library.caches import MetadataBackup
m = self.gui.library_view.model()
- if m.metadata_backup is not None:
- m.metadata_backup.stop()
- d = CheckIntegrity(m.db, self)
- d.exec_()
- m.metadata_backup = MetadataBackup(m.db)
- m.metadata_backup.start()
+ m.stop_metadata_backup()
+ try:
+ d = CheckIntegrity(m.db, self)
+ d.exec_()
+ finally:
+ m.start_metadata_backup()
def open_config_dir(self, *args):
from calibre.utils.config import config_dir
From fef738c53b8d5a980423d1930e6a94d4ffc8a6a8 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Tue, 28 Sep 2010 18:08:54 -0600
Subject: [PATCH 172/207] ...
---
src/calibre/manual/faq.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/calibre/manual/faq.rst b/src/calibre/manual/faq.rst
index c9f6abe2c0..3cf171bc1b 100644
--- a/src/calibre/manual/faq.rst
+++ b/src/calibre/manual/faq.rst
@@ -289,7 +289,7 @@ Yes, you can. Follow the instructions in the answer above for adding custom colu
How do I move my |app| library from one computer to another?
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Simply copy the |app| library folder from the old to the new computer. You can find out what the library folder is by clicking the calibre icon in the toolbar. The very first item is the path to the library folder. Now on the new computer, start |app| for the first time. It will run the Welcome Wizard asking you for the location of the |app| library. Point it to the previously copied folder. If the computer you are transferring too already has a calibre installation, then the Welcome wizard wont run. In that case, click the calibre icon in the tooolbar and point it to the newly copied directory. You will now have two calibre libraries on your computer and you can switch between them by clicking the calibre icon on the toolbar.
+Simply copy the |app| library folder from the old to the new computer. You can find out what the library folder is by clicking the calibre icon in the toolbar. The very first item is the path to the library folder. Now on the new computer, start |app| for the first time. It will run the Welcome Wizard asking you for the location of the |app| library. Point it to the previously copied folder. If the computer you are transferring to already has a calibre installation, then the Welcome wizard wont run. In that case, click the calibre icon in the tooolbar and point it to the newly copied directory. You will now have two calibre libraries on your computer and you can switch between them by clicking the calibre icon on the toolbar.
Note that if you are transferring between different types of computers (for example Windows to OS X) then after doing the above you should also go to :guilabel:`Preferences->Advanced->Miscellaneous` and click the "Check database integrity button". It will warn you about missing files, if any, which you should then transfer by hand.
From ca0e8729d20c1c857254186b9f658edfdbaf0c5e Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Tue, 28 Sep 2010 18:15:31 -0600
Subject: [PATCH 173/207] Handle formatting of recursive compisite templates
---
src/calibre/library/save_to_disk.py | 15 ++++++++++++++-
1 file changed, 14 insertions(+), 1 deletion(-)
diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py
index e479d27121..088b6352af 100644
--- a/src/calibre/library/save_to_disk.py
+++ b/src/calibre/library/save_to_disk.py
@@ -106,18 +106,31 @@ class SafeFormat(TemplateFormatter):
'''
Provides a format function that substitutes '' for any missing value
'''
+
+ composite_values = {}
+
def get_value(self, key, args, kwargs):
try:
b = self.book.get_user_metadata(key, False)
key = key.lower()
if b is not None and b['datatype'] == 'composite':
- return self.vformat(b['display']['composite_template'], [], kwargs)
+ if key in self.composite_values:
+ return self.composite_values[key]
+ self.composite_values[key] = 'RECURSIVE_COMPOSITE FIELD (S2D) ' + key
+ self.composite_values[key] = \
+ self.vformat(b['display']['composite_template'], [], kwargs)
+ return self.composite_values[key]
if kwargs[key]:
return self.sanitize(kwargs[key.lower()])
return ''
except:
return ''
+ def safe_format(self, fmt, kwargs, error_value, book, sanitize=None):
+ self.composite_values = {}
+ return TemplateFormatter.safe_format(self, fmt, kwargs, error_value,
+ book, sanitize)
+
safe_formatter = SafeFormat()
def get_components(template, mi, id, timefmt='%b %Y', length=250,
From c8477338a68b55a5fb92af145a516785a96ab273 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Tue, 28 Sep 2010 18:21:52 -0600
Subject: [PATCH 174/207] Do not have the fetch news dialog close when the user
presses Enter
---
src/calibre/gui2/dialogs/scheduler.py | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/src/calibre/gui2/dialogs/scheduler.py b/src/calibre/gui2/dialogs/scheduler.py
index fd8184933f..30f4a2d8a2 100644
--- a/src/calibre/gui2/dialogs/scheduler.py
+++ b/src/calibre/gui2/dialogs/scheduler.py
@@ -57,6 +57,10 @@ class SchedulerDialog(QDialog, Ui_Dialog):
self.old_news.setValue(gconf['oldest_news'])
+ def keyPressEvent(self, ev):
+ if ev.key() not in (Qt.Key_Enter, Qt.Key_Return):
+ return QDialog.keyPressEvent(self, ev)
+
def break_cycles(self):
self.disconnect(self.recipe_model, SIGNAL('searched(PyQt_PyObject)'),
self.search_done)
From d3053b8a8612d24aec9c81dafd5567defdb37a50 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Tue, 28 Sep 2010 18:45:41 -0600
Subject: [PATCH 175/207] Support for the JetBook Mini
---
src/calibre/customize/builtins.py | 3 ++-
src/calibre/devices/__init__.py | 12 +++++++++---
src/calibre/devices/jetbook/driver.py | 23 +++++++++++++++++++++++
src/calibre/gui2/wizard/__init__.py | 8 ++++++++
4 files changed, 42 insertions(+), 4 deletions(-)
diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py
index ef3da9ce20..50d8e29373 100644
--- a/src/calibre/customize/builtins.py
+++ b/src/calibre/customize/builtins.py
@@ -446,7 +446,7 @@ from calibre.devices.eb600.driver import EB600, COOL_ER, SHINEBOOK, \
BOOQ, ELONEX, POCKETBOOK301, MENTOR
from calibre.devices.iliad.driver import ILIAD
from calibre.devices.irexdr.driver import IREXDR1000, IREXDR800
-from calibre.devices.jetbook.driver import JETBOOK, MIBUK
+from calibre.devices.jetbook.driver import JETBOOK, MIBUK, JETBOOK_MINI
from calibre.devices.kindle.driver import KINDLE, KINDLE2, KINDLE_DX
from calibre.devices.nook.driver import NOOK
from calibre.devices.prs505.driver import PRS505
@@ -520,6 +520,7 @@ plugins += [
IREXDR1000,
IREXDR800,
JETBOOK,
+ JETBOOK_MINI,
MIBUK,
SHINEBOOK,
POCKETBOOK360,
diff --git a/src/calibre/devices/__init__.py b/src/calibre/devices/__init__.py
index 956d18e903..24e606e022 100644
--- a/src/calibre/devices/__init__.py
+++ b/src/calibre/devices/__init__.py
@@ -95,13 +95,19 @@ def debug(ioreg_to_tmp=False, buf=None):
ioreg += 'Output from osx_get_usb_drives:\n'+drives+'\n\n'
ioreg += Device.run_ioreg()
connected_devices = []
- for dev in sorted(device_plugins(), cmp=lambda
- x,y:cmp(x.__class__.__name__, y.__class__.__name__)):
- out('Looking for', dev.__class__.__name__)
+ devplugins = list(sorted(device_plugins(), cmp=lambda
+ x,y:cmp(x.__class__.__name__, y.__class__.__name__)))
+ out('Available plugins:', ' '.join([x.__class__.__name__ for x in
+ devplugins]))
+ out(' ')
+ out('Looking for devices...')
+ for dev in devplugins:
connected, det = s.is_device_connected(dev, debug=True)
if connected:
+ out('\t\tDetected possible device', dev.__class__.__name__)
connected_devices.append((dev, det))
+ out(' ')
errors = {}
success = False
out('Devices possibly connected:', end=' ')
diff --git a/src/calibre/devices/jetbook/driver.py b/src/calibre/devices/jetbook/driver.py
index 6ee1c07464..5fd3929aaf 100644
--- a/src/calibre/devices/jetbook/driver.py
+++ b/src/calibre/devices/jetbook/driver.py
@@ -99,4 +99,27 @@ class MIBUK(USBMS):
VENDOR_NAME = 'LINUX'
WINDOWS_MAIN_MEM = 'WOLDERMIBUK'
+class JETBOOK_MINI(USBMS):
+
+ '''
+ ['0x4b8',
+ '0x507',
+ '0x100',
+ 'ECTACO',
+ 'ECTACO ATA/ATAPI Bridge (Bulk-Only)',
+ 'Rev.0.20']
+ '''
+ FORMATS = ['fb2', 'txt']
+
+ name = 'JetBook Mini'
+ description = _('Communicate with the JetBook Mini reader.')
+ author = 'Kovid Goyal'
+
+ VENDOR_ID = [0x4b8]
+ PRODUCT_ID = [0x507]
+ BCD = [0x100]
+ VENDOR_NAME = 'ECTACO'
+ WINDOWS_MAIN_MEM = '' # Matches PROD_
+ SUPPORTS_SUB_DIRS = True
+
diff --git a/src/calibre/gui2/wizard/__init__.py b/src/calibre/gui2/wizard/__init__.py
index ef58ec3a90..37b7c7bd7c 100644
--- a/src/calibre/gui2/wizard/__init__.py
+++ b/src/calibre/gui2/wizard/__init__.py
@@ -73,6 +73,14 @@ class JetBook(Device):
manufacturer = 'Ectaco'
id = 'jetbook'
+class JetBookMini(Device):
+
+ output_profile = 'jetbook5'
+ output_format = 'FB2'
+ name = 'JetBook Mini'
+ manufacturer = 'Ectaco'
+ id = 'jetbookmini'
+
class KindleDX(Kindle):
output_profile = 'kindle_dx'
From fce4ab97b696ef3d2addb04643980266272b4380 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Tue, 28 Sep 2010 18:50:02 -0600
Subject: [PATCH 176/207] ...
---
src/calibre/devices/__init__.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/src/calibre/devices/__init__.py b/src/calibre/devices/__init__.py
index 24e606e022..1918a36cc8 100644
--- a/src/calibre/devices/__init__.py
+++ b/src/calibre/devices/__init__.py
@@ -56,6 +56,7 @@ def get_connected_device():
return dev
def debug(ioreg_to_tmp=False, buf=None):
+ import textwrap
from calibre.customize.ui import device_plugins
from calibre.devices.scanner import DeviceScanner, win_pnp_drives
from calibre.constants import iswindows, isosx, __version__
@@ -97,8 +98,8 @@ def debug(ioreg_to_tmp=False, buf=None):
connected_devices = []
devplugins = list(sorted(device_plugins(), cmp=lambda
x,y:cmp(x.__class__.__name__, y.__class__.__name__)))
- out('Available plugins:', ' '.join([x.__class__.__name__ for x in
- devplugins]))
+ out('Available plugins:', textwrap.fill(' '.join([x.__class__.__name__ for x in
+ devplugins])))
out(' ')
out('Looking for devices...')
for dev in devplugins:
From 1c9335aa5ec10a1bc2dba97bed55513c9550669f Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Tue, 28 Sep 2010 19:03:54 -0600
Subject: [PATCH 177/207] Fix regression that caused the filename to not be set
as the title when reading metadata fails
---
src/calibre/ebooks/metadata/meta.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/calibre/ebooks/metadata/meta.py b/src/calibre/ebooks/metadata/meta.py
index 68deca5e10..b02ae2dbff 100644
--- a/src/calibre/ebooks/metadata/meta.py
+++ b/src/calibre/ebooks/metadata/meta.py
@@ -181,7 +181,7 @@ def metadata_from_filename(name, pat=None):
mi.isbn = si
except (IndexError, ValueError):
pass
- if not mi.title:
+ if mi.is_null('title'):
mi.title = name
return mi
From 3018b6ac7c4f7b026d5ca847734653faa1e4d0b7 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Tue, 28 Sep 2010 19:07:08 -0600
Subject: [PATCH 178/207] ...
---
src/calibre/devices/jetbook/driver.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/src/calibre/devices/jetbook/driver.py b/src/calibre/devices/jetbook/driver.py
index 5fd3929aaf..f108de3347 100644
--- a/src/calibre/devices/jetbook/driver.py
+++ b/src/calibre/devices/jetbook/driver.py
@@ -111,7 +111,8 @@ class JETBOOK_MINI(USBMS):
'''
FORMATS = ['fb2', 'txt']
- name = 'JetBook Mini'
+ gui_name = 'JetBook Mini'
+ name = 'JetBook Mini Device Interface'
description = _('Communicate with the JetBook Mini reader.')
author = 'Kovid Goyal'
@@ -120,6 +121,8 @@ class JETBOOK_MINI(USBMS):
BCD = [0x100]
VENDOR_NAME = 'ECTACO'
WINDOWS_MAIN_MEM = '' # Matches PROD_
+ MAIN_MEMORY_VOLUME_LABEL = 'Jetbook Mini'
+
SUPPORTS_SUB_DIRS = True
From 700dbe7df7fb8b43b6392870aaaec900f0a234e8 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Wed, 29 Sep 2010 12:51:18 +0100
Subject: [PATCH 179/207] 1) add dirtied when renaming items 2) make bulk edit
use the GUI thread 3) add a 'books remaiing' menu item
---
src/calibre/gui2/actions/choose_library.py | 21 +-
src/calibre/gui2/dialogs/metadata_bulk.py | 241 +++++++++++++--------
src/calibre/library/caches.py | 38 ++--
src/calibre/library/custom_columns.py | 2 +
src/calibre/library/database2.py | 48 +++-
5 files changed, 236 insertions(+), 114 deletions(-)
diff --git a/src/calibre/gui2/actions/choose_library.py b/src/calibre/gui2/actions/choose_library.py
index 79406da40c..d3045fecf4 100644
--- a/src/calibre/gui2/actions/choose_library.py
+++ b/src/calibre/gui2/actions/choose_library.py
@@ -14,7 +14,7 @@ from calibre import isbytestring
from calibre.constants import filesystem_encoding
from calibre.utils.config import prefs
from calibre.gui2 import gprefs, warning_dialog, Dispatcher, error_dialog, \
- question_dialog
+ question_dialog, info_dialog
from calibre.gui2.actions import InterfaceAction
class LibraryUsageStats(object):
@@ -115,6 +115,14 @@ class ChooseLibraryAction(InterfaceAction):
type=Qt.QueuedConnection)
self.choose_menu.addAction(ac)
+ self.rename_separator = self.choose_menu.addSeparator()
+
+ self.create_action(spec=(_('Library backup status...'), 'lt.png', None,
+ None), attr='action_backup_status')
+ self.action_backup_status.triggered.connect(self.backup_status,
+ type=Qt.QueuedConnection)
+ self.choose_menu.addAction(self.action_backup_status)
+
def library_name(self):
db = self.gui.library_view.model().db
path = db.library_path
@@ -206,6 +214,17 @@ class ChooseLibraryAction(InterfaceAction):
self.stats.remove(location)
self.build_menus()
+ def backup_status(self, location):
+ dirty_text = 'no'
+ try:
+ print 'here'
+ dirty_text = \
+ unicode(self.gui.library_view.model().db.dirty_queue_length())
+ except:
+ dirty_text = _('none')
+ info_dialog(self.gui, _('Backup status'), '
'+
+ _('Book metadata files remaining to be written: %s') % dirty_text,
+ show=True)
def switch_requested(self, location):
if not self.change_library_allowed():
diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py
index b0ce0a1e6d..4fc85f2b30 100644
--- a/src/calibre/gui2/dialogs/metadata_bulk.py
+++ b/src/calibre/gui2/dialogs/metadata_bulk.py
@@ -3,42 +3,109 @@ __copyright__ = '2008, Kovid Goyal '
'''Dialog to edit metadata in bulk'''
-from threading import Thread
-import re, string
+import re
-from PyQt4.Qt import Qt, QDialog, QGridLayout
+from PyQt4.Qt import Qt, QDialog, QGridLayout, QVBoxLayout, QFont, QLabel, \
+ pyqtSignal
from PyQt4 import QtGui
from calibre.gui2.dialogs.metadata_bulk_ui import Ui_MetadataBulkDialog
from calibre.gui2.dialogs.tag_editor import TagEditor
from calibre.ebooks.metadata import string_to_authors, authors_to_string
from calibre.gui2.custom_column_widgets import populate_metadata_page
-from calibre.gui2.dialogs.progress import BlockingBusy
-from calibre.gui2 import error_dialog, Dispatcher
+from calibre.gui2 import error_dialog
+from calibre.gui2.progress_indicator import ProgressIndicator
from calibre.utils.config import dynamic
-class Worker(Thread):
+class MyBlockingBusy(QDialog):
+
+ do_one_signal = pyqtSignal()
+
+ phases = ['',
+ _('Title/Author'),
+ _('Standard metadata'),
+ _('Custom metadata'),
+ _('Search/Replace'),
+ ]
+
+ def __init__(self, msg, args, db, ids, cc_widgets, s_r_func,
+ parent=None, window_title=_('Working')):
+ QDialog.__init__(self, parent)
+
+ self._layout = QVBoxLayout()
+ self.setLayout(self._layout)
+ self.msg_text = msg
+ self.msg = QLabel(msg+' ') # Ensure dialog is wide enough
+ #self.msg.setWordWrap(True)
+ self.font = QFont()
+ self.font.setPointSize(self.font.pointSize() + 8)
+ self.msg.setFont(self.font)
+ self.pi = ProgressIndicator(self)
+ self.pi.setDisplaySize(100)
+ self._layout.addWidget(self.pi, 0, Qt.AlignHCenter)
+ self._layout.addSpacing(15)
+ self._layout.addWidget(self.msg, 0, Qt.AlignHCenter)
+ self.setWindowTitle(window_title)
+ self.resize(self.sizeHint())
+ self.start()
- def __init__(self, args, db, ids, cc_widgets, callback):
- Thread.__init__(self)
self.args = args
self.db = db
self.ids = ids
self.error = None
- self.callback = callback
self.cc_widgets = cc_widgets
+ self.s_r_func = s_r_func
+ self.do_one_signal.connect(self.do_one_safe, Qt.QueuedConnection)
- def doit(self):
+ def start(self):
+ self.pi.startAnimation()
+
+ def stop(self):
+ self.pi.stopAnimation()
+
+ def accept(self):
+ self.stop()
+ return QDialog.accept(self)
+
+ def exec_(self):
+ self.current_index = 0
+ self.current_phase = 1
+ self.do_one_signal.emit()
+ return QDialog.exec_(self)
+
+ def do_one_safe(self):
+ try:
+ if self.current_index >= len(self.ids):
+ self.current_phase += 1
+ self.current_index = 0
+ if self.current_phase > 4:
+ self.db.commit()
+ return self.accept()
+ id = self.ids[self.current_index]
+ self.msg.setText(self.msg_text.format(self.phases[self.current_phase],
+ (self.current_index*100)/len(self.ids)))
+ self.do_one(id)
+ except Exception, err:
+ import traceback
+ try:
+ err = unicode(err)
+ except:
+ err = repr(err)
+ self.error = (err, traceback.format_exc())
+ return self.accept()
+
+ def do_one(self, id):
remove, add, au, aus, do_aus, rating, pub, do_series, \
do_autonumber, do_remove_format, remove_format, do_swap_ta, \
do_remove_conv, do_auto_author, series, do_series_restart, \
series_start_value, do_title_case, clear_series = self.args
+
# first loop: do author and title. These will commit at the end of each
# operation, because each operation modifies the file system. We want to
# try hard to keep the DB and the file system in sync, even in the face
# of exceptions or forced exits.
- for id in self.ids:
+ if self.current_phase == 1:
title_set = False
if do_swap_ta:
title = self.db.title(id, index_is_id=True)
@@ -58,9 +125,8 @@ class Worker(Thread):
self.db.set_title(id, title.title(), notify=False)
if au:
self.db.set_authors(id, string_to_authors(au), notify=False)
-
- # All of these just affect the DB, so we can tolerate a total rollback
- for id in self.ids:
+ elif self.current_phase == 2:
+ # All of these just affect the DB, so we can tolerate a total rollback
if do_auto_author:
x = self.db.author_sort_from_book(id, index_is_id=True)
if x:
@@ -93,37 +159,19 @@ class Worker(Thread):
if do_remove_conv:
self.db.delete_conversion_options(id, 'PIPE', commit=False)
- self.db.commit()
+ elif self.current_phase == 3:
+ # both of these are fast enough to just do them all
+ for w in self.cc_widgets:
+ w.commit(self.ids)
+ self.db.bulk_modify_tags(self.ids, add=add, remove=remove,
+ notify=False)
+ self.current_index = len(self.ids)
+ elif self.current_phase == 4:
+ self.s_r_func(id)
+ # do the next one
+ self.current_index += 1
+ self.do_one_signal.emit()
- for w in self.cc_widgets:
- w.commit(self.ids)
- self.db.bulk_modify_tags(self.ids, add=add, remove=remove,
- notify=False)
-
- def run(self):
- try:
- self.doit()
- except Exception, err:
- import traceback
- try:
- err = unicode(err)
- except:
- err = repr(err)
- self.error = (err, traceback.format_exc())
-
- self.callback()
-
-class SafeFormat(string.Formatter):
- '''
- Provides a format function that substitutes '' for any missing value
- '''
- def get_value(self, key, args, vals):
- v = vals.get(key, None)
- if v is None:
- return ''
- if isinstance(v, (tuple, list)):
- v = ','.join(v)
- return v
class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
@@ -452,7 +500,7 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
self.s_r_set_colors()
break
- def do_search_replace(self):
+ def do_search_replace(self, id):
source = unicode(self.search_field.currentText())
if not source or not self.s_r_obj:
return
@@ -461,48 +509,45 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
dest = source
dfm = self.db.field_metadata[dest]
- for id in self.ids:
- mi = self.db.get_metadata(id, index_is_id=True,)
- val = mi.get(source)
- if val is None:
- continue
- val = self.s_r_do_regexp(mi)
- val = self.s_r_do_destination(mi, val)
- if dfm['is_multiple']:
- if dfm['is_custom']:
- # The standard tags and authors values want to be lists.
- # All custom columns are to be strings
- val = dfm['is_multiple'].join(val)
- if dest == 'authors' and len(val) == 0:
- error_dialog(self, _('Search/replace invalid'),
- _('Authors cannot be set to the empty string. '
- 'Book title %s not processed')%mi.title,
- show=True)
- continue
- else:
- val = self.s_r_replace_mode_separator().join(val)
- if dest == 'title' and len(val) == 0:
- error_dialog(self, _('Search/replace invalid'),
- _('Title cannot be set to the empty string. '
- 'Book title %s not processed')%mi.title,
- show=True)
- continue
-
+ mi = self.db.get_metadata(id, index_is_id=True,)
+ val = mi.get(source)
+ if val is None:
+ return
+ val = self.s_r_do_regexp(mi)
+ val = self.s_r_do_destination(mi, val)
+ if dfm['is_multiple']:
if dfm['is_custom']:
- extra = self.db.get_custom_extra(id, label=dfm['label'], index_is_id=True)
- self.db.set_custom(id, val, label=dfm['label'], extra=extra,
- commit=False)
+ # The standard tags and authors values want to be lists.
+ # All custom columns are to be strings
+ val = dfm['is_multiple'].join(val)
+ if dest == 'authors' and len(val) == 0:
+ error_dialog(self, _('Search/replace invalid'),
+ _('Authors cannot be set to the empty string. '
+ 'Book title %s not processed')%mi.title,
+ show=True)
+ return
+ else:
+ val = self.s_r_replace_mode_separator().join(val)
+ if dest == 'title' and len(val) == 0:
+ error_dialog(self, _('Search/replace invalid'),
+ _('Title cannot be set to the empty string. '
+ 'Book title %s not processed')%mi.title,
+ show=True)
+ return
+
+ if dfm['is_custom']:
+ extra = self.db.get_custom_extra(id, label=dfm['label'], index_is_id=True)
+ self.db.set_custom(id, val, label=dfm['label'], extra=extra,
+ commit=False)
+ else:
+ if dest == 'comments':
+ setter = self.db.set_comment
else:
- if dest == 'comments':
- setter = self.db.set_comment
- else:
- setter = getattr(self.db, 'set_'+dest)
- if dest in ['title', 'authors']:
- setter(id, val, notify=False)
- else:
- setter(id, val, notify=False, commit=False)
- self.db.commit()
- dynamic['s_r_search_mode'] = self.search_mode.currentIndex()
+ setter = getattr(self.db, 'set_'+dest)
+ if dest in ['title', 'authors']:
+ setter(id, val, notify=False)
+ else:
+ setter(id, val, notify=False, commit=False)
def create_custom_column_editors(self):
w = self.central_widget.widget(1)
@@ -525,11 +570,11 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
def initalize_authors(self):
all_authors = self.db.all_authors()
- all_authors.sort(cmp=lambda x, y : cmp(x[1], y[1]))
+ all_authors.sort(cmp=lambda x, y : cmp(x[1].lower(), y[1].lower()))
for i in all_authors:
id, name = i
- name = authors_to_string([name.strip().replace('|', ',') for n in name.split(',')])
+ name = name.strip().replace('|', ',')
self.authors.addItem(name)
self.authors.setEditText('')
@@ -613,28 +658,32 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
do_remove_conv, do_auto_author, series, do_series_restart,
series_start_value, do_title_case, clear_series)
- bb = BlockingBusy(_('Applying changes to %d books. This may take a while.')
- %len(self.ids), parent=self)
- self.worker = Worker(args, self.db, self.ids,
+# bb = BlockingBusy(_('Applying changes to %d books. This may take a while.')
+# %len(self.ids), parent=self)
+# self.worker = Worker(args, self.db, self.ids,
+# getattr(self, 'custom_column_widgets', []),
+# Dispatcher(bb.accept, parent=bb))
+
+ bb = MyBlockingBusy(_('Applying changes to %d books.\nPhase {0} {1}%%.')
+ %len(self.ids), args, self.db, self.ids,
getattr(self, 'custom_column_widgets', []),
- Dispatcher(bb.accept, parent=bb))
+ self.do_search_replace, parent=self)
# The metadata backup thread causes database commits
# which can slow down bulk editing of large numbers of books
self.model.stop_metadata_backup()
try:
- self.worker.start()
+# self.worker.start()
bb.exec_()
finally:
self.model.start_metadata_backup()
- if self.worker.error is not None:
+ if bb.error is not None:
return error_dialog(self, _('Failed'),
- self.worker.error[0], det_msg=self.worker.error[1],
+ bb.error[0], det_msg=bb.error[1],
show=True)
- self.do_search_replace()
-
+ dynamic['s_r_search_mode'] = self.search_mode.currentIndex()
self.db.clean()
return QDialog.accept(self)
diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index 281d1485b7..a36dbe57a9 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -138,25 +138,37 @@ class CoverCache(Thread): # {{{
def run(self):
while self.keep_running:
try:
- time.sleep(0.050) # Limit 20/second to not overwhelm the GUI
+ # The GUI puts the same ID into the queue many times. The code
+ # below emptys the queue, building a set of unique values. When
+ # the queue is empty, do the work
+ ids = set()
id_ = self.load_queue.get(True, 2)
+ ids.add(id_)
+ try:
+ while True:
+ # Give the gui some time to put values into the queue
+ id_ = self.load_queue.get(True, 0.5)
+ ids.add(id_)
+ except Empty:
+ pass
except Empty:
continue
except:
#Happens during interpreter shutdown
break
- try:
- img = self._image_for_id(id_)
- except:
- import traceback
- traceback.print_exc()
- continue
- try:
- with self.lock:
- self.cache[id_] = img
- except:
- # Happens during interpreter shutdown
- break
+ for id_ in ids:
+ time.sleep(0.050) # Limit 20/second to not overwhelm the GUI
+ try:
+ img = self._image_for_id(id_)
+ except:
+ traceback.print_exc()
+ continue
+ try:
+ with self.lock:
+ self.cache[id_] = img
+ except:
+ # Happens during interpreter shutdown
+ break
def set_cache(self, ids):
with self.lock:
diff --git a/src/calibre/library/custom_columns.py b/src/calibre/library/custom_columns.py
index 97c8565177..fdd78e89f8 100644
--- a/src/calibre/library/custom_columns.py
+++ b/src/calibre/library/custom_columns.py
@@ -214,6 +214,7 @@ class CustomColumns(object):
'SELECT id FROM %s WHERE value=?'%table, (new_name,), all=False)
if new_id is None or old_id == new_id:
self.conn.execute('UPDATE %s SET value=? WHERE id=?'%table, (new_name, old_id))
+ new_id = old_id
else:
# New id exists. If the column is_multiple, then process like
# tags, otherwise process like publishers (see database2)
@@ -226,6 +227,7 @@ class CustomColumns(object):
self.conn.execute('''UPDATE %s SET value=?
WHERE value=?'''%lt, (new_id, old_id,))
self.conn.execute('DELETE FROM %s WHERE id=?'%table, (old_id,))
+ self.dirty_books_referencing('#'+data['label'], new_id, commit=False)
self.conn.commit()
def delete_custom_item_using_id(self, id, label=None, num=None):
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index 192de21df3..85fb955448 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -47,13 +47,21 @@ def delete_file(path):
def delete_tree(path, permanent=False):
if permanent:
- shutil.rmtree(path)
+ try:
+ # For completely mysterious reasons, sometimes a file is left open
+ # leading to access errors. If we get an exception, wait and hope
+ # that whatever has the file (the O/S?) lets go of it.
+ shutil.rmtree(path)
+ except:
+ traceback.print_exc()
+ time.sleep(1)
+ shutil.rmtree(path)
else:
try:
if not permanent:
winshell.delete_file(path, silent=True, no_confirm=True)
except:
- shutil.rmtree(path)
+ delete_tree(path, permanent=True)
copyfile = os.link if hasattr(os, 'link') else shutil.copyfile
@@ -520,6 +528,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
try:
f = open(path, 'rb')
except (IOError, OSError):
+ try:
+ f.close()
+ print 'cover exception left file open!', path
+ except:
+ pass
time.sleep(0.2)
f = open(path, 'rb')
if as_image:
@@ -627,6 +640,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
if commit:
self.conn.commit()
+ def dirty_queue_length(self):
+ return len(self.dirtied_cache)
+
def commit_dirty_cache(self):
'''
Set the dirty indication for every book in the cache. The vast majority
@@ -1286,7 +1302,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
val=mi.get(key),
extra=mi.get_extra(key),
label=user_mi[key]['label'], commit=False)
- self.commit()
+ self.conn.commit()
self.notify('metadata', [id])
def authors_sort_strings(self, id, index_is_id=False):
@@ -1444,6 +1460,19 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
# Convenience methods for tags_list_editor
# Note: we generally do not need to refresh_ids because library_view will
# refresh everything.
+
+ def dirty_books_referencing(self, field, id, commit=True):
+ # Get the list of books to dirty -- all books that reference the item
+ table = self.field_metadata[field]['table']
+ link = self.field_metadata[field]['link_column']
+ bks = self.conn.get(
+ 'SELECT book from books_{0}_link WHERE {1}=?'.format(table, link),
+ (id,))
+ books = []
+ for (book_id,) in bks:
+ books.append(book_id)
+ self.dirtied(books, commit=commit)
+
def get_tags_with_ids(self):
result = self.conn.get('SELECT id,name FROM tags')
if not result:
@@ -1460,6 +1489,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
# there is a change of case
self.conn.execute('''UPDATE tags SET name=?
WHERE id=?''', (new_name, old_id))
+ self.dirty_books_referencing('tags', new_id, commit=False)
+ new_id = old_id
else:
# It is possible that by renaming a tag, the tag will appear
# twice on a book. This will throw an integrity error, aborting
@@ -1477,9 +1508,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
WHERE tag=?''',(new_id, old_id,))
# Get rid of the no-longer used publisher
self.conn.execute('DELETE FROM tags WHERE id=?', (old_id,))
+ self.dirty_books_referencing('tags', new_id, commit=False)
self.conn.commit()
def delete_tag_using_id(self, id):
+ self.dirty_books_referencing('tags', id, commit=False)
self.conn.execute('DELETE FROM books_tags_link WHERE tag=?', (id,))
self.conn.execute('DELETE FROM tags WHERE id=?', (id,))
self.conn.commit()
@@ -1496,6 +1529,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
'''SELECT id from series
WHERE name=?''', (new_name,), all=False)
if new_id is None or old_id == new_id:
+ new_id = old_id
self.conn.execute('UPDATE series SET name=? WHERE id=?',
(new_name, old_id))
else:
@@ -1519,15 +1553,17 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
SET series_index=?
WHERE id=?''',(index, book_id,))
index = index + 1
+ self.dirty_books_referencing('series', new_id, commit=False)
self.conn.commit()
def delete_series_using_id(self, id):
+ self.dirty_books_referencing('series', id, commit=False)
books = self.conn.get('SELECT book from books_series_link WHERE series=?', (id,))
self.conn.execute('DELETE FROM books_series_link WHERE series=?', (id,))
self.conn.execute('DELETE FROM series WHERE id=?', (id,))
- self.conn.commit()
for (book_id,) in books:
self.conn.execute('UPDATE books SET series_index=1.0 WHERE id=?', (book_id,))
+ self.conn.commit()
def get_publishers_with_ids(self):
result = self.conn.get('SELECT id,name FROM publishers')
@@ -1541,6 +1577,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
'''SELECT id from publishers
WHERE name=?''', (new_name,), all=False)
if new_id is None or old_id == new_id:
+ new_id = old_id
# New name doesn't exist. Simply change the old name
self.conn.execute('UPDATE publishers SET name=? WHERE id=?', \
(new_name, old_id))
@@ -1551,9 +1588,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
WHERE publisher=?''',(new_id, old_id,))
# Get rid of the no-longer used publisher
self.conn.execute('DELETE FROM publishers WHERE id=?', (old_id,))
+ self.dirty_books_referencing('publisher', new_id, commit=False)
self.conn.commit()
def delete_publisher_using_id(self, old_id):
+ self.dirty_books_referencing('publisher', id, commit=False)
self.conn.execute('''DELETE FROM books_publishers_link
WHERE publisher=?''', (old_id,))
self.conn.execute('DELETE FROM publishers WHERE id=?', (old_id,))
@@ -1634,6 +1673,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
# Now delete the old author from the DB
bks = self.conn.get('SELECT book FROM books_authors_link WHERE author=?', (old_id,))
self.conn.execute('DELETE FROM authors WHERE id=?', (old_id,))
+ self.dirtied(books, commit=False)
self.conn.commit()
# the authors are now changed, either by changing the author's name
# or replacing the author in the list. Now must fix up the books.
From 5aadbb2dcd311272f9230b1b9149ed4833c6ff47 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Wed, 29 Sep 2010 14:33:11 +0100
Subject: [PATCH 180/207] 1) change plugboards to templates (pass one) 2) fix
recursion detection in base.py 3) fix lack of refresh in model when editing
custom fields on the GUI 4) change the name of the plugboard eval function in
base.py 5) move recursion detection base code to formatter
---
src/calibre/ebooks/metadata/book/base.py | 55 +++++++++++------------
src/calibre/gui2/device.py | 2 +-
src/calibre/gui2/library/models.py | 4 +-
src/calibre/gui2/preferences/plugboard.py | 36 +++++++--------
src/calibre/gui2/preferences/plugboard.ui | 5 ++-
src/calibre/library/caches.py | 4 +-
src/calibre/library/save_to_disk.py | 9 +---
src/calibre/utils/formatter.py | 5 +++
8 files changed, 59 insertions(+), 61 deletions(-)
diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index 951a55da10..56df573cee 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -37,7 +37,13 @@ class SafeFormat(TemplateFormatter):
def get_value(self, key, args, kwargs):
try:
- ign, v = self.book.format_field(key.lower(), series_with_index=False)
+ b = self.book.get_user_metadata(key, False)
+ if b and b['datatype'] == 'int' and self.book.get(key, 0) == 0:
+ v = ''
+ elif b and b['datatype'] == 'float' and b.get(key, 0.0) == 0.0:
+ v = ''
+ else:
+ ign, v = self.book.format_field(key.lower(), series_with_index=False)
if v is None:
return ''
if v == '':
@@ -65,7 +71,6 @@ class Metadata(object):
'''
_data = copy.deepcopy(NULL_VALUES)
object.__setattr__(self, '_data', _data)
- _data['_curseq'] = _data['_compseq'] = 0
if other is not None:
self.smart_update(other)
else:
@@ -94,29 +99,22 @@ class Metadata(object):
if field in _data['user_metadata'].iterkeys():
d = _data['user_metadata'][field]
val = d['#value#']
- if d['datatype'] != 'composite' or \
- (_data['_curseq'] == _data['_compseq'] and val is not None):
+ if d['datatype'] != 'composite':
return val
- # Data in the structure has changed. Recompute the composite fields
- _data['_compseq'] = _data['_curseq']
- for ck in _data['user_metadata']:
- cf = _data['user_metadata'][ck]
- if cf['datatype'] != 'composite':
- continue
- cf['#value#'] = 'RECURSIVE_COMPOSITE FIELD ' + field
- cf['#value#'] = composite_formatter.safe_format(
- cf['display']['composite_template'],
+ if val is None:
+ d['#value#'] = 'RECURSIVE_COMPOSITE FIELD (Metadata) ' + field
+ val = d['#value#'] = composite_formatter.safe_format(
+ d['display']['composite_template'],
self,
_('TEMPLATE ERROR'),
self).strip()
- return d['#value#']
+ return val
raise AttributeError(
'Metadata object has no attribute named: '+ repr(field))
def __setattr__(self, field, val, extra=None):
_data = object.__getattribute__(self, '_data')
- _data['_curseq'] += 1
if field in TOP_LEVEL_CLASSIFIERS:
_data['classifiers'].update({field: val})
elif field in STANDARD_METADATA_FIELDS:
@@ -124,7 +122,10 @@ class Metadata(object):
val = NULL_VALUES.get(field, None)
_data[field] = val
elif field in _data['user_metadata'].iterkeys():
- _data['user_metadata'][field]['#value#'] = val
+ if _data['user_metadata'][field]['datatype'] == 'composite':
+ _data['user_metadata'][field]['#value#'] = None
+ else:
+ _data['user_metadata'][field]['#value#'] = val
_data['user_metadata'][field]['#extra#'] = extra
else:
# You are allowed to stick arbitrary attributes onto this object as
@@ -294,28 +295,24 @@ class Metadata(object):
_data = object.__getattribute__(self, '_data')
_data['user_metadata'][field] = metadata
- def copy_specific_attributes(self, other, attrs):
+ def template_to_attribute(self, other, attrs):
'''
- Takes a dict {src:dest, src:dest} and copys other[src] to self[dest].
- This is on a best-efforts basis. Some assignments can make no sense.
+ Takes a dict {src:dest, src:dest}, evaluates the template in the context
+ of other, then copies the result to self[dest]. This is on a best-
+ efforts basis. Some assignments can make no sense.
'''
if not attrs:
return
for src in attrs:
try:
- sfm = other.metadata_for_field(src)
+ val = composite_formatter.safe_format\
+ (src, other, 'PLUGBOARD TEMPLATE ERROR', other)
dfm = self.metadata_for_field(attrs[src])
if dfm['is_multiple']:
- if sfm['is_multiple']:
- self.set(attrs[src], other.get(src))
- else:
- self.set(attrs[src],
- [f.strip() for f in other.get(src).split(',')
- if f.strip()])
- elif sfm['is_multiple']:
- self.set(attrs[src], ','.join(other.get(src)))
+ self.set(attrs[src],
+ [f.strip() for f in val.split(',') if f.strip()])
else:
- self.set(attrs[src], other.get(src))
+ self.set(attrs[src], val)
except:
traceback.print_exc()
pass
diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py
index 4c866b1855..3da4fddb5d 100644
--- a/src/calibre/gui2/device.py
+++ b/src/calibre/gui2/device.py
@@ -349,7 +349,7 @@ class DeviceManager(Thread): # {{{
with open(f, 'r+b') as stream:
if cpb:
newmi = mi.deepcopy()
- newmi.copy_specific_attributes(mi, cpb)
+ newmi.template_to_attribute(mi, cpb)
else:
newmi = mi
set_metadata(stream, newmi, stream_type=ext)
diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index 9da5420681..a808fd9c43 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -750,8 +750,10 @@ class BooksModel(QAbstractTableModel): # {{{
self.refresh(reset=True)
return True
- self.db.set_custom(self.db.id(row), val, extra=s_index,
+ id = self.db.id(row)
+ self.db.set_custom(id, val, extra=s_index,
label=label, num=None, append=False, notify=True)
+ self.refresh_ids([id], current_row=row)
return True
def setData(self, index, value, role):
diff --git a/src/calibre/gui2/preferences/plugboard.py b/src/calibre/gui2/preferences/plugboard.py
index 124654b643..011131ae48 100644
--- a/src/calibre/gui2/preferences/plugboard.py
+++ b/src/calibre/gui2/preferences/plugboard.py
@@ -56,18 +56,19 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
self.formats.insert(1, plugboard_any_format_value)
self.new_format.addItems(self.formats)
- self.fields = ['']
- for f in self.db.all_field_keys():
- if self.db.field_metadata[f].get('rec_index', None) is not None and\
- self.db.field_metadata[f]['datatype'] is not None and \
- self.db.field_metadata[f]['search_terms']:
- self.fields.append(f)
- self.fields.sort(cmp=field_cmp)
+ self.source_fields = ['']
+ for f in self.db.custom_field_keys():
+ if self.db.field_metadata[f]['datatype'] == 'composite':
+ self.source_fields.append(f)
+ self.source_fields.sort(cmp=field_cmp)
+
+ self.dest_fields = ['', 'authors', 'author_sort', 'publisher',
+ 'tags', 'title']
self.source_widgets = []
self.dest_widgets = []
for i in range(0, 10):
- w = QtGui.QComboBox(self)
+ w = QtGui.QLineEdit(self)
self.source_widgets.append(w)
self.fields_layout.addWidget(w, 5+i, 0, 1, 1)
w = QtGui.QComboBox(self)
@@ -101,14 +102,13 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
self.ok_button.setEnabled(True)
self.del_button.setEnabled(True)
for w in self.source_widgets:
- w.addItems(self.fields)
+ w.clear()
for w in self.dest_widgets:
- w.addItems(self.fields)
+ w.addItems(self.dest_fields)
def set_field(self, i, src, dst):
- idx = self.fields.index(src)
- self.source_widgets[i].setCurrentIndex(idx)
- idx = self.fields.index(dst)
+ self.source_widgets[i].setText(src)
+ idx = self.dest_fields.index(dst)
self.dest_widgets[i].setCurrentIndex(idx)
def edit_device_changed(self, txt):
@@ -216,11 +216,11 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
def ok_clicked(self):
pb = {}
for i in range(0, len(self.source_widgets)):
- s = self.source_widgets[i].currentIndex()
- if s != 0:
+ s = unicode(self.source_widgets[i].text())
+ if s:
d = self.dest_widgets[i].currentIndex()
if d != 0:
- pb[self.fields[s]] = self.fields[d]
+ pb[s] = self.dest_fields[d]
if len(pb) == 0:
if self.current_format in self.current_plugboards:
fpb = self.current_plugboards[self.current_format]
@@ -266,9 +266,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
if d not in self.current_plugboards[f]:
continue
ops = []
- for op in self.fields:
- if op not in self.current_plugboards[f][d]:
- continue
+ for op in self.current_plugboards[f][d]:
ops.append(op + '->' + self.current_plugboards[f][d][op])
txt += '%s:%s [%s]\n'%(f, d, ', '.join(ops))
self.existing_plugboards.setPlainText(txt)
diff --git a/src/calibre/gui2/preferences/plugboard.ui b/src/calibre/gui2/preferences/plugboard.ui
index f88af8ff50..79a07be1f7 100644
--- a/src/calibre/gui2/preferences/plugboard.ui
+++ b/src/calibre/gui2/preferences/plugboard.ui
@@ -87,6 +87,9 @@
QPlainTextEdit::NoWrap
+
+ true
+
@@ -109,7 +112,7 @@
- Source field
+ Source templateQt::AlignCenter
diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index a36dbe57a9..42720c5e83 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -672,7 +672,7 @@ class ResultCache(SearchQueryParser): # {{{
if len(self.composites) > 0:
mi = db.get_metadata(id, index_is_id=True)
for k,c in self.composites:
- self._data[id][c] = mi.format_field(k)[1]
+ self._data[id][c] = mi.get(k)
self._map[0:0] = ids
self._map_filtered[0:0] = ids
@@ -702,7 +702,7 @@ class ResultCache(SearchQueryParser): # {{{
if len(self.composites) > 0:
mi = db.get_metadata(item[0], index_is_id=True)
for k,c in self.composites:
- item[c] = mi.format_field(k)[1]
+ item[c] = mi.get(k)
self._map = [i[0] for i in self._data if i is not None]
if field is not None:
diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py
index a2c8a62694..113ebf823a 100644
--- a/src/calibre/library/save_to_disk.py
+++ b/src/calibre/library/save_to_disk.py
@@ -112,8 +112,6 @@ class SafeFormat(TemplateFormatter):
Provides a format function that substitutes '' for any missing value
'''
- composite_values = {}
-
def get_value(self, key, args, kwargs):
try:
b = self.book.get_user_metadata(key, False)
@@ -131,11 +129,6 @@ class SafeFormat(TemplateFormatter):
except:
return ''
- def safe_format(self, fmt, kwargs, error_value, book, sanitize=None):
- self.composite_values = {}
- return TemplateFormatter.safe_format(self, fmt, kwargs, error_value,
- book, sanitize)
-
safe_formatter = SafeFormat()
def get_components(template, mi, id, timefmt='%b %Y', length=250,
@@ -279,7 +272,7 @@ def save_book_to_disk(id, db, root, opts, length):
try:
if cpb:
newmi = mi.deepcopy()
- newmi.copy_specific_attributes(mi, cpb)
+ newmi.template_to_attribute(mi, cpb)
else:
newmi = mi
set_metadata(stream, newmi, fmt)
diff --git a/src/calibre/utils/formatter.py b/src/calibre/utils/formatter.py
index f95a6deee5..502574dd3c 100644
--- a/src/calibre/utils/formatter.py
+++ b/src/calibre/utils/formatter.py
@@ -11,6 +11,10 @@ class TemplateFormatter(string.Formatter):
Provides a format function that substitutes '' for any missing value
'''
+ # Dict to do recursion detection. It is up the the individual get_value
+ # method to use it. It is cleared when starting to format a template
+ composite_values = {}
+
def __init__(self):
string.Formatter.__init__(self)
self.book = None
@@ -114,6 +118,7 @@ class TemplateFormatter(string.Formatter):
self.kwargs = kwargs
self.book = book
self.sanitize = sanitize
+ self.composite_values = {}
try:
ans = self.vformat(fmt, [], kwargs).strip()
except:
From 1b41568d4c9aa67331041a09210faa1299345e29 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Wed, 29 Sep 2010 15:12:01 +0100
Subject: [PATCH 181/207] 1) Add validation to plugboard gui 2) allow
plugboards to use metadata fields with no field metadata (e.g., language)
---
src/calibre/ebooks/metadata/book/base.py | 2 +-
src/calibre/gui2/preferences/plugboard.py | 21 ++++++-
src/calibre/gui2/preferences/plugboard.ui | 76 ++++++++++++++++++-----
3 files changed, 81 insertions(+), 18 deletions(-)
diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index 56df573cee..17aa2d5603 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -308,7 +308,7 @@ class Metadata(object):
val = composite_formatter.safe_format\
(src, other, 'PLUGBOARD TEMPLATE ERROR', other)
dfm = self.metadata_for_field(attrs[src])
- if dfm['is_multiple']:
+ if dfm and dfm['is_multiple']:
self.set(attrs[src],
[f.strip() for f in val.split(',') if f.strip()])
else:
diff --git a/src/calibre/gui2/preferences/plugboard.py b/src/calibre/gui2/preferences/plugboard.py
index 011131ae48..3742eb24d0 100644
--- a/src/calibre/gui2/preferences/plugboard.py
+++ b/src/calibre/gui2/preferences/plugboard.py
@@ -13,6 +13,7 @@ from calibre.gui2.preferences.plugboard_ui import Ui_Form
from calibre.customize.ui import metadata_writers, device_plugins
from calibre.library.save_to_disk import plugboard_any_format_value, \
plugboard_any_device_value, plugboard_save_to_disk_value
+from calibre.utils.formatter import validation_formatter
class ConfigWidget(ConfigWidgetBase, Ui_Form):
@@ -62,12 +63,13 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
self.source_fields.append(f)
self.source_fields.sort(cmp=field_cmp)
- self.dest_fields = ['', 'authors', 'author_sort', 'publisher',
- 'tags', 'title']
+ self.dest_fields = ['',
+ 'authors', 'author_sort', 'language', 'publisher',
+ 'tags', 'title', 'title_sort']
self.source_widgets = []
self.dest_widgets = []
- for i in range(0, 10):
+ for i in range(0, len(self.dest_fields)-1):
w = QtGui.QLineEdit(self)
self.source_widgets.append(w)
self.fields_layout.addWidget(w, 5+i, 0, 1, 1)
@@ -220,7 +222,20 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
if s:
d = self.dest_widgets[i].currentIndex()
if d != 0:
+ try:
+ validation_formatter.validate(s)
+ except Exception, err:
+ error_dialog(self, _('Invalid template'),
+ '
'+_('The destination field cannot be blank'),
+ show=True)
+ return
+
if len(pb) == 0:
if self.current_format in self.current_plugboards:
fpb = self.current_plugboards[self.current_format]
diff --git a/src/calibre/gui2/preferences/plugboard.ui b/src/calibre/gui2/preferences/plugboard.ui
index 79a07be1f7..4a3192aab5 100644
--- a/src/calibre/gui2/preferences/plugboard.ui
+++ b/src/calibre/gui2/preferences/plugboard.ui
@@ -17,7 +17,12 @@
- Here you can control what metadata calibre uses when saving or sending books:
+ Here you can change the metadata calibre uses when saving or sending books. One possibility is to alter the title to contain series informaton. Another would be to change the author sort.
+
+Use this dialog to define for a format (or all formats) and a device (or all devices) the template to be used to find the value to assign to a destination field. Often the templates will contain simple references to composite columns, but this is not necessary. You can put arbitrary templates in the source box.
+
+
+ Qt::PlainTexttrue
@@ -129,7 +134,7 @@
-
+ Qt::Vertical
@@ -143,18 +148,61 @@
-
-
- Save
-
-
-
-
-
-
- Delete
-
-
+
+
+
+
+ Qt::Horizontal
+
+
+
+ 40
+ 20
+
+
+
+
+
+
+
+ Save plugboard
+
+
+
+
+
+
+ Qt::Horizontal
+
+
+
+ 40
+ 20
+
+
+
+
+
+
+
+ Delete plugboard
+
+
+
+
+
+
+ Qt::Horizontal
+
+
+
+ 40
+ 20
+
+
+
+
+
From 084b0cff49bdfbf3664881e69e4fe3692e0bfc29 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Wed, 29 Sep 2010 15:21:02 +0100
Subject: [PATCH 182/207] Fix intro text a bit.
---
src/calibre/gui2/preferences/plugboard.ui | 19 +++++++++++++++----
1 file changed, 15 insertions(+), 4 deletions(-)
diff --git a/src/calibre/gui2/preferences/plugboard.ui b/src/calibre/gui2/preferences/plugboard.ui
index 4a3192aab5..efe500aebd 100644
--- a/src/calibre/gui2/preferences/plugboard.ui
+++ b/src/calibre/gui2/preferences/plugboard.ui
@@ -17,9 +17,13 @@
- Here you can change the metadata calibre uses when saving or sending books. One possibility is to alter the title to contain series informaton. Another would be to change the author sort.
+ Here you can change the metadata calibre uses to update a book when saving to disk or sending to device.
-Use this dialog to define for a format (or all formats) and a device (or all devices) the template to be used to find the value to assign to a destination field. Often the templates will contain simple references to composite columns, but this is not necessary. You can put arbitrary templates in the source box.
+Use this dialog to define a 'plugboard' for for a format (or all formats) and a device (or all devices). The plugboard spefies what template is connected to what field. The template is used to find compute a value, and that value is assigned to the connected field.
+
+Often templates will contain simple references to composite columns, but this is not necessary. You can use any template in a source box that you can use elsewhere in calibre.
+
+One possible use for a plugboard is to alter the title to contain series informaton. Another would be to change the author sort, something that mobi users might do to force it to use the ';' that the kindle requires. A third would be to specify the language.Qt::PlainText
@@ -29,7 +33,14 @@ Use this dialog to define for a format (or all formats) and a device (or all dev
-
+
+
+
+ Qt::Horizontal
+
+
+
+
@@ -112,7 +123,7 @@ Use this dialog to define for a format (or all formats) and a device (or all dev
-
+
From bc2abb333b0cd053729f226cd8ec5adf1054d52f Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Wed, 29 Sep 2010 18:59:07 +0100
Subject: [PATCH 183/207] Merge from trunk
---
src/calibre/gui2/actions/choose_library.py | 1 -
src/calibre/gui2/dialogs/metadata_bulk.py | 7 -------
2 files changed, 8 deletions(-)
diff --git a/src/calibre/gui2/actions/choose_library.py b/src/calibre/gui2/actions/choose_library.py
index d3045fecf4..2f8beab976 100644
--- a/src/calibre/gui2/actions/choose_library.py
+++ b/src/calibre/gui2/actions/choose_library.py
@@ -217,7 +217,6 @@ class ChooseLibraryAction(InterfaceAction):
def backup_status(self, location):
dirty_text = 'no'
try:
- print 'here'
dirty_text = \
unicode(self.gui.library_view.model().db.dirty_queue_length())
except:
diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py
index 4fc85f2b30..09a196ca58 100644
--- a/src/calibre/gui2/dialogs/metadata_bulk.py
+++ b/src/calibre/gui2/dialogs/metadata_bulk.py
@@ -658,12 +658,6 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
do_remove_conv, do_auto_author, series, do_series_restart,
series_start_value, do_title_case, clear_series)
-# bb = BlockingBusy(_('Applying changes to %d books. This may take a while.')
-# %len(self.ids), parent=self)
-# self.worker = Worker(args, self.db, self.ids,
-# getattr(self, 'custom_column_widgets', []),
-# Dispatcher(bb.accept, parent=bb))
-
bb = MyBlockingBusy(_('Applying changes to %d books.\nPhase {0} {1}%%.')
%len(self.ids), args, self.db, self.ids,
getattr(self, 'custom_column_widgets', []),
@@ -673,7 +667,6 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
# which can slow down bulk editing of large numbers of books
self.model.stop_metadata_backup()
try:
-# self.worker.start()
bb.exec_()
finally:
self.model.start_metadata_backup()
From 77019c66bfc07b52eaca7210924e1d910c236486 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Wed, 29 Sep 2010 12:03:37 -0600
Subject: [PATCH 184/207] ...
---
src/calibre/library/database2.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index 08dd74af29..ca8824ae1c 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -1484,7 +1484,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
# there is a change of case
self.conn.execute('''UPDATE tags SET name=?
WHERE id=?''', (new_name, old_id))
- self.dirty_books_referencing('tags', new_id, commit=False)
new_id = old_id
else:
# It is possible that by renaming a tag, the tag will appear
From f75f02a933934453b1e2f028d6454b49b777a93e Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Wed, 29 Sep 2010 19:04:37 +0100
Subject: [PATCH 185/207] Fix typo in plugboard UI
---
src/calibre/gui2/preferences/plugboard.ui | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/calibre/gui2/preferences/plugboard.ui b/src/calibre/gui2/preferences/plugboard.ui
index 6329a78ce1..f2ff6fb223 100644
--- a/src/calibre/gui2/preferences/plugboard.ui
+++ b/src/calibre/gui2/preferences/plugboard.ui
@@ -19,7 +19,7 @@
Here you can change the metadata calibre uses to update a book when saving to disk or sending to device.
-Use this dialog to define a 'plugboard' for for a format (or all formats) and a device (or all devices). The plugboard spefies what template is connected to what field. The template is used to compute a value, and that value is assigned to the connected field.
+Use this dialog to define a 'plugboard' for a format (or all formats) and a device (or all devices). The plugboard spefies what template is connected to what field. The template is used to compute a value, and that value is assigned to the connected field.
Often templates will contain simple references to composite columns, but this is not necessary. You can use any template in a source box that you can use elsewhere in calibre.
From 2f6fa5c8a8d763450d6ffcd2c68d77ceffae6a4e Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Wed, 29 Sep 2010 12:49:52 -0600
Subject: [PATCH 186/207] Fix reselect after bulk edit very slow
---
src/calibre/gui2/actions/edit_metadata.py | 5 ++-
src/calibre/gui2/library/views.py | 48 +++++++++++------------
2 files changed, 27 insertions(+), 26 deletions(-)
diff --git a/src/calibre/gui2/actions/edit_metadata.py b/src/calibre/gui2/actions/edit_metadata.py
index cc74b3c515..17c6da9a4c 100644
--- a/src/calibre/gui2/actions/edit_metadata.py
+++ b/src/calibre/gui2/actions/edit_metadata.py
@@ -188,8 +188,9 @@ class EditMetadataAction(InterfaceAction):
finally:
self.gui.tags_view.blockSignals(False)
if changed:
- self.gui.library_view.model().resort(reset=False)
- self.gui.library_view.model().research()
+ m = self.gui.library_view.model()
+ m.resort(reset=False)
+ m.research()
self.gui.tags_view.recount()
if self.gui.cover_flow:
self.gui.cover_flow.dataChanged()
diff --git a/src/calibre/gui2/library/views.py b/src/calibre/gui2/library/views.py
index b113866ecc..4b6bda1d2a 100644
--- a/src/calibre/gui2/library/views.py
+++ b/src/calibre/gui2/library/views.py
@@ -9,7 +9,7 @@ import os
from functools import partial
from PyQt4.Qt import QTableView, Qt, QAbstractItemView, QMenu, pyqtSignal, \
- QModelIndex, QIcon
+ QModelIndex, QIcon, QItemSelection
from calibre.gui2.library.delegates import RatingDelegate, PubDateDelegate, \
TextDelegate, DateDelegate, TagsDelegate, CcTextDelegate, \
@@ -488,29 +488,29 @@ class BooksView(QTableView): # {{{
Select rows identified by identifiers. identifiers can be a set of ids,
row numbers or QModelIndexes.
'''
- selmode = self.selectionMode()
- self.setSelectionMode(QAbstractItemView.MultiSelection)
- try:
- rows = set([x.row() if hasattr(x, 'row') else x for x in
- identifiers])
- if using_ids:
- rows = set([])
- identifiers = set(identifiers)
- m = self.model()
- for row in range(m.rowCount(QModelIndex())):
- if m.id(row) in identifiers:
- rows.add(row)
- if rows:
- row = list(sorted(rows))[0]
- if change_current:
- self.set_current_row(row, select=False)
- if scroll:
- self.scroll_to_row(row)
- self.clearSelection()
- for r in rows:
- self.selectRow(r)
- finally:
- self.setSelectionMode(selmode)
+ rows = set([x.row() if hasattr(x, 'row') else x for x in
+ identifiers])
+ if using_ids:
+ rows = set([])
+ identifiers = set(identifiers)
+ m = self.model()
+ for row in xrange(m.rowCount(QModelIndex())):
+ if m.id(row) in identifiers:
+ rows.add(row)
+ rows = list(sorted(rows))
+ if rows:
+ row = rows[0]
+ if change_current:
+ self.set_current_row(row, select=False)
+ if scroll:
+ self.scroll_to_row(row)
+ sm = self.selectionModel()
+ sel = QItemSelection()
+ m = self.model()
+ max_col = m.columnCount(QModelIndex()) - 1
+ for row in rows:
+ sel.select(m.index(row, 0), m.index(row, max_col))
+ sm.select(sel, sm.ClearAndSelect)
def close(self):
self._model.close()
From 314b212b59b4e3e16a613fb02088b24c2d1bf92f Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Wed, 29 Sep 2010 13:03:47 -0600
Subject: [PATCH 187/207] Tweak epub: Warning to close open files.
---
src/calibre/gui2/dialogs/tweak_epub.ui | 8 ++---
src/calibre/utils/zipfile.py | 43 +++++++++++++-------------
2 files changed, 25 insertions(+), 26 deletions(-)
diff --git a/src/calibre/gui2/dialogs/tweak_epub.ui b/src/calibre/gui2/dialogs/tweak_epub.ui
index ccd33f44ab..063460aaae 100644
--- a/src/calibre/gui2/dialogs/tweak_epub.ui
+++ b/src/calibre/gui2/dialogs/tweak_epub.ui
@@ -32,7 +32,7 @@
&Explode ePub
-
+ :/images/wizard.png:/images/wizard.png
@@ -49,7 +49,7 @@
&Rebuild ePub
-
+ :/images/exec.png:/images/exec.png
@@ -63,7 +63,7 @@
&Cancel
-
+ :/images/window-close.png:/images/window-close.png
@@ -71,7 +71,7 @@
- Explode the ePub to display contents in a file browser window. To tweak individual files, right-click, then 'Open with...' your editor of choice. When tweaks are complete, close the file browser window. Rebuild the ePub, updating your calibre library.
+ <p>Explode the ePub to display contents in a file browser window. To tweak individual files, right-click, then 'Open with...' your editor of choice. When tweaks are complete, close the file browser window <b>and the editor windows you used to edit files in the epub</b>.</p><p>Rebuild the ePub, updating your calibre library.</p>true
diff --git a/src/calibre/utils/zipfile.py b/src/calibre/utils/zipfile.py
index 8f22b5f9e2..dbcc125274 100644
--- a/src/calibre/utils/zipfile.py
+++ b/src/calibre/utils/zipfile.py
@@ -1147,28 +1147,27 @@ class ZipFile:
self._writecheck(zinfo)
self._didModify = True
- fp = open(filename, "rb")
- # Must overwrite CRC and sizes with correct data later
- zinfo.CRC = CRC = 0
- zinfo.compress_size = compress_size = 0
- zinfo.file_size = file_size = 0
- self.fp.write(zinfo.FileHeader())
- if zinfo.compress_type == ZIP_DEFLATED:
- cmpr = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION,
- zlib.DEFLATED, -15)
- else:
- cmpr = None
- while 1:
- buf = fp.read(1024 * 8)
- if not buf:
- break
- file_size = file_size + len(buf)
- CRC = crc32(buf, CRC) & 0xffffffff
- if cmpr:
- buf = cmpr.compress(buf)
- compress_size = compress_size + len(buf)
- self.fp.write(buf)
- fp.close()
+ with open(filename, "rb") as fp:
+ # Must overwrite CRC and sizes with correct data later
+ zinfo.CRC = CRC = 0
+ zinfo.compress_size = compress_size = 0
+ zinfo.file_size = file_size = 0
+ self.fp.write(zinfo.FileHeader())
+ if zinfo.compress_type == ZIP_DEFLATED:
+ cmpr = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION,
+ zlib.DEFLATED, -15)
+ else:
+ cmpr = None
+ while 1:
+ buf = fp.read(1024 * 8)
+ if not buf:
+ break
+ file_size = file_size + len(buf)
+ CRC = crc32(buf, CRC) & 0xffffffff
+ if cmpr:
+ buf = cmpr.compress(buf)
+ compress_size = compress_size + len(buf)
+ self.fp.write(buf)
if cmpr:
buf = cmpr.flush()
compress_size = compress_size + len(buf)
From cfee94ad0b28f5b6d2267fd5d14e3f75d1ef6a01 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Wed, 29 Sep 2010 13:04:14 -0600
Subject: [PATCH 188/207] Sixth beta
---
src/calibre/constants.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/calibre/constants.py b/src/calibre/constants.py
index 6cab1d32e7..7cb4d78cf8 100644
--- a/src/calibre/constants.py
+++ b/src/calibre/constants.py
@@ -2,7 +2,7 @@ __license__ = 'GPL v3'
__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
__docformat__ = 'restructuredtext en'
__appname__ = 'calibre'
-__version__ = '0.7.904'
+__version__ = '0.7.905'
__author__ = "Kovid Goyal "
import re
From f2e0b501440e3034740904f6e91e8765dea30ff0 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Wed, 29 Sep 2010 21:48:40 +0100
Subject: [PATCH 189/207] Fix search/replace
---
src/calibre/gui2/dialogs/metadata_bulk.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py
index 347ed36d7c..6b41434347 100644
--- a/src/calibre/gui2/dialogs/metadata_bulk.py
+++ b/src/calibre/gui2/dialogs/metadata_bulk.py
@@ -169,7 +169,6 @@ class MyBlockingBusy(QDialog):
self.current_index = len(self.ids)
elif self.current_phase == 4:
self.s_r_func(id)
- self.current_index = len(self.ids)
# do the next one
self.current_index += 1
self.do_one_signal.emit()
From 57b6d6c0d1ed553dd25a4758b95b6ba0b29cf065 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Wed, 29 Sep 2010 15:28:11 -0600
Subject: [PATCH 190/207] Change plugboard image to an icon instead of an image
---
imgsrc/plugboard.svg | 7257 +++++++++++++++++++++++++++
resources/images/plugboard.png | Bin 31806 -> 3694 bytes
resources/recipes/popscience.recipe | 1 -
3 files changed, 7257 insertions(+), 1 deletion(-)
create mode 100644 imgsrc/plugboard.svg
diff --git a/imgsrc/plugboard.svg b/imgsrc/plugboard.svg
new file mode 100644
index 0000000000..9aa0996193
--- /dev/null
+++ b/imgsrc/plugboard.svg
@@ -0,0 +1,7257 @@
+
+
+
+
diff --git a/resources/images/plugboard.png b/resources/images/plugboard.png
index 88f0869b8d9de4552184308b934889c5c83ec85f..345fa6440e612468ad76b3e3fa5bf07c89f18250 100644
GIT binary patch
literal 3694
zcmWkxc{o&S7(YxHMX5U^OURNK85+!ubWKKXH6K=YBJSMFx@qigtG*nd#>_bN0IJxW3*LP>{n$=Ez>wV{dby{LhXjgVN`HU8S
zjTYBoBS_@T8FY5UguTDj8bf2bi-63@#=xY`dJYd&o^y7CiMPMzdK4Zt8R}YJTQ#cX
zxZA2TZaXAq=1}Q@Uol$Q;~bVLu6brhY&G^Fy<_q
zHi|1Ye6lUT?p^57FlEef=Ddxw3{UK26sKg2(ty(`7jvVzv@EF)E+o=tiQqPWzE7BT
zmcIwa48*N<%|{=JT|so6C-sSy;48&29B1?_ETj*fHGhV?v~qhUrR4|
zVVn}H{EB(fRe=Ri>OxI8Rx4LMp>>suC215D7dPmcnN1BKzOYB-G4HO9B^}*k6C@6X
zv-64rtw2tRMTvV}Kfwei)owG*g21*!l&t&D
ze-q9CzybdZLzU-(c1=sm40m)V94eGTLqPx$KoCyx$S*F|T3cV&w!+hUVq(}9KA1K1
z#;DOw`R@2vRaR6~FbNH-=o;kI*WdP$T--LgFMOCS{|KHU1I=|#A>4-qp9jRz
zFPVW$z)LuVzPNKxBu`o!Q1n^-^`_76LrslO*WpG>5TDItCn9D~e{ZI<_2l@lXlr4B
z^fW7o59qRq>5k)7aLwqLQ>Q(hhHL1Fa6pOaBvJq;=HjYPgViB@?G{$-FJJI9LCw3<
z0o_ocL^8R_+uM66yZaQ&?$>1g2!^R_vvghG`^b*j4@=Fjhgh-7s~&y1IIb^YR^^RbvJq19)X(VDPhlU?BTqr0TVikNbByV
zU!xfeBgW0fwd_@IjK^R3@$_o*(QhqO0NojR=Z+a;@VW!Ffc@wzey=uopufMyGA5Jw
zs^=V-!ob)VbuVQ;cdEhnbZvR0IOw5RQg=f`L(8XU`xU}h<4wMOFeABCj*W(~T%ddT
z_;hJ$Y4OXGq!JPmWF&ZEmxB}_SK}ji6n#o^#Cu_A_5L57(OPHEo!fdl^!})gMc}9u
zoR=~(d1Jz@#ihk-De+F)1LZ28wQ)ue7knu9SyIyVhc%?W|N1v+-CO4)!n(QdszQe#
z0MF8wLqqh<)ad@+UfJuHSg&{adEoK*9uP|zVPRqAzSB<&E6U2sFx0AZ#8+{u*Af6g
z5WIrRap40j+lj21(A=!1fZcD_F$sdu
z=HcOCwm4dsATgMRGQ{D|+QeK$tMgioA2~w6nzgZ5PDGbBQD;3-R4XgvP5>RLk(f6>
zKcAc@UQ-SaP)<VJ~aZ;5Dq8?Wp*xUyp-d+U~f8dOD)gX1DP
z!B#eH@}m_b^kT)YV*l28k0HP^vYV#QiAk%nAIh^%rQkTuM?};(k5v@b*48os&Anf~
zG&Tn8uJ1i;F~Q;9)zs9`rOX>q2%D}L+eeSaRl}{+Vz1o#@2&wZc9%}4TgG${UwuB<
zS?N({+jVZGZR+rKv%)Fbx;VM-Yd!-v!0BDx*hrOkn{K?$%~n%P_E&**(sx$AcZW+d
zq*5@^dG?05ks|W4#iVPk3C9@|EiKB7OqW&6CLxX%9@y%-l%@GEST+ok#ww&8ZY4HA
zQYdquZhY!_aB$Gh$;l}Y(J@DXVaz^0D&@}(q~o8}ahsQ=W+ALX*X)+C{=a*!Uw-fg
zA#@+sIZZ8m7{mm)q!%!O<-li*Am(M4@4wZ(MXAuYPw!6yzjF-JIWE`^7){6Y30$Jc
z{*9>H(iD#X9B$;}FK|*T9+h&%Y#Lyx9-5k(T-s@}1VNk~d2PsmJd-u$`g_g>QIgE0)#
zW5aFdp`p>U`t|GAPi>ZnI-Bk|NCmu>vGqy{VPa)@`I7L+5lBbQ(A*+!TDMV&@utD^
z=O;fyLP8ef_!T8sSXiWs$o^b5lb<{n`mJn^OnrwxLxq0s9`5@oWfeM%=ND%AJN8o_
z-o2v$JGIKP#FH$vcqQ>RW
zNgU>gf$oR4<(y9W&hX23x!1d}3K0hvhu;2;PZ1z+|5u)um*+5`wDllCJnEPv?{e_W
z-hNRrD(@Kt0y&AkEG-ru{p7Fda2X%Rboru^66d+?CF&MBcW{v8){MWQ@wFD1O(J}%
zB$FLSxp$obfCEeQAr7LFdbSIbC5`&^adPFLs*!cXx9?zdm33hgN6$}RvY!LEFuDa5
za8jk$(pM%h^bZdYS!!J-YK#)|<2gkX_)z+)36h-%5`BpS!U)7SA
z_oJJI0s@LTtnE^)S|(e^&ztxP+c?U+1P4ZD`xln~
zAW;xYjUkiumX?;#D84wxe?&w@nFzMFa033Dn(lB;o8_K=M>snc;AoI@-SIz0g2Nmg
z9sL-D5+|66S7A=kKAqlaUEV)FKAzpd3(U3vvtW~tRV2RV{>WP@
zLaHub_dBJdUmfT4_4IxlelXo%{v7$&LNWO^1MmQZ0Eq40id8S*4H85^8mPC~R4P^2
zb*g@+(j7I)2o$1obpk-M;jVBB1)PE42LJ-n8U$&j371cO3l@)J#poUw;BaAWv*5A9
zOG-+Vibv8L{CAcYK$fH8Q|39)4@547*dra|I<2Wtp}#vjJDEg9MLj_g3aE9PIoWS*
z{f9;9r&Ff(f}685pU+*!U?0F300RFQQ8nPbK;(vYfsP(-gadeq#bWQIrKQpFm3H$t
zEt)5Ds7u`01G`g37J=SkSL17ItE*X~@*b!{v7kO@;`H=>)Pq|7F)z>B0?u_9m-oI5
z#M80xU;M@L6+5P1yk
zezFaMK`v99rLV7VaB8Y=2=b{dSzw}Vy@#=OZpm#;|#M(
z7~Po}H^1PCRrJ+(7Em}rE;~tD*w-j)K3LXkm~th9?l0GYCp#(WE#=FXGepqN3ZkK!
zVoCYs<>l`h8;?-Ho*Y(D0f`(Q8ynlFPBAkCLx3)ZR#5;tBdWj*C;%C}GN7unT!~c!
zff>&l`fDN8+~48iz*YfB*+G0te!nyYIn-}wYCEZ@sMr}A8uktj>NJ*?mhSFt()hv1
zOfT>rLIU!dhlFPWStS0d=NeQ=rcO0<$sjh8ZXowwoP$oGqsZOQvi+>*`IGEX*$`
zXnsy#9lLd1I|Ide_QfO6yCo&4f7YgY@nCe3=$AUMWFS~2+$#M>)In;2Lx_bq@7d>X
z5VA*ub-u!EcVk9>ol=6ouVBo~%+vtx$4H(Rm(Bg=E~;?2-@Di0w9ubf>6reXF!9+f
z6iOWQ6OtVW(`yL`VInddbO&!`;FOBVkr7U?2cKsufSmRVr=!X1AxvX
zwA0!OhYJ)J6~&y6JC(irt#pB%o!v+~OX(>@$SaIiH{ILYJG#brGTjapvUhaMIIzd;WG02p<&QJ8;w)c+PO=-*Y}
z=2!8*jm}rwDgXeu#`3=f1QZr?0sv5e4oc1JQL$K`&UEl(5Km`OgYpXL^~L;3+G|F1
zQss>XKURtiMw2MdN$q0l*1Ho>q&+k$gEze_-Q-fuo>PZf%AWI}
zWLRiu=;3_-KB>5-AvkMjGFC>hEmm~)v&Vji`5JG;(F?o-t%(N{Cyd5dFofd%`?t~2
z6U$_}Xdu4ct?*vE4w7dA&=h6%Q2Jw2KrjWqBC0{Dpf@cJDFtd1WtNVaw^h8w6
zZZ1eiee>>SK4y{?+5GLd?~{PwH-G+%J_A8Y(t>!1`iub?su;rC0BMB(G7w5EP`r_uQ6B8f-~|rt)r3#j{k;g
zmvEK;SW`44i8Z7NP>&c$1b>=7atQP@HLCY>(y)1z?I>8}0Zq-QpWX3Y=J|Cl$l;Re
zrNNmSdgIx{f;Q}J^cmTj=-x6ZYp`xf)>u`;JK7}^)>VwIfeI-QiRn#eehS~}vqi^Q
z`*QiaCy4ti5vnm#4~L~hAH7uOIqhKeMJ03J54m@1Gn4@MzWK#0B|7oA8y*$DBzg4u
zp!bc>j2wa{`GDaIz7*7FUQfjh)6$&kS-!hm+ajYuDUCG&j)*Kj0&G&l^o?@qa4#xfjt11gJJiX%~C98bv-S9B*
zDGWW)OQ3@0S0+c_jL8YTcW~+qm&;o}Fb)7x%GuT^InGp0c6Tc)vcFt!uk!8=f7IPG
zz(FG=i-ymQN$_%5#g$ZjdPR%bJ=>R^G#lZk-_8!mS{z*4Jd2n@6Nq8Uyq$ZD`2E9Q
zllfaO^&j549>8+`qcyt;w*e-*7J9%fZ`VgN
zXCZO*a+5ds*3mE@-dJ4z>7lLNW$nPfd(HSt+?C#Z{r&ob$6F;=zq=xfW<9O6vHqpE
zf+K$uSFctEe%|q