From b3cbbd3ea8e05e1cd75b12e70d28ca1decc505d4 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Thu, 26 Aug 2010 14:32:57 +0100 Subject: [PATCH] Initial attempt, including full cached metadata, json serialization, and custom fields in save paths. Custom fields in collections probably work, but they haven't been tested. --- src/calibre/devices/apple/driver.py | 7 +- src/calibre/devices/interface.py | 4 +- src/calibre/devices/kobo/books.py | 21 +- src/calibre/devices/usbms/books.py | 43 +-- src/calibre/devices/usbms/driver.py | 21 +- src/calibre/ebooks/metadata/__init__.py | 216 +------------ src/calibre/ebooks/metadata/book/__init__.py | 41 ++- src/calibre/ebooks/metadata/book/base.py | 296 +++++++++++++++--- .../ebooks/metadata/book/json_codec.py | 125 ++++++++ src/calibre/ebooks/metadata/isbndb.py | 6 +- src/calibre/ebooks/metadata/opf2.py | 9 +- .../gui2/dialogs/config/save_template.py | 29 +- src/calibre/gui2/library/models.py | 3 +- src/calibre/library/database2.py | 31 +- src/calibre/library/save_to_disk.py | 19 +- 15 files changed, 514 insertions(+), 357 deletions(-) create mode 100644 src/calibre/ebooks/metadata/book/json_codec.py diff --git a/src/calibre/devices/apple/driver.py b/src/calibre/devices/apple/driver.py index 916c88f203..75517e9df7 100644 --- a/src/calibre/devices/apple/driver.py +++ b/src/calibre/devices/apple/driver.py @@ -13,7 +13,8 @@ from calibre.devices.errors import UserFeedback from calibre.devices.usbms.deviceconfig import DeviceConfig from calibre.devices.interface import DevicePlugin from calibre.ebooks.BeautifulSoup import BeautifulSoup -from calibre.ebooks.metadata import MetaInformation, authors_to_string +from calibre.ebooks.metadata import authors_to_string +from calibre.ebooks.metadata.book.base import Metadata from calibre.ebooks.metadata.epub import set_metadata from calibre.library.server.utils import strftime from calibre.utils.config import config_dir @@ -2998,14 +2999,14 @@ class BookList(list): ''' return {} -class Book(MetaInformation): +class Book(Metadata): ''' A simple class describing a book in the iTunes Books Library. - See ebooks.metadata.__init__ for all fields ''' def __init__(self,title,author): - MetaInformation.__init__(self, title, authors=[author]) + Metadata.__init__(self, title, authors=[author]) @dynamic_property def title_sorter(self): diff --git a/src/calibre/devices/interface.py b/src/calibre/devices/interface.py index 1384fa03d9..7783173c9c 100644 --- a/src/calibre/devices/interface.py +++ b/src/calibre/devices/interface.py @@ -316,7 +316,7 @@ class DevicePlugin(Plugin): being uploaded to the device. :param names: A list of file names that the books should have once uploaded to the device. len(names) == len(files) - :param metadata: If not None, it is a list of :class:`MetaInformation` objects. + :param metadata: If not None, it is a list of :class:`Metadata` objects. The idea is to use the metadata to determine where on the device to put the book. len(metadata) == len(files). Apart from the regular cover (path to cover), there may also be a thumbnail attribute, which should @@ -335,7 +335,7 @@ class DevicePlugin(Plugin): the device. :param locations: Result of a call to L{upload_books} - :param metadata: List of :class:`MetaInformation` objects, same as for + :param metadata: List of :class:`Metadata` objects, same as for :meth:`upload_books`. :param booklists: A tuple containing the result of calls to (:meth:`books(oncard=None)`, diff --git a/src/calibre/devices/kobo/books.py b/src/calibre/devices/kobo/books.py index a5b2e98d2f..11ab42c29f 100644 --- a/src/calibre/devices/kobo/books.py +++ b/src/calibre/devices/kobo/books.py @@ -7,11 +7,11 @@ import os import re import time -from calibre.ebooks.metadata import MetaInformation +from calibre.ebooks.metadata.book.base import Metadata from calibre.constants import filesystem_encoding, preferred_encoding from calibre import isbytestring -class Book(MetaInformation): +class Book(Metadata): BOOK_ATTRS = ['lpath', 'size', 'mime', 'device_collections', '_new_book'] @@ -23,9 +23,9 @@ class Book(MetaInformation): 'uuid', ] - def __init__(self, prefix, lpath, title, authors, mime, date, ContentType, thumbnail_name, other=None): - - MetaInformation.__init__(self, '') + def __init__(self, prefix, lpath, title, authors, mime, date, ContentType, + thumbnail_name, other=None): + Metadata.__init__(self, '') self.device_collections = [] self._new_book = False @@ -34,7 +34,7 @@ class Book(MetaInformation): self.path = self.path.replace('/', '\\') self.lpath = lpath.replace('\\', '/') else: - self.lpath = lpath + self.lpath = lpath self.title = title if not authors: @@ -52,10 +52,9 @@ class Book(MetaInformation): else: self.datetime = time.gmtime(os.path.getctime(self.path)) except: - self.datetime = time.gmtime() - - if thumbnail_name is not None: - self.thumbnail = ImageWrapper(thumbnail_name) + self.datetime = time.gmtime() + if thumbnail_name is not None: + self.thumbnail = ImageWrapper(thumbnail_name) self.tags = [] if other: self.smart_update(other) @@ -90,7 +89,7 @@ class Book(MetaInformation): in C{other} takes precedence, unless the information in C{other} is NULL. ''' - MetaInformation.smart_update(self, other) + Metadata.smart_update(self, other) for attr in self.BOOK_ATTRS: if hasattr(other, attr): diff --git a/src/calibre/devices/usbms/books.py b/src/calibre/devices/usbms/books.py index 959f26199c..e3c405ee4e 100644 --- a/src/calibre/devices/usbms/books.py +++ b/src/calibre/devices/usbms/books.py @@ -6,29 +6,18 @@ __docformat__ = 'restructuredtext en' import os, re, time, sys -from calibre.ebooks.metadata import MetaInformation +from calibre.ebooks.metadata.book.base import Metadata from calibre.devices.mime import mime_type_ext from calibre.devices.interface import BookList as _BookList from calibre.constants import filesystem_encoding, preferred_encoding from calibre import isbytestring from calibre.utils.config import prefs -class Book(MetaInformation): - - BOOK_ATTRS = ['lpath', 'size', 'mime', 'device_collections', '_new_book'] - - JSON_ATTRS = [ - 'lpath', 'title', 'authors', 'mime', 'size', 'tags', 'author_sort', - 'title_sort', 'comments', 'category', 'publisher', 'series', - 'series_index', 'rating', 'isbn', 'language', 'application_id', - 'book_producer', 'lccn', 'lcc', 'ddc', 'rights', 'publication_type', - 'uuid', - ] - +class Book(Metadata): def __init__(self, prefix, lpath, size=None, other=None): from calibre.ebooks.metadata.meta import path_to_ext - MetaInformation.__init__(self, '') + Metadata.__init__(self, '') self._new_book = False self.device_collections = [] @@ -72,32 +61,6 @@ class Book(MetaInformation): def thumbnail(self): return None - def smart_update(self, other, replace_metadata=False): - ''' - Merge the information in C{other} into self. In case of conflicts, the information - in C{other} takes precedence, unless the information in C{other} is NULL. - ''' - - MetaInformation.smart_update(self, other, replace_metadata) - - for attr in self.BOOK_ATTRS: - if hasattr(other, attr): - val = getattr(other, attr, None) - setattr(self, attr, val) - - def to_json(self): - json = {} - for attr in self.JSON_ATTRS: - val = getattr(self, attr) - if isbytestring(val): - enc = filesystem_encoding if attr == 'lpath' else preferred_encoding - val = val.decode(enc, 'replace') - elif isinstance(val, (list, tuple)): - val = [x.decode(preferred_encoding, 'replace') if - isbytestring(x) else x for x in val] - json[attr] = val - return json - class BookList(_BookList): def __init__(self, oncard, prefix, settings): diff --git a/src/calibre/devices/usbms/driver.py b/src/calibre/devices/usbms/driver.py index 0d28f06f49..a0d1d9dbf8 100644 --- a/src/calibre/devices/usbms/driver.py +++ b/src/calibre/devices/usbms/driver.py @@ -13,7 +13,6 @@ for a particular device. import os import re import time -import json from itertools import cycle from calibre import prints, isbytestring @@ -21,6 +20,7 @@ from calibre.constants import filesystem_encoding, DEBUG from calibre.devices.usbms.cli import CLI from calibre.devices.usbms.device import Device from calibre.devices.usbms.books import BookList, Book +from calibre.ebooks.metadata.book.json_codec import JsonCodec BASE_TIME = None def debug_print(*args): @@ -288,6 +288,7 @@ class USBMS(CLI, Device): # at the end just before the return def sync_booklists(self, booklists, end_session=True): debug_print('USBMS: starting sync_booklists') + json_codec = JsonCodec() if not os.path.exists(self.normalize_path(self._main_prefix)): os.makedirs(self.normalize_path(self._main_prefix)) @@ -296,10 +297,8 @@ class USBMS(CLI, Device): if prefix is not None and isinstance(booklists[listid], self.booklist_class): if not os.path.exists(prefix): os.makedirs(self.normalize_path(prefix)) - js = [item.to_json() for item in booklists[listid] if - hasattr(item, 'to_json')] with open(self.normalize_path(os.path.join(prefix, self.METADATA_CACHE)), 'wb') as f: - f.write(json.dumps(js, indent=2, encoding='utf-8')) + json_codec.encode_to_file(f, booklists[listid]) write_prefix(self._main_prefix, 0) write_prefix(self._card_a_prefix, 1) write_prefix(self._card_b_prefix, 2) @@ -345,19 +344,13 @@ class USBMS(CLI, Device): @classmethod def parse_metadata_cache(cls, bl, prefix, name): - # bl = cls.booklist_class() - js = [] + json_codec = JsonCodec() need_sync = False cache_file = cls.normalize_path(os.path.join(prefix, name)) if os.access(cache_file, os.R_OK): try: with open(cache_file, 'rb') as f: - js = json.load(f, encoding='utf-8') - for item in js: - book = cls.book_class(prefix, item.get('lpath', None)) - for key in item.keys(): - setattr(book, key, item[key]) - bl.append(book) + json_codec.decode_from_file(f, bl, cls.book_class, prefix) except: import traceback traceback.print_exc() @@ -392,7 +385,7 @@ class USBMS(CLI, Device): @classmethod def book_from_path(cls, prefix, lpath): - from calibre.ebooks.metadata import MetaInformation + from calibre.ebooks.metadata.book.base import Metadata if cls.settings().read_metadata or cls.MUST_READ_METADATA: mi = cls.metadata_from_path(cls.normalize_path(os.path.join(prefix, lpath))) @@ -401,7 +394,7 @@ class USBMS(CLI, Device): mi = metadata_from_filename(cls.normalize_path(os.path.basename(lpath)), cls.build_template_regexp()) if mi is None: - mi = MetaInformation(os.path.splitext(os.path.basename(lpath))[0], + mi = Metadata(os.path.splitext(os.path.basename(lpath))[0], [_('Unknown')]) size = os.stat(cls.normalize_path(os.path.join(prefix, lpath))).st_size book = cls.book_class(prefix, lpath, other=mi, size=size) diff --git a/src/calibre/ebooks/metadata/__init__.py b/src/calibre/ebooks/metadata/__init__.py index d4a21e2c8c..fb894d3bbd 100644 --- a/src/calibre/ebooks/metadata/__init__.py +++ b/src/calibre/ebooks/metadata/__init__.py @@ -221,214 +221,18 @@ class ResourceCollection(object): -class MetaInformation(object): - '''Convenient encapsulation of book metadata''' - - @staticmethod - def copy(mi): - ans = MetaInformation(mi.title, mi.authors) - for attr in ('author_sort', 'title_sort', 'comments', 'category', - 'publisher', 'series', 'series_index', 'rating', - 'isbn', 'tags', 'cover_data', 'application_id', 'guide', - 'manifest', 'spine', 'toc', 'cover', 'language', - 'book_producer', 'timestamp', 'lccn', 'lcc', 'ddc', - 'author_sort_map', - 'pubdate', 'rights', 'publication_type', 'uuid'): - if hasattr(mi, attr): - setattr(ans, attr, getattr(mi, attr)) - - def __init__(self, title, authors=(_('Unknown'),)): - ''' +def MetaInformation(title, authors=(_('Unknown'),)): + ''' Convenient encapsulation of book metadata, needed for compatibility @param title: title or ``_('Unknown')`` or a MetaInformation object @param authors: List of strings or [] - ''' - mi = None - if hasattr(title, 'title') and hasattr(title, 'authors'): - mi = title - title = mi.title - authors = mi.authors - self.title = title - self.author = list(authors) if authors else []# Needed for backward compatibility - #: List of strings or [] - self.authors = list(authors) if authors else [] - self.tags = getattr(mi, 'tags', []) - #: mi.cover_data = (ext, data) - self.cover_data = getattr(mi, 'cover_data', (None, None)) - self.author_sort_map = getattr(mi, 'author_sort_map', {}) - - for x in ('author_sort', 'title_sort', 'comments', 'category', 'publisher', - 'series', 'series_index', 'rating', 'isbn', 'language', - 'application_id', 'manifest', 'toc', 'spine', 'guide', 'cover', - 'book_producer', 'timestamp', 'lccn', 'lcc', 'ddc', 'pubdate', - 'rights', 'publication_type', 'uuid', - ): - setattr(self, x, getattr(mi, x, None)) - - def print_all_attributes(self): - for x in ('title','author', 'author_sort', 'title_sort', 'comments', 'category', 'publisher', - 'series', 'series_index', 'tags', 'rating', 'isbn', 'language', - 'application_id', 'manifest', 'toc', 'spine', 'guide', 'cover', - 'book_producer', 'timestamp', 'lccn', 'lcc', 'ddc', 'pubdate', - 'rights', 'publication_type', 'uuid', 'author_sort_map' - ): - prints(x, getattr(self, x, 'None')) - - def smart_update(self, mi, replace_metadata=False): - ''' - Merge the information in C{mi} into self. In case of conflicts, the - information in C{mi} takes precedence, unless the information in mi is - NULL. If replace_metadata is True, then the information in mi always - takes precedence. - ''' - if mi.title and mi.title != _('Unknown'): - self.title = mi.title - - if mi.authors and mi.authors[0] != _('Unknown'): - self.authors = mi.authors - - for attr in ('author_sort', 'title_sort', 'category', - 'publisher', 'series', 'series_index', 'rating', - 'isbn', 'application_id', 'manifest', 'spine', 'toc', - 'cover', 'guide', 'book_producer', - 'timestamp', 'lccn', 'lcc', 'ddc', 'pubdate', 'rights', - 'publication_type', 'uuid'): - if replace_metadata: - setattr(self, attr, getattr(mi, attr, 1.0 if \ - attr == 'series_index' else None)) - elif hasattr(mi, attr): - val = getattr(mi, attr) - if val is not None: - setattr(self, attr, val) - - if replace_metadata: - self.tags = mi.tags - elif mi.tags: - self.tags += mi.tags - self.tags = list(set(self.tags)) - - if mi.author_sort_map: - self.author_sort_map.update(mi.author_sort_map) - - if getattr(mi, 'cover_data', False): - other_cover = mi.cover_data[-1] - self_cover = self.cover_data[-1] if self.cover_data else '' - if not self_cover: self_cover = '' - if not other_cover: other_cover = '' - if len(other_cover) > len(self_cover): - self.cover_data = mi.cover_data - - if replace_metadata: - self.comments = getattr(mi, 'comments', '') - else: - my_comments = getattr(self, 'comments', '') - other_comments = getattr(mi, 'comments', '') - if not my_comments: - my_comments = '' - if not other_comments: - other_comments = '' - if len(other_comments.strip()) > len(my_comments.strip()): - self.comments = other_comments - - other_lang = getattr(mi, 'language', None) - if other_lang and other_lang.lower() != 'und': - self.language = other_lang - - - def format_series_index(self): - try: - x = float(self.series_index) - except ValueError: - x = 1 - return fmt_sidx(x) - - def authors_from_string(self, raw): - self.authors = string_to_authors(raw) - - def format_authors(self): - return authors_to_string(self.authors) - - def format_tags(self): - return u', '.join([unicode(t) for t in self.tags]) - - def format_rating(self): - return unicode(self.rating) - - def __unicode__(self): - ans = [] - def fmt(x, y): - ans.append(u'%-20s: %s'%(unicode(x), unicode(y))) - - fmt('Title', self.title) - if self.title_sort: - fmt('Title sort', self.title_sort) - if self.authors: - fmt('Author(s)', authors_to_string(self.authors) + \ - ((' [' + self.author_sort + ']') if self.author_sort else '')) - if self.publisher: - fmt('Publisher', self.publisher) - if getattr(self, 'book_producer', False): - fmt('Book Producer', self.book_producer) - if self.category: - fmt('Category', self.category) - if self.comments: - fmt('Comments', self.comments) - if self.isbn: - fmt('ISBN', self.isbn) - if self.tags: - fmt('Tags', u', '.join([unicode(t) for t in self.tags])) - if self.series: - fmt('Series', self.series + ' #%s'%self.format_series_index()) - if self.language: - fmt('Language', self.language) - if self.rating is not None: - fmt('Rating', self.rating) - if self.timestamp is not None: - fmt('Timestamp', isoformat(self.timestamp)) - if self.pubdate is not None: - fmt('Published', isoformat(self.pubdate)) - if self.rights is not None: - fmt('Rights', unicode(self.rights)) - if self.lccn: - fmt('LCCN', unicode(self.lccn)) - if self.lcc: - fmt('LCC', unicode(self.lcc)) - if self.ddc: - fmt('DDC', unicode(self.ddc)) - - return u'\n'.join(ans) - - def to_html(self): - ans = [(_('Title'), unicode(self.title))] - ans += [(_('Author(s)'), (authors_to_string(self.authors) if self.authors else _('Unknown')))] - ans += [(_('Publisher'), unicode(self.publisher))] - ans += [(_('Producer'), unicode(self.book_producer))] - ans += [(_('Comments'), unicode(self.comments))] - ans += [('ISBN', unicode(self.isbn))] - if self.lccn: - ans += [('LCCN', unicode(self.lccn))] - if self.lcc: - ans += [('LCC', unicode(self.lcc))] - if self.ddc: - ans += [('DDC', unicode(self.ddc))] - ans += [(_('Tags'), u', '.join([unicode(t) for t in self.tags]))] - if self.series: - ans += [(_('Series'), unicode(self.series)+ ' #%s'%self.format_series_index())] - ans += [(_('Language'), unicode(self.language))] - if self.timestamp is not None: - ans += [(_('Timestamp'), unicode(self.timestamp.isoformat(' ')))] - if self.pubdate is not None: - ans += [(_('Published'), unicode(self.pubdate.isoformat(' ')))] - if self.rights is not None: - ans += [(_('Rights'), unicode(self.rights))] - for i, x in enumerate(ans): - ans[i] = u'%s%s'%x - return u'%s
'%u'\n'.join(ans) - - def __str__(self): - return self.__unicode__().encode('utf-8') - - def __nonzero__(self): - return bool(self.title or self.author or self.comments or self.tags) + ''' + from calibre.ebooks.metadata.book.base import Metadata + mi = None + if hasattr(title, 'title') and hasattr(title, 'authors'): + mi = title + title = mi.title + authors = mi.authors + return Metadata(title, authors, mi) def check_isbn10(isbn): try: diff --git a/src/calibre/ebooks/metadata/book/__init__.py b/src/calibre/ebooks/metadata/book/__init__.py index c3b95f1188..9de7ca1c6b 100644 --- a/src/calibre/ebooks/metadata/book/__init__.py +++ b/src/calibre/ebooks/metadata/book/__init__.py @@ -24,6 +24,8 @@ SOCIAL_METADATA_FIELDS = frozenset([ # For example: {'isbn':'123456789', 'doi':'xxxx', ... } 'classifiers', 'isbn', # Pseudo field for convenience, should get/set isbn classifier + # TODO: not sure what this is, but it is used by OPF + 'category', ]) @@ -69,7 +71,8 @@ BOOK_STRUCTURE_FIELDS = frozenset([ ]) USER_METADATA_FIELDS = frozenset([ - # A dict of a form to be specified + # A dict of dicts similar to field_metadata. Each field description dict + # also contains a value field with the key #value#. 'user_metadata', ]) @@ -86,16 +89,42 @@ DEVICE_METADATA_FIELDS = frozenset([ CALIBRE_METADATA_FIELDS = frozenset([ # An application id # Semantics to be defined. Is it a db key? a db name + key? A uuid? + # (It is currently set to the db_id.) 'application_id', + # the calibre primary key of the item. May want to remove this once Sony's no longer use it + 'db_id', ] ) +ALL_METADATA_FIELDS = SOCIAL_METADATA_FIELDS.union( + PUBLICATION_METADATA_FIELDS).union( + BOOK_STRUCTURE_FIELDS).union( + USER_METADATA_FIELDS).union( + DEVICE_METADATA_FIELDS).union( + CALIBRE_METADATA_FIELDS) -SERIALIZABLE_FIELDS = SOCIAL_METADATA_FIELDS.union( - USER_METADATA_FIELDS).union( - PUBLICATION_METADATA_FIELDS).union( - CALIBRE_METADATA_FIELDS).union( - frozenset(['lpath'])) # I don't think we need device_collections +# All fields except custom fields +STANDARD_METADATA_FIELDS = SOCIAL_METADATA_FIELDS.union( + PUBLICATION_METADATA_FIELDS).union( + BOOK_STRUCTURE_FIELDS).union( + DEVICE_METADATA_FIELDS).union( + CALIBRE_METADATA_FIELDS) + +# Metadata fields that smart update should copy without special handling +COPYABLE_METADATA_FIELDS = SOCIAL_METADATA_FIELDS.union( + PUBLICATION_METADATA_FIELDS).union( + BOOK_STRUCTURE_FIELDS).union( + DEVICE_METADATA_FIELDS).union( + CALIBRE_METADATA_FIELDS) - \ + frozenset(['title', 'authors', 'comments', 'cover_data']) + +SERIALIZABLE_FIELDS = SOCIAL_METADATA_FIELDS.union( + USER_METADATA_FIELDS).union( + PUBLICATION_METADATA_FIELDS).union( + CALIBRE_METADATA_FIELDS).union( + DEVICE_METADATA_FIELDS) - \ + frozenset(['device_collections']) + # I don't think we need device_collections # Serialization of covers/thumbnails will have to be handled carefully, maybe # as an option to the serializer class diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py index 3fed47091f..697de8d890 100644 --- a/src/calibre/ebooks/metadata/book/base.py +++ b/src/calibre/ebooks/metadata/book/base.py @@ -6,8 +6,13 @@ __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' import copy +import traceback + +from calibre import prints +from calibre.ebooks.metadata.book import COPYABLE_METADATA_FIELDS +from calibre.ebooks.metadata.book import STANDARD_METADATA_FIELDS +from calibre.utils.date import isoformat -from calibre.ebooks.metadata.book import RESERVED_METADATA_FIELDS NULL_VALUES = { 'user_metadata': {}, @@ -24,98 +29,313 @@ NULL_VALUES = { class Metadata(object): ''' - This class must expose a superset of the API of MetaInformation in terms - of attribute access and methods. Only the __init__ method is different. - MetaInformation will simply become a function that creates and fills in - the attributes of this class. + A class representing all the metadata for a book. Please keep the method based API of this class to a minimum. Every method becomes a reserved field name. ''' - def __init__(self): + def __init__(self, title, authors=(_('Unknown'),), other=None): + ''' + @param title: title or ``_('Unknown')`` + @param authors: List of strings or [] + @param other: None or a metadata object + ''' object.__setattr__(self, '_data', copy.deepcopy(NULL_VALUES)) + if other is not None: + self.smart_update(other) + else: + if title: + self.title = title + if authors: + #: List of strings or [] + self.author = list(authors) if authors else []# Needed for backward compatibility + self.authors = list(authors) if authors else [] def __getattribute__(self, field): _data = object.__getattribute__(self, '_data') - if field in RESERVED_METADATA_FIELDS: + if field in STANDARD_METADATA_FIELDS: return _data.get(field, None) try: return object.__getattribute__(self, field) except AttributeError: pass if field in _data['user_metadata'].iterkeys(): - # TODO: getting user metadata values - pass + return _data['user_metadata'][field]['#value#'] raise AttributeError( 'Metadata object has no attribute named: '+ repr(field)) - def __setattr__(self, field, val): _data = object.__getattribute__(self, '_data') - if field in RESERVED_METADATA_FIELDS: - if field != 'user_metadata': - if not val: - val = NULL_VALUES[field] - _data[field] = val - else: - raise AttributeError('You cannot set user_metadata directly.') + if field in STANDARD_METADATA_FIELDS: + if not val: + val = NULL_VALUES.get(field, None) + _data[field] = val elif field in _data['user_metadata'].iterkeys(): - # TODO: Setting custom column values - pass + _data['user_metadata'][field]['#value#'] = val else: # You are allowed to stick arbitrary attributes onto this object as # long as they dont conflict with global or user metadata names # Don't abuse this privilege self.__dict__[field] = val + def get(self, field): + return self.__getattribute__(field) + + def set(self, field, val): + self.__setattr__(field, val) + @property - def user_metadata_names(self): + def user_metadata_keys(self): 'The set of user metadata names this object knows about' _data = object.__getattribute__(self, '_data') return frozenset(_data['user_metadata'].iterkeys()) - # Old MetaInformation API {{{ - def copy(self): - pass + @property + def all_user_metadata(self): + ''' + return a dict containing all the custom field metadata associated with + the book. Return a deep copy, just in case the user wants to change + values in the dict (json does). + ''' + _data = object.__getattribute__(self, '_data') + _data = _data['user_metadata'] + res = {} + for k in _data: + res[k] = copy.deepcopy(_data[k]) + return res + + def get_user_metadata(self, field): + ''' + return field metadata from the object if it is there. Otherwise return + None. field is the key name, not the label. Return a shallow copy, + just in case the user wants to change values in the dict (json does). + ''' + _data = object.__getattribute__(self, '_data') + _data = _data['user_metadata'] + if field in _data: + return copy.deepcopy(_data[field]) + return None + + def set_all_user_metadata(self, metadata): + ''' + store custom field metadata into the object. Field is the key name + not the label + ''' + if metadata is None: + traceback.print_stack() + else: + for key in metadata: + self.set_user_metadata(key, metadata[key]) + + def set_user_metadata(self, field, metadata): + ''' + store custom field metadata for one column into the object. Field is + the key name not the label + ''' + if field is not None: + if metadata is None: + traceback.print_stack() + metadata = copy.deepcopy(metadata) + if '#value#' not in metadata: + metadata['#value#'] = None + _data = object.__getattribute__(self, '_data') + _data['user_metadata'][field] = metadata + + @property + def all_attributes(self): + result = {} + _data = object.__getattribute__(self, '_data') + for attr in STANDARD_METADATA_FIELDS: + v = _data.get(attr, None) + if v is not None: + result[attr] = v + for attr in self.user_metadata_keys: + if self.get(attr) is not None: + result[attr] = self.get(attr) + return result + + # Old Metadata API {{{ + @staticmethod + def copy(mi): + ans = Metadata(mi.title, mi.authors) + for attr in STANDARD_METADATA_FIELDS: + if hasattr(mi, attr): + setattr(ans, attr, copy.deepcopy(getattr(mi, attr))) + for x in mi.user_metadata_keys: + meta = mi.get_user_metadata(x) + if meta is not None: + ans.set_user_metadata(x, meta) # get... did the deep copy def print_all_attributes(self): - pass + for x in STANDARD_METADATA_FIELDS: + prints('%s:'%x, getattr(self, x, 'None')) + for x in self.user_metadata_keys: + meta = self.get_user_metadata(x) + if meta is not None: + prints(x, meta) + prints('--------------') def smart_update(self, other, replace_metadata=False): - pass + ''' + Merge the information in C{other} into self. In case of conflicts, the information + in C{other} takes precedence, unless the information in other is NULL. + ''' + if other.title and other.title != _('Unknown'): + self.title = other.title + + if other.authors and other.authors[0] != _('Unknown'): + self.authors = other.authors + + for attr in COPYABLE_METADATA_FIELDS: + if replace_metadata: + setattr(self, attr, getattr(other, attr, 1.0 if \ + attr == 'series_index' else None)) + elif hasattr(other, attr): + val = getattr(other, attr) + if val is not None: + setattr(self, attr, copy.deepcopy(val)) + + if replace_metadata: + self.tags = other.tags + elif other.tags: + self.tags += other.tags + self.tags = list(set(self.tags)) + + if getattr(other, 'author_sort_map', None): + self.author_sort_map.update(other.author_sort_map) + + if getattr(other, 'cover_data', False): + other_cover = other.cover_data[-1] + self_cover = self.cover_data[-1] if self.cover_data else '' + if not self_cover: self_cover = '' + if not other_cover: other_cover = '' + if len(other_cover) > len(self_cover): + self.cover_data = other.cover_data + + if getattr(other, 'user_metadata_keys', None): + for x in other.user_metadata_keys: + meta = other.get_user_metadata(x) + if meta is not None or replace_metadata: + self.set_user_metadata(x, meta) # get... did the deepcopy + + if replace_metadata: + self.comments = getattr(other, 'comments', '') + else: + my_comments = getattr(self, 'comments', '') + other_comments = getattr(other, 'comments', '') + if not my_comments: + my_comments = '' + if not other_comments: + other_comments = '' + if len(other_comments.strip()) > len(my_comments.strip()): + self.comments = other_comments + + other_lang = getattr(other, 'language', None) + if other_lang and other_lang.lower() != 'und': + self.language = other_lang + def format_series_index(self): - pass + from calibre.ebooks.metadata import fmt_sidx + try: + x = float(self.series_index) + except ValueError: + x = 1 + return fmt_sidx(x) def authors_from_string(self, raw): - pass + from calibre.ebooks.metadata import string_to_authors + self.authors = string_to_authors(raw) def format_authors(self): - pass + from calibre.ebooks.metadata import authors_to_string + return authors_to_string(self.authors) def format_tags(self): - pass + return u', '.join([unicode(t) for t in self.tags]) def format_rating(self): return unicode(self.rating) def __unicode__(self): - pass + from calibre.ebooks.metadata import authors_to_string + ans = [] + def fmt(x, y): + ans.append(u'%-20s: %s'%(unicode(x), unicode(y))) + + fmt('Title', self.title) + if self.title_sort: + fmt('Title sort', self.title_sort) + if self.authors: + fmt('Author(s)', authors_to_string(self.authors) + \ + ((' [' + self.author_sort + ']') if self.author_sort else '')) + if self.publisher: + fmt('Publisher', self.publisher) + if getattr(self, 'book_producer', False): + fmt('Book Producer', self.book_producer) + if self.category: + fmt('Category', self.category) + if self.comments: + fmt('Comments', self.comments) + if self.isbn: + fmt('ISBN', self.isbn) + if self.tags: + fmt('Tags', u', '.join([unicode(t) for t in self.tags])) + if self.series: + fmt('Series', self.series + ' #%s'%self.format_series_index()) + if self.language: + fmt('Language', self.language) + if self.rating is not None: + fmt('Rating', self.rating) + if self.timestamp is not None: + fmt('Timestamp', isoformat(self.timestamp)) + if self.pubdate is not None: + fmt('Published', isoformat(self.pubdate)) + if self.rights is not None: + fmt('Rights', unicode(self.rights)) + if self.lccn: + fmt('LCCN', unicode(self.lccn)) + if self.lcc: + fmt('LCC', unicode(self.lcc)) + if self.ddc: + fmt('DDC', unicode(self.ddc)) + # CUSTFIELD: What to do about custom fields? + return u'\n'.join(ans) def to_html(self): - pass + from calibre.ebooks.metadata import authors_to_string + ans = [(_('Title'), unicode(self.title))] + ans += [(_('Author(s)'), (authors_to_string(self.authors) if self.authors else _('Unknown')))] + ans += [(_('Publisher'), unicode(self.publisher))] + ans += [(_('Producer'), unicode(self.book_producer))] + ans += [(_('Comments'), unicode(self.comments))] + ans += [('ISBN', unicode(self.isbn))] + if self.lccn: + ans += [('LCCN', unicode(self.lccn))] + if self.lcc: + ans += [('LCC', unicode(self.lcc))] + if self.ddc: + ans += [('DDC', unicode(self.ddc))] + ans += [(_('Tags'), u', '.join([unicode(t) for t in self.tags]))] + if self.series: + ans += [(_('Series'), unicode(self.series)+ ' #%s'%self.format_series_index())] + ans += [(_('Language'), unicode(self.language))] + if self.timestamp is not None: + ans += [(_('Timestamp'), unicode(self.timestamp.isoformat(' ')))] + if self.pubdate is not None: + ans += [(_('Published'), unicode(self.pubdate.isoformat(' ')))] + if self.rights is not None: + ans += [(_('Rights'), unicode(self.rights))] + for i, x in enumerate(ans): + ans[i] = u'%s%s'%x + # CUSTFIELD: What to do about custom fields + return u'%s
'%u'\n'.join(ans) def __str__(self): return self.__unicode__().encode('utf-8') def __nonzero__(self): - return True + return bool(self.title or self.author or self.comments or self.tags) # }}} - -# We don't need reserved field names for this object any more. Lets just use a -# protocol like the last char of a user field label should be _ when using this -# object -# So mi.tags returns the builtin tags and mi.tags_ returns the user tags - diff --git a/src/calibre/ebooks/metadata/book/json_codec.py b/src/calibre/ebooks/metadata/book/json_codec.py new file mode 100644 index 0000000000..5e13650f0e --- /dev/null +++ b/src/calibre/ebooks/metadata/book/json_codec.py @@ -0,0 +1,125 @@ +''' +Created on 4 Jun 2010 + +@author: charles +''' + +from base64 import b64encode, b64decode +import json +import traceback +from PIL import Image + +from . import SERIALIZABLE_FIELDS +from calibre.constants import filesystem_encoding, preferred_encoding +from calibre.library.field_metadata import FieldMetadata +from calibre.utils.date import parse_date, isoformat, UNDEFINED_DATE + +# Translate datetimes to and from strings. The string form is the datetime in +# UTC. The returned date is also UTC +def string_to_datetime(src): + if src == "None": + return None +# dt = strptime(src, '%d %m %Y %H:%M:%S', assume_utc=True, as_utc=True) +# if dt == UNDEFINED_DATE: +# return None + return parse_date(src) + +def datetime_to_string(dateval): + if dateval is None or dateval == UNDEFINED_DATE: + return "None" +# tt = date_to_utc(dateval).timetuple() +# res = "%02d %02d %04d %02d:%02d:%02d"%(tt.tm_mday, tt.tm_mon, tt.tm_year, +# tt.tm_hour, tt.tm_min, tt.tm_sec) + return isoformat(dateval) + +def encode_thumbnail(thumbnail): + ''' + Encode the image part of a thumbnail, then return the 3 part tuple + ''' + if thumbnail is None: + return None + return (thumbnail[0], thumbnail[1], b64encode(str(thumbnail[2]))) + +def decode_thumbnail(tup): + ''' + Decode an encoded thumbnail into its 3 component parts + ''' + if tup is None: + return None + return (tup[0], tup[1], b64decode(tup[2])) + +class JsonCodec(object): + + def __init__(self): + self.field_metadata = FieldMetadata() + + def encode_to_file(self, file, booklist): + json.dump(self.encode_booklist_metadata(booklist), file, indent=2, encoding='utf-8') + + def encode_booklist_metadata(self, booklist): + result = [] + for book in booklist: + result.append(self.encode_book_metadata(book)) + return result + + def encode_book_metadata(self, book): + result = {} + for key in SERIALIZABLE_FIELDS: + result[key] = self.encode_metadata_attr(book, key) + return result + + def encode_metadata_attr(self, book, key): + if key == 'user_metadata': + meta = book.all_user_metadata + for k in meta: + if meta[k]['datatype'] == 'datetime': + meta[k]['#value#'] = datetime_to_string(meta[k]['#value#']) + return meta + if key in self.field_metadata: + datatype = self.field_metadata[key]['datatype'] + else: + datatype = None + value = book.get(key) + if key == 'thumbnail': + return encode_thumbnail(value) + elif isinstance(value, str): # str includes bytes + enc = filesystem_encoding if key == 'lpath' else preferred_encoding + return value.decode(enc, 'replace') + elif isinstance(value, (list, tuple)): + return [x.decode(preferred_encoding, 'replace') if + isinstance(x, str) else x for x in value] + elif datatype == 'datetime': + return datetime_to_string(value) + else: + return value + + def decode_from_file(self, file, booklist, book_class, prefix): + js = [] + try: + js = json.load(file, encoding='utf-8') + for item in js: + book = book_class(prefix, item.get('lpath', None)) + for key in item.keys(): + meta = self.decode_metadata(key, item[key]) + if key == 'user_metadata': + book.set_all_user_metadata(meta) + else: + setattr(book, key, meta) + booklist.append(book) + except: + print 'exception during JSON decoding' + traceback.print_exc() + booklist = [] + + def decode_metadata(self, key, value): + if key == 'user_metadata': + for k in value: + if value[k]['datatype'] == 'datetime': + value[k]['#value#'] = string_to_datetime(value[k]['#value#']) + return value + elif key in self.field_metadata: + if self.field_metadata[key]['datatype'] == 'datetime': + return string_to_datetime(value) + if key == 'thumbnail': + return decode_thumbnail(value) + return value diff --git a/src/calibre/ebooks/metadata/isbndb.py b/src/calibre/ebooks/metadata/isbndb.py index 356cc3f1b1..b5fc5830c8 100644 --- a/src/calibre/ebooks/metadata/isbndb.py +++ b/src/calibre/ebooks/metadata/isbndb.py @@ -8,7 +8,7 @@ import sys, re from urllib import quote from calibre.utils.config import OptionParser -from calibre.ebooks.metadata import MetaInformation +from calibre.ebooks.metadata.book.base import Metadata from calibre.ebooks.BeautifulSoup import BeautifulStoneSoup from calibre import browser @@ -42,10 +42,10 @@ def fetch_metadata(url, max=100, timeout=5.): return books -class ISBNDBMetadata(MetaInformation): +class ISBNDBMetadata(Metadata): def __init__(self, book): - MetaInformation.__init__(self, None, []) + Metadata.__init__(self, None, []) self.isbn = book.get('isbn13', book.get('isbn')) self.title = book.find('titlelong').string diff --git a/src/calibre/ebooks/metadata/opf2.py b/src/calibre/ebooks/metadata/opf2.py index f93b614ef2..0ab6d3bbc0 100644 --- a/src/calibre/ebooks/metadata/opf2.py +++ b/src/calibre/ebooks/metadata/opf2.py @@ -16,7 +16,8 @@ from lxml import etree from calibre.ebooks.chardet import xml_to_unicode from calibre.constants import __appname__, __version__, filesystem_encoding from calibre.ebooks.metadata.toc import TOC -from calibre.ebooks.metadata import MetaInformation, string_to_authors +from calibre.ebooks.metadata import string_to_authors +from calibre.ebooks.metadata.book.base import Metadata from calibre.utils.date import parse_date, isoformat from calibre.utils.localization import get_lang @@ -926,16 +927,16 @@ class OPF(object): setattr(self, attr, val) -class OPFCreator(MetaInformation): +class OPFCreator(Metadata): - def __init__(self, base_path, *args, **kwargs): + def __init__(self, base_path, other): ''' Initialize. @param base_path: An absolute path to the directory in which this OPF file will eventually be. This is used by the L{create_manifest} method to convert paths to files into relative paths. ''' - MetaInformation.__init__(self, *args, **kwargs) + Metadata.__init__(self, title='', other=other) self.base_path = os.path.abspath(base_path) if self.application_id is None: self.application_id = str(uuid.uuid4()) diff --git a/src/calibre/gui2/dialogs/config/save_template.py b/src/calibre/gui2/dialogs/config/save_template.py index 71eb15f4aa..2157d5b3bf 100644 --- a/src/calibre/gui2/dialogs/config/save_template.py +++ b/src/calibre/gui2/dialogs/config/save_template.py @@ -34,25 +34,24 @@ class SaveTemplate(QWidget, Ui_Form): self.option_name = name def validate(self): - tmpl = preprocess_template(self.opt_template.text()) - fa = {} - for x in FORMAT_ARG_DESCS.keys(): - fa[x]='random long string' - try: - tmpl.format(**fa) - except Exception, err: - error_dialog(self, _('Invalid template'), - '

'+_('The template %s is invalid:')%tmpl + \ - '
'+str(err), show=True) - return False + # TODO: I haven't figured out how to get the custom columns into here, + # so for the moment make all templates valid. return True +# tmpl = preprocess_template(self.opt_template.text()) +# fa = {} +# for x in FORMAT_ARG_DESCS.keys(): +# fa[x]='random long string' +# try: +# tmpl.format(**fa) +# except Exception, err: +# error_dialog(self, _('Invalid template'), +# '

'+_('The template %s is invalid:')%tmpl + \ +# '
'+str(err), show=True) +# return False +# return True def save_settings(self, config, name): val = unicode(self.opt_template.text()) config.set(name, val) self.opt_template.save_history(self.option_name+'_template_history') - - - - diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index 89008735fe..fdf21ecc23 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -372,7 +372,6 @@ class BooksModel(QAbstractTableModel): # {{{ return ans def get_metadata(self, rows, rows_are_ids=False, full_metadata=False): - # Should this add the custom columns? It doesn't at the moment metadata, _full_metadata = [], [] if not rows_are_ids: rows = [self.db.id(row.row()) for row in rows] @@ -1053,7 +1052,7 @@ class DeviceBooksModel(BooksModel): # {{{ if hasattr(cdata, 'image_path'): img.load(cdata.image_path) else: - img.loadFromData(cdata) + img.loadFromData(cdata[2]) if img.isNull(): img = self.default_image data['cover'] = img diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index ef74188bdf..9f21fe0eda 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -509,15 +509,15 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): ''' Convenience method to return metadata as a L{MetaInformation} object. ''' - aum = self.authors(idx, index_is_id=index_is_id) - if aum: aum = [a.strip().replace('|', ',') for a in aum.split(',')] + aut_list = self.authors_with_sort_strings(idx, index_is_id=index_is_id) + aum = [] + aus = {} + for (author, author_sort) in aut_list: + aum.append(author) + aus[author] = author_sort mi = MetaInformation(self.title(idx, index_is_id=index_is_id), aum) mi.author_sort = self.author_sort(idx, index_is_id=index_is_id) - if mi.authors: - mi.author_sort_map = {} - for name, sort in zip(mi.authors, self.authors_sort_strings(idx, - index_is_id)): - mi.author_sort_map[name] = sort + mi.author_sort_map = aus mi.comments = self.comments(idx, index_is_id=index_is_id) mi.publisher = self.publisher(idx, index_is_id=index_is_id) mi.timestamp = self.timestamp(idx, index_is_id=index_is_id) @@ -534,6 +534,10 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): mi.isbn = self.isbn(idx, index_is_id=index_is_id) id = idx if index_is_id else self.id(idx) mi.application_id = id + for key,meta in self.field_metadata.iteritems(): + if meta['is_custom']: + mi.set_user_metadata(key, meta) + mi.set(key, self.get_custom(idx, label=meta['label'], index_is_id=index_is_id)) if get_cover: mi.cover = self.cover(id, index_is_id=True, as_path=True) return mi @@ -1049,6 +1053,19 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): result.append(sort) return result + # Given a book, return the map of author sort strings for the book's authors + def authors_with_sort_strings(self, id, index_is_id=False): + id = id if index_is_id else self.id(id) + aut_strings = self.conn.get(''' + SELECT authors.name, authors.sort + FROM authors, books_authors_link as bl + WHERE bl.book=? and authors.id=bl.author + ORDER BY bl.id''', (id,)) + result = [] + for (author, sort,) in aut_strings: + result.append((author.replace('|', ','), sort)) + return result + # Given a book, return the author_sort string for authors of the book def author_sort_from_book(self, id, index_is_id=False): auts = self.authors_sort_strings(id, index_is_id) diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py index 15020855f7..258ea7ba9e 100644 --- a/src/calibre/library/save_to_disk.py +++ b/src/calibre/library/save_to_disk.py @@ -105,6 +105,8 @@ def safe_format(x, format_args): pass except AttributeError: # Thrown if user used a non existing attribute pass + except KeyError: # Thrown if user used custom field w/value None + pass return '' def get_components(template, mi, id, timefmt='%b %Y', length=250, @@ -113,13 +115,12 @@ def get_components(template, mi, id, timefmt='%b %Y', length=250, library_order = tweaks['save_template_title_series_sorting'] == 'library_order' tsfmt = title_sort if library_order else lambda x: x format_args = dict(**FORMAT_ARGS) + format_args.update(mi.all_attributes) if mi.title: format_args['title'] = tsfmt(mi.title) if mi.authors: format_args['authors'] = mi.format_authors() format_args['author'] = format_args['authors'] - if mi.author_sort: - format_args['author_sort'] = mi.author_sort if mi.tags: format_args['tags'] = mi.format_tags() if format_args['tags'].startswith('/'): @@ -132,15 +133,21 @@ def get_components(template, mi, id, timefmt='%b %Y', length=250, template = re.sub(r'\{series_index[^}]*?\}', '', template) if mi.rating is not None: format_args['rating'] = mi.format_rating() - if mi.isbn: - format_args['isbn'] = mi.isbn - if mi.publisher: - format_args['publisher'] = mi.publisher if hasattr(mi.timestamp, 'timetuple'): format_args['timestamp'] = strftime(timefmt, mi.timestamp.timetuple()) if hasattr(mi.pubdate, 'timetuple'): format_args['pubdate'] = strftime(timefmt, mi.pubdate.timetuple()) format_args['id'] = str(id) + + # These are not necessary any more. The values are set by + # 'format_args.update' above, and there is no special formatting +# if mi.author_sort: +# format_args['author_sort'] = mi.author_sort +# if mi.isbn: +# format_args['isbn'] = mi.isbn +# if mi.publisher: +# format_args['publisher'] = mi.publisher + components = [x.strip() for x in template.split('/') if x.strip()] components = [safe_format(x, format_args) for x in components] components = [sanitize_func(x) for x in components if x]