Initial attempt, including full cached metadata, json serialization, and custom fields in save paths. Custom fields in collections probably work, but they haven't been tested.

src/calibre/devices/apple/driver.py

@@ -13,7 +13,8 @@ from calibre.devices.errors import UserFeedback
 from calibre.devices.usbms.deviceconfig import DeviceConfig
 from calibre.devices.interface import DevicePlugin
 from calibre.ebooks.BeautifulSoup import BeautifulSoup
-from calibre.ebooks.metadata import MetaInformation, authors_to_string
+from calibre.ebooks.metadata import authors_to_string
+from calibre.ebooks.metadata.book.base import Metadata
 from calibre.ebooks.metadata.epub import set_metadata
 from calibre.library.server.utils import strftime
 from calibre.utils.config import config_dir
@@ -2998,14 +2999,14 @@ class BookList(list):
         '''
         return {}
 
-class Book(MetaInformation):
+class Book(Metadata):
     '''
     A simple class describing a book in the iTunes Books Library.
     - See ebooks.metadata.__init__ for all fields
     '''
     def __init__(self,title,author):
 
-        MetaInformation.__init__(self, title, authors=[author])
+        Metadata.__init__(self, title, authors=[author])
 
     @dynamic_property
     def title_sorter(self):

src/calibre/devices/interface.py

@@ -316,7 +316,7 @@ class DevicePlugin(Plugin):
                       being uploaded to the device.
         :param names: A list of file names that the books should have
                       once uploaded to the device. len(names) == len(files)
-        :param metadata: If not None, it is a list of :class:`MetaInformation` objects.
+        :param metadata: If not None, it is a list of :class:`Metadata` objects.
                          The idea is to use the metadata to determine where on the device to
                          put the book. len(metadata) == len(files). Apart from the regular
                          cover (path to cover), there may also be a thumbnail attribute, which should
@@ -335,7 +335,7 @@ class DevicePlugin(Plugin):
         the device.
 
         :param locations: Result of a call to L{upload_books}
-        :param metadata: List of :class:`MetaInformation` objects, same as for
+        :param metadata: List of :class:`Metadata` objects, same as for
                          :meth:`upload_books`.
         :param booklists: A tuple containing the result of calls to
                           (:meth:`books(oncard=None)`,

src/calibre/devices/kobo/books.py

@@ -7,11 +7,11 @@ import os
 import re
 import time
 
-from calibre.ebooks.metadata import MetaInformation
+from calibre.ebooks.metadata.book.base import Metadata
 from calibre.constants import filesystem_encoding, preferred_encoding
 from calibre import isbytestring
 
-class Book(MetaInformation):
+class Book(Metadata):
 
     BOOK_ATTRS = ['lpath', 'size', 'mime', 'device_collections', '_new_book']
 
@@ -23,9 +23,9 @@ class Book(MetaInformation):
         'uuid',
     ]
 
-    def __init__(self, prefix, lpath, title, authors, mime, date, ContentType, thumbnail_name, other=None):
+    def __init__(self, prefix, lpath, title, authors, mime, date, ContentType,
-
+                 thumbnail_name, other=None):
-        MetaInformation.__init__(self, '')
+        Metadata.__init__(self, '')
         self.device_collections = []
         self._new_book = False
 
@@ -53,7 +53,6 @@ class Book(MetaInformation):
                 self.datetime = time.gmtime(os.path.getctime(self.path))
         except:
             self.datetime = time.gmtime()
-
         if thumbnail_name is not None:
             self.thumbnail = ImageWrapper(thumbnail_name)
         self.tags = []
@@ -90,7 +89,7 @@ class Book(MetaInformation):
         in C{other} takes precedence, unless the information in C{other} is NULL.
         '''
 
-        MetaInformation.smart_update(self, other)
+        Metadata.smart_update(self, other)
 
         for attr in self.BOOK_ATTRS:
             if hasattr(other, attr):

src/calibre/devices/usbms/books.py

@@ -6,29 +6,18 @@ __docformat__ = 'restructuredtext en'
 
 import os, re, time, sys
 
-from calibre.ebooks.metadata import MetaInformation
+from calibre.ebooks.metadata.book.base import Metadata
 from calibre.devices.mime import mime_type_ext
 from calibre.devices.interface import BookList as _BookList
 from calibre.constants import filesystem_encoding, preferred_encoding
 from calibre import isbytestring
 from calibre.utils.config import prefs
 
-class Book(MetaInformation):
+class Book(Metadata):
-
-    BOOK_ATTRS = ['lpath', 'size', 'mime', 'device_collections', '_new_book']
-
-    JSON_ATTRS = [
-        'lpath', 'title', 'authors', 'mime', 'size', 'tags', 'author_sort',
-        'title_sort', 'comments', 'category', 'publisher', 'series',
-        'series_index', 'rating', 'isbn', 'language', 'application_id',
-        'book_producer', 'lccn', 'lcc', 'ddc', 'rights', 'publication_type',
-        'uuid',
-    ]
-
     def __init__(self, prefix, lpath, size=None, other=None):
         from calibre.ebooks.metadata.meta import path_to_ext
 
-        MetaInformation.__init__(self, '')
+        Metadata.__init__(self, '')
 
         self._new_book = False
         self.device_collections = []
@@ -72,32 +61,6 @@ class Book(MetaInformation):
     def thumbnail(self):
         return None
 
-    def smart_update(self, other, replace_metadata=False):
-        '''
-        Merge the information in C{other} into self. In case of conflicts, the information
-        in C{other} takes precedence, unless the information in C{other} is NULL.
-        '''
-
-        MetaInformation.smart_update(self, other, replace_metadata)
-
-        for attr in self.BOOK_ATTRS:
-            if hasattr(other, attr):
-                val = getattr(other, attr, None)
-                setattr(self, attr, val)
-
-    def to_json(self):
-        json = {}
-        for attr in self.JSON_ATTRS:
-            val = getattr(self, attr)
-            if isbytestring(val):
-                enc = filesystem_encoding if attr == 'lpath' else preferred_encoding
-                val = val.decode(enc, 'replace')
-            elif isinstance(val, (list, tuple)):
-                val = [x.decode(preferred_encoding, 'replace') if
-                        isbytestring(x) else x for x in val]
-            json[attr] = val
-        return json
-
 class BookList(_BookList):
 
     def __init__(self, oncard, prefix, settings):

src/calibre/devices/usbms/driver.py

@@ -13,7 +13,6 @@ for a particular device.
 import os
 import re
 import time
-import json
 from itertools import cycle
 
 from calibre import prints, isbytestring
@@ -21,6 +20,7 @@ from calibre.constants import filesystem_encoding, DEBUG
 from calibre.devices.usbms.cli import CLI
 from calibre.devices.usbms.device import Device
 from calibre.devices.usbms.books import BookList, Book
+from calibre.ebooks.metadata.book.json_codec import JsonCodec
 
 BASE_TIME = None
 def debug_print(*args):
@@ -288,6 +288,7 @@ class USBMS(CLI, Device):
     # at the end just before the return
     def sync_booklists(self, booklists, end_session=True):
         debug_print('USBMS: starting sync_booklists')
+        json_codec = JsonCodec()
 
         if not os.path.exists(self.normalize_path(self._main_prefix)):
             os.makedirs(self.normalize_path(self._main_prefix))
@@ -296,10 +297,8 @@ class USBMS(CLI, Device):
             if prefix is not None and isinstance(booklists[listid], self.booklist_class):
                 if not os.path.exists(prefix):
                     os.makedirs(self.normalize_path(prefix))
-                js = [item.to_json() for item in booklists[listid] if
-                        hasattr(item, 'to_json')]
                 with open(self.normalize_path(os.path.join(prefix, self.METADATA_CACHE)), 'wb') as f:
-                    f.write(json.dumps(js, indent=2, encoding='utf-8'))
+                    json_codec.encode_to_file(f, booklists[listid])
         write_prefix(self._main_prefix, 0)
         write_prefix(self._card_a_prefix, 1)
         write_prefix(self._card_b_prefix, 2)
@@ -345,19 +344,13 @@ class USBMS(CLI, Device):
 
     @classmethod
     def parse_metadata_cache(cls, bl, prefix, name):
-        # bl = cls.booklist_class()
+        json_codec = JsonCodec()
-        js = []
         need_sync = False
         cache_file = cls.normalize_path(os.path.join(prefix, name))
         if os.access(cache_file, os.R_OK):
             try:
                 with open(cache_file, 'rb') as f:
-                    js = json.load(f, encoding='utf-8')
+                    json_codec.decode_from_file(f, bl, cls.book_class, prefix)
-                for item in js:
-                    book = cls.book_class(prefix, item.get('lpath', None))
-                    for key in item.keys():
-                        setattr(book, key, item[key])
-                    bl.append(book)
             except:
                 import traceback
                 traceback.print_exc()
@@ -392,7 +385,7 @@ class USBMS(CLI, Device):
 
     @classmethod
     def book_from_path(cls, prefix, lpath):
-        from calibre.ebooks.metadata import MetaInformation
+        from calibre.ebooks.metadata.book.base import Metadata
 
         if cls.settings().read_metadata or cls.MUST_READ_METADATA:
             mi = cls.metadata_from_path(cls.normalize_path(os.path.join(prefix, lpath)))
@@ -401,7 +394,7 @@ class USBMS(CLI, Device):
             mi = metadata_from_filename(cls.normalize_path(os.path.basename(lpath)),
                                         cls.build_template_regexp())
         if mi is None:
-            mi = MetaInformation(os.path.splitext(os.path.basename(lpath))[0],
+            mi = Metadata(os.path.splitext(os.path.basename(lpath))[0],
                     [_('Unknown')])
         size = os.stat(cls.normalize_path(os.path.join(prefix, lpath))).st_size
         book = cls.book_class(prefix, lpath, other=mi, size=size)

src/calibre/ebooks/metadata/__init__.py

@@ -221,214 +221,18 @@ class ResourceCollection(object):
 
 
 
-class MetaInformation(object):
+def MetaInformation(title, authors=(_('Unknown'),)):
-    '''Convenient encapsulation of book metadata'''
+    ''' Convenient encapsulation of book metadata, needed for compatibility
-
-    @staticmethod
-    def copy(mi):
-        ans = MetaInformation(mi.title, mi.authors)
-        for attr in ('author_sort', 'title_sort', 'comments', 'category',
-                     'publisher', 'series', 'series_index', 'rating',
-                     'isbn', 'tags', 'cover_data', 'application_id', 'guide',
-                     'manifest', 'spine', 'toc', 'cover', 'language',
-                     'book_producer', 'timestamp', 'lccn', 'lcc', 'ddc',
-                     'author_sort_map',
-                     'pubdate', 'rights', 'publication_type', 'uuid'):
-            if hasattr(mi, attr):
-                setattr(ans, attr, getattr(mi, attr))
-
-    def __init__(self, title, authors=(_('Unknown'),)):
-        '''
         @param title: title or ``_('Unknown')`` or a MetaInformation object
         @param authors: List of strings or []
     '''
+    from calibre.ebooks.metadata.book.base import Metadata
     mi = None
     if hasattr(title, 'title') and hasattr(title, 'authors'):
         mi = title
         title = mi.title
         authors = mi.authors
-        self.title = title
+    return Metadata(title, authors, mi)
-        self.author = list(authors) if authors else []# Needed for backward compatibility
-        #: List of strings or []
-        self.authors = list(authors) if authors else []
-        self.tags = getattr(mi, 'tags', [])
-        #: mi.cover_data = (ext, data)
-        self.cover_data   = getattr(mi, 'cover_data', (None, None))
-        self.author_sort_map = getattr(mi, 'author_sort_map', {})
-
-        for x in ('author_sort', 'title_sort', 'comments', 'category', 'publisher',
-                  'series', 'series_index', 'rating', 'isbn', 'language',
-                  'application_id', 'manifest', 'toc', 'spine', 'guide', 'cover',
-                  'book_producer', 'timestamp', 'lccn', 'lcc', 'ddc', 'pubdate',
-                  'rights', 'publication_type', 'uuid',
-                  ):
-            setattr(self, x, getattr(mi, x, None))
-
-    def print_all_attributes(self):
-        for x in ('title','author', 'author_sort', 'title_sort', 'comments', 'category', 'publisher',
-                  'series', 'series_index', 'tags', 'rating', 'isbn', 'language',
-                  'application_id', 'manifest', 'toc', 'spine', 'guide', 'cover',
-                  'book_producer', 'timestamp', 'lccn', 'lcc', 'ddc', 'pubdate',
-                  'rights', 'publication_type', 'uuid', 'author_sort_map'
-                  ):
-            prints(x, getattr(self, x, 'None'))
-
-    def smart_update(self, mi, replace_metadata=False):
-        '''
-        Merge the information in C{mi} into self. In case of conflicts, the
-        information in C{mi} takes precedence, unless the information in mi is
-        NULL. If replace_metadata is True, then the information in mi always
-        takes precedence.
-        '''
-        if mi.title and mi.title != _('Unknown'):
-            self.title = mi.title
-
-        if mi.authors and mi.authors[0] != _('Unknown'):
-            self.authors = mi.authors
-
-        for attr in ('author_sort', 'title_sort', 'category',
-                     'publisher', 'series', 'series_index', 'rating',
-                     'isbn', 'application_id', 'manifest', 'spine', 'toc',
-                     'cover', 'guide', 'book_producer',
-                     'timestamp', 'lccn', 'lcc', 'ddc', 'pubdate', 'rights',
-                     'publication_type', 'uuid'):
-            if replace_metadata:
-                setattr(self, attr, getattr(mi, attr, 1.0 if \
-                        attr == 'series_index' else None))
-            elif hasattr(mi, attr):
-                val = getattr(mi, attr)
-                if val is not None:
-                    setattr(self, attr, val)
-
-        if replace_metadata:
-            self.tags = mi.tags
-        elif mi.tags:
-            self.tags += mi.tags
-        self.tags = list(set(self.tags))
-
-        if mi.author_sort_map:
-            self.author_sort_map.update(mi.author_sort_map)
-
-        if getattr(mi, 'cover_data', False):
-            other_cover = mi.cover_data[-1]
-            self_cover = self.cover_data[-1] if self.cover_data else ''
-            if not self_cover: self_cover = ''
-            if not other_cover: other_cover = ''
-            if len(other_cover) > len(self_cover):
-                self.cover_data = mi.cover_data
-
-        if replace_metadata:
-            self.comments = getattr(mi, 'comments', '')
-        else:
-            my_comments = getattr(self, 'comments', '')
-            other_comments = getattr(mi, 'comments', '')
-            if not my_comments:
-                my_comments = ''
-            if not other_comments:
-                other_comments = ''
-            if len(other_comments.strip()) > len(my_comments.strip()):
-                self.comments = other_comments
-
-        other_lang = getattr(mi, 'language', None)
-        if other_lang and other_lang.lower() != 'und':
-            self.language = other_lang
-
-
-    def format_series_index(self):
-        try:
-            x = float(self.series_index)
-        except ValueError:
-            x = 1
-        return fmt_sidx(x)
-
-    def authors_from_string(self, raw):
-        self.authors = string_to_authors(raw)
-
-    def format_authors(self):
-        return authors_to_string(self.authors)
-
-    def format_tags(self):
-        return u', '.join([unicode(t) for t in self.tags])
-
-    def format_rating(self):
-        return unicode(self.rating)
-
-    def __unicode__(self):
-        ans = []
-        def fmt(x, y):
-            ans.append(u'%-20s: %s'%(unicode(x), unicode(y)))
-
-        fmt('Title', self.title)
-        if self.title_sort:
-            fmt('Title sort', self.title_sort)
-        if self.authors:
-            fmt('Author(s)',  authors_to_string(self.authors) + \
-               ((' [' + self.author_sort + ']') if self.author_sort else ''))
-        if self.publisher:
-            fmt('Publisher', self.publisher)
-        if getattr(self, 'book_producer', False):
-            fmt('Book Producer', self.book_producer)
-        if self.category:
-            fmt('Category', self.category)
-        if self.comments:
-            fmt('Comments', self.comments)
-        if self.isbn:
-            fmt('ISBN', self.isbn)
-        if self.tags:
-            fmt('Tags', u', '.join([unicode(t) for t in self.tags]))
-        if self.series:
-            fmt('Series', self.series + ' #%s'%self.format_series_index())
-        if self.language:
-            fmt('Language', self.language)
-        if self.rating is not None:
-            fmt('Rating', self.rating)
-        if self.timestamp is not None:
-            fmt('Timestamp', isoformat(self.timestamp))
-        if self.pubdate is not None:
-            fmt('Published', isoformat(self.pubdate))
-        if self.rights is not None:
-            fmt('Rights', unicode(self.rights))
-        if self.lccn:
-            fmt('LCCN', unicode(self.lccn))
-        if self.lcc:
-            fmt('LCC', unicode(self.lcc))
-        if self.ddc:
-            fmt('DDC', unicode(self.ddc))
-
-        return u'\n'.join(ans)
-
-    def to_html(self):
-        ans = [(_('Title'), unicode(self.title))]
-        ans += [(_('Author(s)'), (authors_to_string(self.authors) if self.authors else _('Unknown')))]
-        ans += [(_('Publisher'), unicode(self.publisher))]
-        ans += [(_('Producer'), unicode(self.book_producer))]
-        ans += [(_('Comments'), unicode(self.comments))]
-        ans += [('ISBN', unicode(self.isbn))]
-        if self.lccn:
-            ans += [('LCCN', unicode(self.lccn))]
-        if self.lcc:
-            ans += [('LCC', unicode(self.lcc))]
-        if self.ddc:
-            ans += [('DDC', unicode(self.ddc))]
-        ans += [(_('Tags'), u', '.join([unicode(t) for t in self.tags]))]
-        if self.series:
-            ans += [(_('Series'), unicode(self.series)+ ' #%s'%self.format_series_index())]
-        ans += [(_('Language'), unicode(self.language))]
-        if self.timestamp is not None:
-            ans += [(_('Timestamp'), unicode(self.timestamp.isoformat(' ')))]
-        if self.pubdate is not None:
-            ans += [(_('Published'), unicode(self.pubdate.isoformat(' ')))]
-        if self.rights is not None:
-            ans += [(_('Rights'), unicode(self.rights))]
-        for i, x in enumerate(ans):
-            ans[i] = u'<tr><td><b>%s</b></td><td>%s</td></tr>'%x
-        return u'<table>%s</table>'%u'\n'.join(ans)
-
-    def __str__(self):
-        return self.__unicode__().encode('utf-8')
-
-    def __nonzero__(self):
-        return bool(self.title or self.author or self.comments or self.tags)
 
 def check_isbn10(isbn):
     try:

src/calibre/ebooks/metadata/book/__init__.py

@@ -24,6 +24,8 @@ SOCIAL_METADATA_FIELDS = frozenset([
     # For example: {'isbn':'123456789', 'doi':'xxxx', ... }
     'classifiers',
     'isbn', # Pseudo field for convenience, should get/set isbn classifier
+    # TODO: not sure what this is, but it is used by OPF
+    'category',
 
 ])
 
@@ -69,7 +71,8 @@ BOOK_STRUCTURE_FIELDS = frozenset([
     ])
 
 USER_METADATA_FIELDS = frozenset([
-    # A dict of a form to be specified
+    # A dict of dicts similar to field_metadata. Each field description dict
+    # also contains a value field with the key #value#.
     'user_metadata',
 ])
 
@@ -86,16 +89,42 @@ DEVICE_METADATA_FIELDS = frozenset([
 CALIBRE_METADATA_FIELDS = frozenset([
     # An application id
     # Semantics to be defined. Is it a db key? a db name + key? A uuid?
+    # (It is currently set to the db_id.)
     'application_id',
+    # the calibre primary key of the item. May want to remove this once Sony's no longer use it
+    'db_id',
     ]
 )
 
+ALL_METADATA_FIELDS =      SOCIAL_METADATA_FIELDS.union(
+                           PUBLICATION_METADATA_FIELDS).union(
+                           BOOK_STRUCTURE_FIELDS).union(
+                           USER_METADATA_FIELDS).union(
+                           DEVICE_METADATA_FIELDS).union(
+                           CALIBRE_METADATA_FIELDS)
+
+# All fields except custom fields
+STANDARD_METADATA_FIELDS = SOCIAL_METADATA_FIELDS.union(
+                           PUBLICATION_METADATA_FIELDS).union(
+                           BOOK_STRUCTURE_FIELDS).union(
+                           DEVICE_METADATA_FIELDS).union(
+                           CALIBRE_METADATA_FIELDS)
+
+# Metadata fields that smart update should copy without special handling
+COPYABLE_METADATA_FIELDS = SOCIAL_METADATA_FIELDS.union(
+                           PUBLICATION_METADATA_FIELDS).union(
+                           BOOK_STRUCTURE_FIELDS).union(
+                           DEVICE_METADATA_FIELDS).union(
+                           CALIBRE_METADATA_FIELDS) - \
+                           frozenset(['title', 'authors', 'comments', 'cover_data'])
 
 SERIALIZABLE_FIELDS =      SOCIAL_METADATA_FIELDS.union(
                            USER_METADATA_FIELDS).union(
                            PUBLICATION_METADATA_FIELDS).union(
                            CALIBRE_METADATA_FIELDS).union(
-        frozenset(['lpath'])) # I don't think we need device_collections
+                           DEVICE_METADATA_FIELDS) - \
+                           frozenset(['device_collections'])
+                      # I don't think we need device_collections
 
 # Serialization of covers/thumbnails will have to be handled carefully, maybe
 # as an option to the serializer class

src/calibre/ebooks/metadata/book/base.py

@@ -6,8 +6,13 @@ __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
 import copy
+import traceback
+
+from calibre import prints
+from calibre.ebooks.metadata.book import COPYABLE_METADATA_FIELDS
+from calibre.ebooks.metadata.book import STANDARD_METADATA_FIELDS
+from calibre.utils.date import isoformat
 
-from calibre.ebooks.metadata.book import RESERVED_METADATA_FIELDS
 
 NULL_VALUES = {
                 'user_metadata': {},
@@ -24,98 +29,313 @@ NULL_VALUES = {
 class Metadata(object):
 
     '''
-    This class must expose a superset of the API of MetaInformation in terms
+    A class representing all the metadata for a book.
-    of attribute access and methods. Only the __init__ method is different.
-    MetaInformation will simply become a function that creates and fills in
-    the attributes of this class.
 
     Please keep the method based API of this class to a minimum. Every method
     becomes a reserved field name.
     '''
 
-    def __init__(self):
+    def __init__(self, title, authors=(_('Unknown'),), other=None):
+        '''
+        @param title: title or ``_('Unknown')``
+        @param authors: List of strings or []
+        @param other: None or a metadata object
+        '''
         object.__setattr__(self, '_data', copy.deepcopy(NULL_VALUES))
+        if other is not None:
+            self.smart_update(other)
+        else:
+            if title:
+                self.title = title
+            if authors:
+                #: List of strings or []
+                self.author = list(authors) if authors else []# Needed for backward compatibility
+                self.authors = list(authors) if authors else []
 
     def __getattribute__(self, field):
         _data = object.__getattribute__(self, '_data')
-        if field in RESERVED_METADATA_FIELDS:
+        if field in STANDARD_METADATA_FIELDS:
             return _data.get(field, None)
         try:
             return object.__getattribute__(self, field)
         except AttributeError:
             pass
         if field in _data['user_metadata'].iterkeys():
-            # TODO: getting user metadata values
+            return _data['user_metadata'][field]['#value#']
-            pass
         raise AttributeError(
                 'Metadata object has no attribute named: '+ repr(field))
 
-
     def __setattr__(self, field, val):
         _data = object.__getattribute__(self, '_data')
-        if field in RESERVED_METADATA_FIELDS:
+        if field in STANDARD_METADATA_FIELDS:
-            if field != 'user_metadata':
             if not val:
-                    val = NULL_VALUES[field]
+                val = NULL_VALUES.get(field, None)
             _data[field] = val
-            else:
-                raise AttributeError('You cannot set user_metadata directly.')
         elif field in _data['user_metadata'].iterkeys():
-            # TODO: Setting custom column values
+            _data['user_metadata'][field]['#value#'] = val
-            pass
         else:
             # You are allowed to stick arbitrary attributes onto this object as
             # long as they dont conflict with global or user metadata names
             # Don't abuse this privilege
             self.__dict__[field] = val
 
+    def get(self, field):
+        return self.__getattribute__(field)
+
+    def set(self, field, val):
+        self.__setattr__(field, val)
+
     @property
-    def user_metadata_names(self):
+    def user_metadata_keys(self):
         'The set of user metadata names this object knows about'
         _data = object.__getattribute__(self, '_data')
         return frozenset(_data['user_metadata'].iterkeys())
 
-    # Old MetaInformation API {{{
+    @property
-    def copy(self):
+    def all_user_metadata(self):
-        pass
+        '''
+        return a dict containing all the custom field metadata associated with
+        the book. Return a deep copy, just in case the user wants to change
+        values in the dict (json does).
+        '''
+        _data = object.__getattribute__(self, '_data')
+        _data = _data['user_metadata']
+        res = {}
+        for k in _data:
+            res[k] = copy.deepcopy(_data[k])
+        return res
+
+    def get_user_metadata(self, field):
+        '''
+        return field metadata from the object if it is there. Otherwise return
+        None. field is the key name, not the label. Return a shallow copy,
+        just in case the user wants to change values in the dict (json does).
+        '''
+        _data = object.__getattribute__(self, '_data')
+        _data = _data['user_metadata']
+        if field in _data:
+            return copy.deepcopy(_data[field])
+        return None
+
+    def set_all_user_metadata(self, metadata):
+        '''
+        store custom field metadata into the object. Field is the key name
+        not the label
+        '''
+        if metadata is None:
+            traceback.print_stack()
+        else:
+            for key in metadata:
+                self.set_user_metadata(key, metadata[key])
+
+    def set_user_metadata(self, field, metadata):
+        '''
+        store custom field metadata for one column into the object. Field is
+        the key name not the label
+        '''
+        if field is not None:
+            if metadata is None:
+                traceback.print_stack()
+            metadata = copy.deepcopy(metadata)
+            if '#value#' not in metadata:
+                metadata['#value#'] = None
+            _data = object.__getattribute__(self, '_data')
+            _data['user_metadata'][field] = metadata
+
+    @property
+    def all_attributes(self):
+        result = {}
+        _data = object.__getattribute__(self, '_data')
+        for attr in STANDARD_METADATA_FIELDS:
+            v = _data.get(attr, None)
+            if v is not None:
+                result[attr] = v
+        for attr in self.user_metadata_keys:
+            if self.get(attr) is not None:
+                result[attr] = self.get(attr)
+        return result
+
+    # Old Metadata API {{{
+    @staticmethod
+    def copy(mi):
+        ans = Metadata(mi.title, mi.authors)
+        for attr in STANDARD_METADATA_FIELDS:
+            if hasattr(mi, attr):
+                setattr(ans, attr, copy.deepcopy(getattr(mi, attr)))
+        for x in mi.user_metadata_keys:
+            meta = mi.get_user_metadata(x)
+            if meta is not None:
+                ans.set_user_metadata(x, meta) # get... did the deep copy
 
     def print_all_attributes(self):
-        pass
+        for x in STANDARD_METADATA_FIELDS:
+            prints('%s:'%x, getattr(self, x, 'None'))
+        for x in self.user_metadata_keys:
+            meta = self.get_user_metadata(x)
+            if meta is not None:
+                prints(x, meta)
+        prints('--------------')
 
     def smart_update(self, other, replace_metadata=False):
-        pass
+        '''
+        Merge the information in C{other} into self. In case of conflicts, the information
+        in C{other} takes precedence, unless the information in other is NULL.
+        '''
+        if other.title and other.title != _('Unknown'):
+            self.title = other.title
+
+        if other.authors and other.authors[0] != _('Unknown'):
+            self.authors = other.authors
+
+        for attr in COPYABLE_METADATA_FIELDS:
+            if replace_metadata:
+                setattr(self, attr, getattr(other, attr, 1.0 if \
+                        attr == 'series_index' else None))
+            elif hasattr(other, attr):
+                val = getattr(other, attr)
+                if val is not None:
+                    setattr(self, attr, copy.deepcopy(val))
+
+        if replace_metadata:
+            self.tags = other.tags
+        elif other.tags:
+            self.tags += other.tags
+        self.tags = list(set(self.tags))
+
+        if getattr(other, 'author_sort_map', None):
+            self.author_sort_map.update(other.author_sort_map)
+
+        if getattr(other, 'cover_data', False):
+            other_cover = other.cover_data[-1]
+            self_cover = self.cover_data[-1] if self.cover_data else ''
+            if not self_cover: self_cover = ''
+            if not other_cover: other_cover = ''
+            if len(other_cover) > len(self_cover):
+                self.cover_data = other.cover_data
+
+        if getattr(other, 'user_metadata_keys', None):
+            for x in other.user_metadata_keys:
+                meta = other.get_user_metadata(x)
+                if meta is not None or replace_metadata:
+                    self.set_user_metadata(x, meta) # get... did the deepcopy
+
+        if replace_metadata:
+            self.comments = getattr(other, 'comments', '')
+        else:
+            my_comments = getattr(self, 'comments', '')
+            other_comments = getattr(other, 'comments', '')
+            if not my_comments:
+                my_comments = ''
+            if not other_comments:
+                other_comments = ''
+            if len(other_comments.strip()) > len(my_comments.strip()):
+                self.comments = other_comments
+
+        other_lang = getattr(other, 'language', None)
+        if other_lang and other_lang.lower() != 'und':
+            self.language = other_lang
+
 
     def format_series_index(self):
-        pass
+        from calibre.ebooks.metadata import fmt_sidx
+        try:
+            x = float(self.series_index)
+        except ValueError:
+            x = 1
+        return fmt_sidx(x)
 
     def authors_from_string(self, raw):
-        pass
+        from calibre.ebooks.metadata import string_to_authors
+        self.authors = string_to_authors(raw)
 
     def format_authors(self):
-        pass
+        from calibre.ebooks.metadata import authors_to_string
+        return authors_to_string(self.authors)
 
     def format_tags(self):
-        pass
+        return u', '.join([unicode(t) for t in self.tags])
 
     def format_rating(self):
         return unicode(self.rating)
 
     def __unicode__(self):
-        pass
+        from calibre.ebooks.metadata import authors_to_string
+        ans = []
+        def fmt(x, y):
+            ans.append(u'%-20s: %s'%(unicode(x), unicode(y)))
+
+        fmt('Title', self.title)
+        if self.title_sort:
+            fmt('Title sort', self.title_sort)
+        if self.authors:
+            fmt('Author(s)',  authors_to_string(self.authors) + \
+               ((' [' + self.author_sort + ']') if self.author_sort else ''))
+        if self.publisher:
+            fmt('Publisher', self.publisher)
+        if getattr(self, 'book_producer', False):
+            fmt('Book Producer', self.book_producer)
+        if self.category:
+            fmt('Category', self.category)
+        if self.comments:
+            fmt('Comments', self.comments)
+        if self.isbn:
+            fmt('ISBN', self.isbn)
+        if self.tags:
+            fmt('Tags', u', '.join([unicode(t) for t in self.tags]))
+        if self.series:
+            fmt('Series', self.series + ' #%s'%self.format_series_index())
+        if self.language:
+            fmt('Language', self.language)
+        if self.rating is not None:
+            fmt('Rating', self.rating)
+        if self.timestamp is not None:
+            fmt('Timestamp', isoformat(self.timestamp))
+        if self.pubdate is not None:
+            fmt('Published', isoformat(self.pubdate))
+        if self.rights is not None:
+            fmt('Rights', unicode(self.rights))
+        if self.lccn:
+            fmt('LCCN', unicode(self.lccn))
+        if self.lcc:
+            fmt('LCC', unicode(self.lcc))
+        if self.ddc:
+            fmt('DDC', unicode(self.ddc))
+        # CUSTFIELD: What to do about custom fields?
+        return u'\n'.join(ans)
 
     def to_html(self):
-        pass
+        from calibre.ebooks.metadata import authors_to_string
+        ans = [(_('Title'), unicode(self.title))]
+        ans += [(_('Author(s)'), (authors_to_string(self.authors) if self.authors else _('Unknown')))]
+        ans += [(_('Publisher'), unicode(self.publisher))]
+        ans += [(_('Producer'), unicode(self.book_producer))]
+        ans += [(_('Comments'), unicode(self.comments))]
+        ans += [('ISBN', unicode(self.isbn))]
+        if self.lccn:
+            ans += [('LCCN', unicode(self.lccn))]
+        if self.lcc:
+            ans += [('LCC', unicode(self.lcc))]
+        if self.ddc:
+            ans += [('DDC', unicode(self.ddc))]
+        ans += [(_('Tags'), u', '.join([unicode(t) for t in self.tags]))]
+        if self.series:
+            ans += [(_('Series'), unicode(self.series)+ ' #%s'%self.format_series_index())]
+        ans += [(_('Language'), unicode(self.language))]
+        if self.timestamp is not None:
+            ans += [(_('Timestamp'), unicode(self.timestamp.isoformat(' ')))]
+        if self.pubdate is not None:
+            ans += [(_('Published'), unicode(self.pubdate.isoformat(' ')))]
+        if self.rights is not None:
+            ans += [(_('Rights'), unicode(self.rights))]
+        for i, x in enumerate(ans):
+            ans[i] = u'<tr><td><b>%s</b></td><td>%s</td></tr>'%x
+        # CUSTFIELD: What to do about custom fields
+        return u'<table>%s</table>'%u'\n'.join(ans)
 
     def __str__(self):
         return self.__unicode__().encode('utf-8')
 
     def __nonzero__(self):
-        return True
+        return bool(self.title or self.author or self.comments or self.tags)
 
     # }}}
-
-# We don't need reserved field names for this object any more. Lets just use a
-# protocol like the last char of a user field label should be _ when using this
-# object
-# So mi.tags returns the builtin tags and mi.tags_ returns the user tags
-

src/calibre/ebooks/metadata/book/json_codec.py

@@ -0,0 +1,125 @@
+'''
+Created on 4 Jun 2010
+
+@author: charles
+'''
+
+from base64 import b64encode, b64decode
+import json
+import traceback
+from PIL import Image
+
+from . import SERIALIZABLE_FIELDS
+from calibre.constants import filesystem_encoding, preferred_encoding
+from calibre.library.field_metadata import FieldMetadata
+from calibre.utils.date import parse_date, isoformat, UNDEFINED_DATE
+
+# Translate datetimes to and from strings. The string form is the datetime in
+# UTC. The returned date is also UTC
+def string_to_datetime(src):
+    if src == "None":
+        return None
+#    dt = strptime(src, '%d %m %Y %H:%M:%S', assume_utc=True, as_utc=True)
+#    if dt == UNDEFINED_DATE:
+#        return None
+    return parse_date(src)
+
+def datetime_to_string(dateval):
+    if dateval is None or dateval == UNDEFINED_DATE:
+        return "None"
+#    tt = date_to_utc(dateval).timetuple()
+#    res = "%02d %02d %04d %02d:%02d:%02d"%(tt.tm_mday, tt.tm_mon, tt.tm_year,
+#                                     tt.tm_hour, tt.tm_min, tt.tm_sec)
+    return isoformat(dateval)
+
+def encode_thumbnail(thumbnail):
+    '''
+    Encode the image part of a thumbnail, then return the 3 part tuple
+    '''
+    if thumbnail is None:
+        return None
+    return (thumbnail[0], thumbnail[1], b64encode(str(thumbnail[2])))
+
+def decode_thumbnail(tup):
+    '''
+    Decode an encoded thumbnail into its 3 component parts
+    '''
+    if tup is None:
+        return None
+    return (tup[0], tup[1], b64decode(tup[2]))
+
+class JsonCodec(object):
+
+    def __init__(self):
+        self.field_metadata = FieldMetadata()
+
+    def encode_to_file(self, file, booklist):
+        json.dump(self.encode_booklist_metadata(booklist), file, indent=2, encoding='utf-8')
+
+    def encode_booklist_metadata(self, booklist):
+        result = []
+        for book in booklist:
+            result.append(self.encode_book_metadata(book))
+        return result
+
+    def encode_book_metadata(self, book):
+        result = {}
+        for key in SERIALIZABLE_FIELDS:
+            result[key] = self.encode_metadata_attr(book, key)
+        return result
+
+    def encode_metadata_attr(self, book, key):
+        if key == 'user_metadata':
+            meta = book.all_user_metadata
+            for k in meta:
+                if meta[k]['datatype'] == 'datetime':
+                    meta[k]['#value#'] = datetime_to_string(meta[k]['#value#'])
+            return meta
+        if key in self.field_metadata:
+            datatype = self.field_metadata[key]['datatype']
+        else:
+            datatype = None
+        value = book.get(key)
+        if key == 'thumbnail':
+            return encode_thumbnail(value)
+        elif isinstance(value, str): # str includes bytes
+            enc = filesystem_encoding if key == 'lpath' else preferred_encoding
+            return value.decode(enc, 'replace')
+        elif isinstance(value, (list, tuple)):
+            return [x.decode(preferred_encoding, 'replace') if
+                    isinstance(x, str) else x for x in value]
+        elif datatype == 'datetime':
+            return datetime_to_string(value)
+        else:
+            return value
+
+    def decode_from_file(self, file, booklist, book_class, prefix):
+        js = []
+        try:
+            js = json.load(file, encoding='utf-8')
+            for item in js:
+                book = book_class(prefix, item.get('lpath', None))
+                for key in item.keys():
+                    meta = self.decode_metadata(key, item[key])
+                    if key == 'user_metadata':
+                        book.set_all_user_metadata(meta)
+                    else:
+                        setattr(book, key, meta)
+                booklist.append(book)
+        except:
+            print 'exception during JSON decoding'
+            traceback.print_exc()
+            booklist = []
+
+    def decode_metadata(self, key, value):
+        if key == 'user_metadata':
+            for k in value:
+                if value[k]['datatype'] == 'datetime':
+                    value[k]['#value#'] = string_to_datetime(value[k]['#value#'])
+            return value
+        elif key in self.field_metadata:
+            if self.field_metadata[key]['datatype'] == 'datetime':
+                return string_to_datetime(value)
+        if key == 'thumbnail':
+            return decode_thumbnail(value)
+        return value

src/calibre/ebooks/metadata/isbndb.py

@@ -8,7 +8,7 @@ import sys, re
 from urllib import quote
 
 from calibre.utils.config import OptionParser
-from calibre.ebooks.metadata import MetaInformation
+from calibre.ebooks.metadata.book.base import Metadata
 from calibre.ebooks.BeautifulSoup import BeautifulStoneSoup
 from calibre import browser
 
@@ -42,10 +42,10 @@ def fetch_metadata(url, max=100, timeout=5.):
     return books
 
 
-class ISBNDBMetadata(MetaInformation):
+class ISBNDBMetadata(Metadata):
 
     def __init__(self, book):
-        MetaInformation.__init__(self, None, [])
+        Metadata.__init__(self, None, [])
 
         self.isbn = book.get('isbn13', book.get('isbn'))
         self.title = book.find('titlelong').string

src/calibre/ebooks/metadata/opf2.py

@@ -16,7 +16,8 @@ from lxml import etree
 from calibre.ebooks.chardet import xml_to_unicode
 from calibre.constants import __appname__, __version__, filesystem_encoding
 from calibre.ebooks.metadata.toc import TOC
-from calibre.ebooks.metadata import MetaInformation, string_to_authors
+from calibre.ebooks.metadata import string_to_authors
+from calibre.ebooks.metadata.book.base import Metadata
 from calibre.utils.date import parse_date, isoformat
 from calibre.utils.localization import get_lang
 
@@ -926,16 +927,16 @@ class OPF(object):
                 setattr(self, attr, val)
 
 
-class OPFCreator(MetaInformation):
+class OPFCreator(Metadata):
 
-    def __init__(self, base_path, *args, **kwargs):
+    def __init__(self, base_path, other):
         '''
         Initialize.
         @param base_path: An absolute path to the directory in which this OPF file
         will eventually be. This is used by the L{create_manifest} method
         to convert paths to files into relative paths.
         '''
-        MetaInformation.__init__(self, *args, **kwargs)
+        Metadata.__init__(self, title='', other=other)
         self.base_path = os.path.abspath(base_path)
         if self.application_id is None:
             self.application_id = str(uuid.uuid4())

src/calibre/gui2/dialogs/config/save_template.py

@@ -34,25 +34,24 @@ class SaveTemplate(QWidget, Ui_Form):
         self.option_name = name
 
     def validate(self):
-        tmpl = preprocess_template(self.opt_template.text())
+        # TODO: I haven't figured out how to get the custom columns into here,
-        fa = {}
+        # so for the moment make all templates valid.
-        for x in FORMAT_ARG_DESCS.keys():
-            fa[x]='random long string'
-        try:
-            tmpl.format(**fa)
-        except Exception, err:
-            error_dialog(self, _('Invalid template'),
-                    '<p>'+_('The template %s is invalid:')%tmpl + \
-                    '<br>'+str(err), show=True)
-            return False
         return True
+#        tmpl = preprocess_template(self.opt_template.text())
+#        fa = {}
+#        for x in FORMAT_ARG_DESCS.keys():
+#            fa[x]='random long string'
+#        try:
+#            tmpl.format(**fa)
+#        except Exception, err:
+#            error_dialog(self, _('Invalid template'),
+#                    '<p>'+_('The template %s is invalid:')%tmpl + \
+#                    '<br>'+str(err), show=True)
+#            return False
+#        return True
 
     def save_settings(self, config, name):
         val = unicode(self.opt_template.text())
         config.set(name, val)
         self.opt_template.save_history(self.option_name+'_template_history')
 
-
-
-
-

src/calibre/gui2/library/models.py

@@ -372,7 +372,6 @@ class BooksModel(QAbstractTableModel): # {{{
         return ans
 
     def get_metadata(self, rows, rows_are_ids=False, full_metadata=False):
-        # Should this add the custom columns? It doesn't at the moment
         metadata, _full_metadata = [], []
         if not rows_are_ids:
             rows = [self.db.id(row.row()) for row in rows]
@@ -1053,7 +1052,7 @@ class DeviceBooksModel(BooksModel): # {{{
             if hasattr(cdata, 'image_path'):
                 img.load(cdata.image_path)
             else:
-                img.loadFromData(cdata)
+                img.loadFromData(cdata[2])
             if img.isNull():
                 img = self.default_image
             data['cover'] = img

src/calibre/library/database2.py

@@ -509,15 +509,15 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         '''
         Convenience method to return metadata as a L{MetaInformation} object.
         '''
-        aum = self.authors(idx, index_is_id=index_is_id)
+        aut_list = self.authors_with_sort_strings(idx, index_is_id=index_is_id)
-        if aum: aum = [a.strip().replace('|', ',') for a in aum.split(',')]
+        aum = []
+        aus = {}
+        for (author, author_sort) in aut_list:
+            aum.append(author)
+            aus[author] = author_sort
         mi = MetaInformation(self.title(idx, index_is_id=index_is_id), aum)
         mi.author_sort = self.author_sort(idx, index_is_id=index_is_id)
-        if mi.authors:
+        mi.author_sort_map = aus
-            mi.author_sort_map = {}
-            for name, sort in zip(mi.authors, self.authors_sort_strings(idx,
-                index_is_id)):
-                mi.author_sort_map[name] = sort
         mi.comments    = self.comments(idx, index_is_id=index_is_id)
         mi.publisher   = self.publisher(idx, index_is_id=index_is_id)
         mi.timestamp   = self.timestamp(idx, index_is_id=index_is_id)
@@ -534,6 +534,10 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         mi.isbn = self.isbn(idx, index_is_id=index_is_id)
         id = idx if index_is_id else self.id(idx)
         mi.application_id = id
+        for key,meta in self.field_metadata.iteritems():
+            if meta['is_custom']:
+                mi.set_user_metadata(key, meta)
+                mi.set(key, self.get_custom(idx, label=meta['label'], index_is_id=index_is_id))
         if get_cover:
             mi.cover = self.cover(id, index_is_id=True, as_path=True)
         return mi
@@ -1049,6 +1053,19 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             result.append(sort)
         return result
 
+    # Given a book, return the map of author sort strings for the book's authors
+    def authors_with_sort_strings(self, id, index_is_id=False):
+        id = id if index_is_id else self.id(id)
+        aut_strings = self.conn.get('''
+                        SELECT authors.name, authors.sort
+                        FROM authors, books_authors_link as bl
+                        WHERE bl.book=? and authors.id=bl.author
+                        ORDER BY bl.id''', (id,))
+        result = []
+        for (author, sort,) in aut_strings:
+            result.append((author.replace('|', ','), sort))
+        return result
+
     # Given a book, return the author_sort string for authors of the book
     def author_sort_from_book(self, id, index_is_id=False):
         auts = self.authors_sort_strings(id, index_is_id)

src/calibre/library/save_to_disk.py

@@ -105,6 +105,8 @@ def safe_format(x, format_args):
         pass
     except AttributeError: # Thrown if user used a non existing attribute
         pass
+    except KeyError: # Thrown if user used custom field w/value None
+        pass
     return ''
 
 def get_components(template, mi, id, timefmt='%b %Y', length=250,
@@ -113,13 +115,12 @@ def get_components(template, mi, id, timefmt='%b %Y', length=250,
     library_order = tweaks['save_template_title_series_sorting'] == 'library_order'
     tsfmt = title_sort if library_order else lambda x: x
     format_args = dict(**FORMAT_ARGS)
+    format_args.update(mi.all_attributes)
     if mi.title:
         format_args['title'] = tsfmt(mi.title)
     if mi.authors:
         format_args['authors'] = mi.format_authors()
         format_args['author'] = format_args['authors']
-    if mi.author_sort:
-        format_args['author_sort'] = mi.author_sort
     if mi.tags:
         format_args['tags'] = mi.format_tags()
         if format_args['tags'].startswith('/'):
@@ -132,15 +133,21 @@ def get_components(template, mi, id, timefmt='%b %Y', length=250,
         template = re.sub(r'\{series_index[^}]*?\}', '', template)
     if mi.rating is not None:
         format_args['rating'] = mi.format_rating()
-    if mi.isbn:
-        format_args['isbn'] = mi.isbn
-    if mi.publisher:
-        format_args['publisher'] = mi.publisher
     if hasattr(mi.timestamp, 'timetuple'):
         format_args['timestamp'] = strftime(timefmt, mi.timestamp.timetuple())
     if hasattr(mi.pubdate, 'timetuple'):
         format_args['pubdate'] = strftime(timefmt, mi.pubdate.timetuple())
     format_args['id'] = str(id)
+
+    # These are not necessary any more. The values are set by
+    # 'format_args.update' above, and there is no special formatting
+#    if mi.author_sort:
+#        format_args['author_sort'] = mi.author_sort
+#    if mi.isbn:
+#        format_args['isbn'] = mi.isbn
+#    if mi.publisher:
+#        format_args['publisher'] = mi.publisher
+
     components = [x.strip() for x in template.split('/') if x.strip()]
     components = [safe_format(x, format_args) for x in components]
     components = [sanitize_func(x) for x in components if x]