Refactor metadata object to support custom columns

resources/content_server/gui.js

@@ -59,14 +59,44 @@ function render_book(book) {
         title = title.slice(0, title.length-2);
         title += ' ({0} MB) '.format(size);
     }
+    title += '<span class="tagdata_short" style="display:all">'
+    if (tags) {
+        t = tags.split(':&:', 2);
+        m = parseInt(t[0]);
+        t = t[1].split(',', m);
+        if (t.length == m) t[m] = '...'
+        title += 'Tags=[{0}] '.format(t.join(','));
+    }
+    custcols = book.attr("custcols").split(',')
+    for ( i = 0; i < custcols.length; i++) {
+        if (custcols[i].length > 0) {
+            vals = book.attr(custcols[i]).split(':#:', 2);
+            if (vals[0].indexOf('#T#') == 0) { //startswith
+                vals[0] = vals[0].substr(3, vals[0].length)
+                t = vals[1].split(':&:', 2);
+                m = parseInt(t[0]);
+                t = t[1].split(',', m);
+                if (t.length == m) t[m] = '...';
+                vals[1] = t.join(',');
+            }
+            title += '{0}=[{1}] '.format(vals[0], vals[1]);
+        }
+    }
+    title += '</span>'
+    title += '<span class="tagdata_long" style="display:none">'
     if (tags) title += 'Tags=[{0}] '.format(tags);
     custcols = book.attr("custcols").split(',')
     for ( i = 0; i < custcols.length; i++) {
         if (custcols[i].length > 0) {
             vals = book.attr(custcols[i]).split(':#:', 2);
+            if (vals[0].indexOf('#T#') == 0) { //startswith
+                vals[0] = vals[0].substr(3, vals[0].length)
+                vals[1] = (vals[1].split(':&:', 2))[1];
+            }
             title += '{0}=[{1}] '.format(vals[0], vals[1]);
         }
     }
+    title += '</span>'
     title += '<img style="display:none" alt="" src="get/cover/{0}" /></span>'.format(id);
     title += '<div class="comments">{0}</div>'.format(comments)
     // Render authors cell
@@ -170,11 +200,15 @@ function fetch_library_books(start, num, timeout, sort, order, search) {
               var cover = row.find('img').attr('src');
               var collapsed = row.find('.comments').css('display') == 'none';
               $("#book_list tbody tr * .comments").css('display', 'none');
+              $("#book_list tbody tr * .tagdata_short").css('display', 'inherit');
+              $("#book_list tbody tr * .tagdata_long").css('display', 'none');
               $('#cover_pane').css('visibility', 'hidden');
               if (collapsed) {
                   row.find('.comments').css('display', 'inherit');
                   $('#cover_pane img').attr('src', cover);
                   $('#cover_pane').css('visibility', 'visible');
+                row.find(".tagdata_short").css('display', 'none');
+                row.find(".tagdata_long").css('display', 'inherit');
               }
           });
 

resources/default_tweaks.py

@@ -93,6 +93,37 @@ save_template_title_series_sorting = 'library_order'
 auto_connect_to_folder = ''
 
 
+# Specify renaming rules for sony collections. Collections on Sonys are named
+# depending upon whether the field is standard or custom. A collection derived
+# from a standard field is named for the value in that field. For example, if
+# the standard 'series' column contains the name 'Darkover', then the series
+# will be named 'Darkover'. A collection derived from a custom field will have
+# the name of the field added to the value. For example, if a custom series
+# column named 'My Series' contains the name 'Darkover', then the collection
+# will be named 'Darkover (My Series)'. If two books have fields that generate
+# the same collection name, then both books will be in that collection. This
+# tweak lets you specify for a standard or custom field the value to be put
+# inside the parentheses. You can use it to add a parenthetical description to a
+# standard field, for example 'Foo (Tag)' instead of the 'Foo'. You can also use
+# it to force multiple fields to end up in the same collection. For example, you
+# could force the values in 'series', '#my_series_1', and '#my_series_2' to
+# appear in collections named 'some_value (Series)', thereby merging all of the
+# fields into one set of collections. The syntax of this tweak is
+# {'field_lookup_name':'name_to_use', 'lookup_name':'name', ...}
+# Example 1: I want three series columns to be merged into one set of
+# collections. If the column lookup names are 'series', '#series_1' and
+# '#series_2', and if I want nothing in the parenthesis, then the value to use
+# in the tweak value would be:
+# sony_collection_renaming_rules={'series':'', '#series_1':'', '#series_2':''}
+# Example 2: I want the word '(Series)' to appear on collections made from
+# series, and the word '(Tag)' to appear on collections made from tags. Use:
+# sony_collection_renaming_rules={'series':'Series', 'tags':'Tag'}
+# Example 3: I want 'series' and '#myseries' to be merged, and for the
+# collection name to have '(Series)' appended. The renaming rule is:
+# sony_collection_renaming_rules={'series':'Series', '#myseries':'Series'}
+sony_collection_renaming_rules={}
+
+
 # Create search terms to apply a query across several built-in search terms.
 # Syntax: {'new term':['existing term 1', 'term 2', ...], 'new':['old'...] ...}
 # Example: create the term 'myseries' that when used as myseries:foo would
@@ -114,3 +145,4 @@ add_new_book_tags_when_importing_books = False
 # Set the maximum number of tags to show per book in the content server
 max_content_server_tags_shown=5
 
+

src/calibre/customize/__init__.py

@@ -218,7 +218,7 @@ class MetadataReaderPlugin(Plugin): # {{{
         with the input data.
         :param type: The type of file. Guaranteed to be one of the entries
         in :attr:`file_types`.
-        :return: A :class:`calibre.ebooks.metadata.MetaInformation` object
+        :return: A :class:`calibre.ebooks.metadata.book.Metadata` object
         '''
         return None
 # }}}
@@ -248,7 +248,7 @@ class MetadataWriterPlugin(Plugin): # {{{
         with the input data.
         :param type: The type of file. Guaranteed to be one of the entries
         in :attr:`file_types`.
-        :param mi: A :class:`calibre.ebooks.metadata.MetaInformation` object
+        :param mi: A :class:`calibre.ebooks.metadata.book.Metadata` object
         '''
         pass
 

src/calibre/devices/apple/driver.py

@@ -13,7 +13,8 @@ from calibre.devices.errors import UserFeedback
 from calibre.devices.usbms.deviceconfig import DeviceConfig
 from calibre.devices.interface import DevicePlugin
 from calibre.ebooks.BeautifulSoup import BeautifulSoup
-from calibre.ebooks.metadata import MetaInformation, authors_to_string
+from calibre.ebooks.metadata import authors_to_string, MetaInformation
+from calibre.ebooks.metadata.book.base import Metadata
 from calibre.ebooks.metadata.epub import set_metadata
 from calibre.library.server.utils import strftime
 from calibre.utils.config import config_dir
@@ -871,7 +872,7 @@ class ITUNES(DriverBase):
         once uploaded to the device. len(names) == len(files)
         :return: A list of 3-element tuples. The list is meant to be passed
         to L{add_books_to_metadata}.
-        :metadata: If not None, it is a list of :class:`MetaInformation` objects.
+        :metadata: If not None, it is a list of :class:`Metadata` objects.
         The idea is to use the metadata to determine where on the device to
         put the book. len(metadata) == len(files). Apart from the regular
         cover (path to cover), there may also be a thumbnail attribute, which should
@@ -2999,14 +3000,14 @@ class BookList(list):
         '''
         return {}
 
-class Book(MetaInformation):
+class Book(Metadata):
     '''
     A simple class describing a book in the iTunes Books Library.
     - See ebooks.metadata.__init__ for all fields
     '''
     def __init__(self,title,author):
 
-        MetaInformation.__init__(self, title, authors=[author])
+        Metadata.__init__(self, title, authors=[author])
 
     @dynamic_property
     def title_sorter(self):

src/calibre/devices/interface.py

@@ -316,7 +316,7 @@ class DevicePlugin(Plugin):
                       being uploaded to the device.
         :param names: A list of file names that the books should have
                       once uploaded to the device. len(names) == len(files)
-        :param metadata: If not None, it is a list of :class:`MetaInformation` objects.
+        :param metadata: If not None, it is a list of :class:`Metadata` objects.
                          The idea is to use the metadata to determine where on the device to
                          put the book. len(metadata) == len(files). Apart from the regular
                          cover (path to cover), there may also be a thumbnail attribute, which should
@@ -335,7 +335,7 @@ class DevicePlugin(Plugin):
         the device.
 
         :param locations: Result of a call to L{upload_books}
-        :param metadata: List of :class:`MetaInformation` objects, same as for
+        :param metadata: List of :class:`Metadata` objects, same as for
                          :meth:`upload_books`.
         :param booklists: A tuple containing the result of calls to
                           (:meth:`books(oncard=None)`,

src/calibre/devices/kobo/books.py

@@ -4,37 +4,15 @@ __copyright__ = '2010, Timothy Legge <timlegge at gmail.com>'
 '''
 
 import os
-import re
 import time
 
-from calibre.ebooks.metadata import MetaInformation
+from calibre.devices.usbms.books import Book as Book_
-from calibre.constants import filesystem_encoding, preferred_encoding
-from calibre import isbytestring
 
-class Book(MetaInformation):
+class Book(Book_):
 
-    BOOK_ATTRS = ['lpath', 'size', 'mime', 'device_collections', '_new_book']
+    def __init__(self, prefix, lpath, title, authors, mime, date, ContentType,
-
+                 thumbnail_name, size=None, other=None):
-    JSON_ATTRS = [
+        Book_.__init__(self, prefix, lpath)
-        'lpath', 'title', 'authors', 'mime', 'size', 'tags', 'author_sort',
-        'title_sort', 'comments', 'category', 'publisher', 'series',
-        'series_index', 'rating', 'isbn', 'language', 'application_id',
-        'book_producer', 'lccn', 'lcc', 'ddc', 'rights', 'publication_type',
-        'uuid', 'device_collections',
-    ]
-
-    def __init__(self, prefix, lpath, title, authors, mime, date, ContentType, thumbnail_name, size=None, other=None):
-
-        MetaInformation.__init__(self, '')
-        self.device_collections = []
-        self._new_book = False
-
-        self.path = os.path.join(prefix, lpath)
-        if os.sep == '\\':
-            self.path = self.path.replace('/', '\\')
-            self.lpath = lpath.replace('\\', '/')
-        else:
-             self.lpath = lpath
 
         self.title = title
         if not authors:
@@ -50,65 +28,14 @@ class Book(MetaInformation):
             else:
                 self.datetime = time.gmtime(os.path.getctime(self.path))
         except:
-             self.datetime = time.gmtime()
+            self.datetime = time.gmtime()
-
+        if thumbnail_name is not None:
-	if thumbnail_name is not None:
+            self.thumbnail = ImageWrapper(thumbnail_name)
-	    self.thumbnail = ImageWrapper(thumbnail_name)
         self.tags = []
         if other:
             self.smart_update(other)
 
-    def __eq__(self, other):
-        return self.path == getattr(other, 'path', None)
-
-    @dynamic_property
-    def db_id(self):
-        doc = '''The database id in the application database that this file corresponds to'''
-        def fget(self):
-            match = re.search(r'_(\d+)$', self.lpath.rpartition('.')[0])
-            if match:
-                return int(match.group(1))
-            return None
-        return property(fget=fget, doc=doc)
-
-    @dynamic_property
-    def title_sorter(self):
-        doc = '''String to sort the title. If absent, title is returned'''
-        def fget(self):
-            return re.sub('^\s*A\s+|^\s*The\s+|^\s*An\s+', '', self.title).rstrip()
-        return property(doc=doc, fget=fget)
-
-    @dynamic_property
-    def thumbnail(self):
-        return None
-
-    def smart_update(self, other, replace_metadata=False):
-        '''
-        Merge the information in C{other} into self. In case of conflicts, the information
-        in C{other} takes precedence, unless the information in C{other} is NULL.
-        '''
-
-        MetaInformation.smart_update(self, other)
-
-        for attr in self.BOOK_ATTRS:
-            if hasattr(other, attr):
-                val = getattr(other, attr, None)
-                setattr(self, attr, val)
-
-    def to_json(self):
-        json = {}
-        for attr in self.JSON_ATTRS:
-            val = getattr(self, attr)
-            if isbytestring(val):
-                 enc = filesystem_encoding if attr == 'lpath' else preferred_encoding
-                 val = val.decode(enc, 'replace')
-            elif isinstance(val, (list, tuple)):
-                val = [x.decode(preferred_encoding, 'replace') if
-                        isbytestring(x) else x for x in val]
-            json[attr] = val
-        return json
-
 class ImageWrapper(object):
     def __init__(self, image_path):
-       self.image_path = image_path
+        self.image_path = image_path
 

src/calibre/devices/kobo/driver.py

@@ -132,7 +132,7 @@ class KOBO(USBMS):
 
         changed = False
         for i, row in enumerate(cursor):
-         #  self.report_progress((i+1) / float(numrows), _('Getting list of books on device...'))
+        #  self.report_progress((i+1) / float(numrows), _('Getting list of books on device...'))
 
             path = self.path_from_contentid(row[3], row[5], oncard)
             mime = mime_type_ext(path_to_ext(row[3]))

src/calibre/devices/usbms/books.py

@@ -6,29 +6,19 @@ __docformat__ = 'restructuredtext en'
 
 import os, re, time, sys
 
-from calibre.ebooks.metadata import MetaInformation
+from calibre.ebooks.metadata.book.base import Metadata
 from calibre.devices.mime import mime_type_ext
 from calibre.devices.interface import BookList as _BookList
-from calibre.constants import filesystem_encoding, preferred_encoding
+from calibre.constants import preferred_encoding
 from calibre import isbytestring
-from calibre.utils.config import prefs
+from calibre.utils.config import prefs, tweaks
-
+from calibre.utils.date import format_date
-class Book(MetaInformation):
-
-    BOOK_ATTRS = ['lpath', 'size', 'mime', 'device_collections', '_new_book']
-
-    JSON_ATTRS = [
-        'lpath', 'title', 'authors', 'mime', 'size', 'tags', 'author_sort',
-        'title_sort', 'comments', 'category', 'publisher', 'series',
-        'series_index', 'rating', 'isbn', 'language', 'application_id',
-        'book_producer', 'lccn', 'lcc', 'ddc', 'rights', 'publication_type',
-        'uuid',
-    ]
 
+class Book(Metadata):
     def __init__(self, prefix, lpath, size=None, other=None):
         from calibre.ebooks.metadata.meta import path_to_ext
 
-        MetaInformation.__init__(self, '')
+        Metadata.__init__(self, '')
 
         self._new_book = False
         self.device_collections = []
@@ -72,32 +62,6 @@ class Book(MetaInformation):
     def thumbnail(self):
         return None
 
-    def smart_update(self, other, replace_metadata=False):
-        '''
-        Merge the information in C{other} into self. In case of conflicts, the information
-        in C{other} takes precedence, unless the information in C{other} is NULL.
-        '''
-
-        MetaInformation.smart_update(self, other, replace_metadata)
-
-        for attr in self.BOOK_ATTRS:
-            if hasattr(other, attr):
-                val = getattr(other, attr, None)
-                setattr(self, attr, val)
-
-    def to_json(self):
-        json = {}
-        for attr in self.JSON_ATTRS:
-            val = getattr(self, attr)
-            if isbytestring(val):
-                enc = filesystem_encoding if attr == 'lpath' else preferred_encoding
-                val = val.decode(enc, 'replace')
-            elif isinstance(val, (list, tuple)):
-                val = [x.decode(preferred_encoding, 'replace') if
-                        isbytestring(x) else x for x in val]
-            json[attr] = val
-        return json
-
 class BookList(_BookList):
 
     def __init__(self, oncard, prefix, settings):
@@ -131,11 +95,38 @@ class CollectionsBookList(BookList):
     def supports_collections(self):
         return True
 
+    def compute_category_name(self, attr, category, cust_field_meta):
+        renames = tweaks['sony_collection_renaming_rules']
+        attr_name = renames.get(attr, None)
+        if attr_name is None:
+            if attr in cust_field_meta:
+                attr_name = '(%s)'%cust_field_meta[attr]['name']
+            else:
+                attr_name = ''
+        elif attr_name != '':
+            attr_name = '(%s)'%attr_name
+
+        if attr not in cust_field_meta:
+            cat_name = '%s %s'%(category, attr_name)
+        else:
+            fm = cust_field_meta[attr]
+            if fm['datatype'] == 'bool':
+                if category:
+                    cat_name = '%s %s'%(_('Yes'), attr_name)
+                else:
+                    cat_name = '%s %s'%(_('No'), attr_name)
+            elif fm['datatype'] == 'datetime':
+                cat_name = '%s %s'%(format_date(category,
+                    fm['display'].get('date_format','dd MMM yyyy')), attr_name)
+            else:
+                cat_name = '%s %s'%(category, attr_name)
+        return cat_name.strip()
+
     def get_collections(self, collection_attributes):
         from calibre.devices.usbms.driver import debug_print
         debug_print('Starting get_collections:', prefs['manage_device_metadata'])
+        debug_print('Renaming rules:', tweaks['sony_collection_renaming_rules'])
         collections = {}
-        series_categories = set([])
         # This map of sets is used to avoid linear searches when testing for
         # book equality
         collections_lpaths = {}
@@ -161,41 +152,55 @@ class CollectionsBookList(BookList):
                 # For existing books, modify the collections only if the user
                 # specified 'on_connect'
                 attrs = collection_attributes
+            meta_vals = book.get_all_non_none_attributes()
+            cust_field_meta = book.get_all_user_metadata(make_copy=False)
             for attr in attrs:
                 attr = attr.strip()
-                val = getattr(book, attr, None)
+                val = meta_vals.get(attr, None)
                 if not val: continue
                 if isbytestring(val):
                     val = val.decode(preferred_encoding, 'replace')
                 if isinstance(val, (list, tuple)):
                     val = list(val)
-                elif isinstance(val, unicode):
+                else:
                     val = [val]
                 for category in val:
-                    if attr == 'tags' and len(category) > 1 and \
+                    is_series = False
-                            category[0] == '[' and category[-1] == ']':
+                    if attr in cust_field_meta: # is a custom field
+                        fm = cust_field_meta[attr]
+                        if fm['datatype'] == 'text' and len(category) > 1 and \
+                                category[0] == '[' and category[-1] == ']':
+                            continue
+                        if fm['datatype'] == 'series':
+                            is_series = True
+                    else:                       # is a standard field
+                        if attr == 'tags' and len(category) > 1 and \
+                                category[0] == '[' and category[-1] == ']':
+                            continue
+                        if attr == 'series' or \
+                                ('series' in collection_attributes and
+                                 meta_vals.get('series', None) == category):
+                            is_series = True
+                    cat_name = self.compute_category_name(attr, category,
+                                                          cust_field_meta)
+                    if cat_name not in collections:
+                        collections[cat_name] = []
+                        collections_lpaths[cat_name] = set()
+                    if lpath in collections_lpaths[cat_name]:
                         continue
-                    if category not in collections:
+                    collections_lpaths[cat_name].add(lpath)
-                        collections[category] = []
+                    if is_series:
-                        collections_lpaths[category] = set()
+                        collections[cat_name].append(
-                    if lpath not in collections_lpaths[category]:
+                            (book, meta_vals.get(attr+'_index', sys.maxint)))
-                        collections_lpaths[category].add(lpath)
+                    else:
-                        collections[category].append(book)
+                        collections[cat_name].append(
-                    if attr == 'series' or \
+                            (book, meta_vals.get('title_sort', 'zzzz')))
-                            ('series' in collection_attributes and
-                             getattr(book, 'series', None) == category):
-                        series_categories.add(category)
         # Sort collections
+        result = {}
         for category, books in collections.items():
-            def tgetter(x):
+            books.sort(cmp=lambda x,y:cmp(x[1], y[1]))
-                return getattr(x, 'title_sort', 'zzzz')
+            result[category] = [x[0] for x in books]
-            books.sort(cmp=lambda x,y:cmp(tgetter(x), tgetter(y)))
+        return result
-            if category in series_categories:
-                # Ensures books are sub sorted by title
-                def getter(x):
-                    return getattr(x, 'series_index', sys.maxint)
-                books.sort(cmp=lambda x,y:cmp(getter(x), getter(y)))
-        return collections
 
     def rebuild_collections(self, booklist, oncard):
         '''

src/calibre/devices/usbms/driver.py

@@ -13,7 +13,6 @@ for a particular device.
 import os
 import re
 import time
-import json
 from itertools import cycle
 
 from calibre import prints, isbytestring
@@ -21,6 +20,7 @@ from calibre.constants import filesystem_encoding, DEBUG
 from calibre.devices.usbms.cli import CLI
 from calibre.devices.usbms.device import Device
 from calibre.devices.usbms.books import BookList, Book
+from calibre.ebooks.metadata.book.json_codec import JsonCodec
 
 BASE_TIME = None
 def debug_print(*args):
@@ -288,6 +288,7 @@ class USBMS(CLI, Device):
     # at the end just before the return
     def sync_booklists(self, booklists, end_session=True):
         debug_print('USBMS: starting sync_booklists')
+        json_codec = JsonCodec()
 
         if not os.path.exists(self.normalize_path(self._main_prefix)):
             os.makedirs(self.normalize_path(self._main_prefix))
@@ -296,10 +297,8 @@ class USBMS(CLI, Device):
             if prefix is not None and isinstance(booklists[listid], self.booklist_class):
                 if not os.path.exists(prefix):
                     os.makedirs(self.normalize_path(prefix))
-                js = [item.to_json() for item in booklists[listid] if
-                        hasattr(item, 'to_json')]
                 with open(self.normalize_path(os.path.join(prefix, self.METADATA_CACHE)), 'wb') as f:
-                    f.write(json.dumps(js, indent=2, encoding='utf-8'))
+                    json_codec.encode_to_file(f, booklists[listid])
         write_prefix(self._main_prefix, 0)
         write_prefix(self._card_a_prefix, 1)
         write_prefix(self._card_b_prefix, 2)
@@ -345,19 +344,13 @@ class USBMS(CLI, Device):
 
     @classmethod
     def parse_metadata_cache(cls, bl, prefix, name):
-        # bl = cls.booklist_class()
+        json_codec = JsonCodec()
-        js = []
         need_sync = False
         cache_file = cls.normalize_path(os.path.join(prefix, name))
         if os.access(cache_file, os.R_OK):
             try:
                 with open(cache_file, 'rb') as f:
-                    js = json.load(f, encoding='utf-8')
+                    json_codec.decode_from_file(f, bl, cls.book_class, prefix)
-                for item in js:
-                    book = cls.book_class(prefix, item.get('lpath', None))
-                    for key in item.keys():
-                        setattr(book, key, item[key])
-                    bl.append(book)
             except:
                 import traceback
                 traceback.print_exc()
@@ -392,7 +385,7 @@ class USBMS(CLI, Device):
 
     @classmethod
     def book_from_path(cls, prefix, lpath):
-        from calibre.ebooks.metadata import MetaInformation
+        from calibre.ebooks.metadata.book.base import Metadata
 
         if cls.settings().read_metadata or cls.MUST_READ_METADATA:
             mi = cls.metadata_from_path(cls.normalize_path(os.path.join(prefix, lpath)))
@@ -401,7 +394,7 @@ class USBMS(CLI, Device):
             mi = metadata_from_filename(cls.normalize_path(os.path.basename(lpath)),
                                         cls.build_template_regexp())
         if mi is None:
-            mi = MetaInformation(os.path.splitext(os.path.basename(lpath))[0],
+            mi = Metadata(os.path.splitext(os.path.basename(lpath))[0],
                     [_('Unknown')])
         size = os.stat(cls.normalize_path(os.path.join(prefix, lpath))).st_size
         book = cls.book_class(prefix, lpath, other=mi, size=size)

src/calibre/ebooks/metadata/__init__.py

@@ -10,10 +10,9 @@ import os, mimetypes, sys, re
 from urllib import unquote, quote
 from urlparse import urlparse
 
-from calibre import relpath, prints
+from calibre import relpath
 
 from calibre.utils.config import tweaks
-from calibre.utils.date import isoformat
 
 _author_pat = re.compile(',?\s+(and|with)\s+', re.IGNORECASE)
 def string_to_authors(raw):
@@ -221,214 +220,18 @@ class ResourceCollection(object):
 
 
 
-class MetaInformation(object):
+def MetaInformation(title, authors=(_('Unknown'),)):
-    '''Convenient encapsulation of book metadata'''
+    ''' Convenient encapsulation of book metadata, needed for compatibility
-
-    @staticmethod
-    def copy(mi):
-        ans = MetaInformation(mi.title, mi.authors)
-        for attr in ('author_sort', 'title_sort', 'comments', 'category',
-                     'publisher', 'series', 'series_index', 'rating',
-                     'isbn', 'tags', 'cover_data', 'application_id', 'guide',
-                     'manifest', 'spine', 'toc', 'cover', 'language',
-                     'book_producer', 'timestamp', 'lccn', 'lcc', 'ddc',
-                     'author_sort_map',
-                     'pubdate', 'rights', 'publication_type', 'uuid'):
-            if hasattr(mi, attr):
-                setattr(ans, attr, getattr(mi, attr))
-
-    def __init__(self, title, authors=(_('Unknown'),)):
-        '''
         @param title: title or ``_('Unknown')`` or a MetaInformation object
         @param authors: List of strings or []
-        '''
+    '''
-        mi = None
+    from calibre.ebooks.metadata.book.base import Metadata
-        if hasattr(title, 'title') and hasattr(title, 'authors'):
+    mi = None
-            mi = title
+    if hasattr(title, 'title') and hasattr(title, 'authors'):
-            title = mi.title
+        mi = title
-            authors = mi.authors
+        title = mi.title
-        self.title = title
+        authors = mi.authors
-        self.author = list(authors) if authors else []# Needed for backward compatibility
+    return Metadata(title, authors, mi)
-        #: List of strings or []
-        self.authors = list(authors) if authors else []
-        self.tags = getattr(mi, 'tags', [])
-        #: mi.cover_data = (ext, data)
-        self.cover_data   = getattr(mi, 'cover_data', (None, None))
-        self.author_sort_map = getattr(mi, 'author_sort_map', {})
-
-        for x in ('author_sort', 'title_sort', 'comments', 'category', 'publisher',
-                  'series', 'series_index', 'rating', 'isbn', 'language',
-                  'application_id', 'manifest', 'toc', 'spine', 'guide', 'cover',
-                  'book_producer', 'timestamp', 'lccn', 'lcc', 'ddc', 'pubdate',
-                  'rights', 'publication_type', 'uuid',
-                  ):
-            setattr(self, x, getattr(mi, x, None))
-
-    def print_all_attributes(self):
-        for x in ('title','author', 'author_sort', 'title_sort', 'comments', 'category', 'publisher',
-                  'series', 'series_index', 'tags', 'rating', 'isbn', 'language',
-                  'application_id', 'manifest', 'toc', 'spine', 'guide', 'cover',
-                  'book_producer', 'timestamp', 'lccn', 'lcc', 'ddc', 'pubdate',
-                  'rights', 'publication_type', 'uuid', 'author_sort_map'
-                  ):
-            prints(x, getattr(self, x, 'None'))
-
-    def smart_update(self, mi, replace_metadata=False):
-        '''
-        Merge the information in C{mi} into self. In case of conflicts, the
-        information in C{mi} takes precedence, unless the information in mi is
-        NULL. If replace_metadata is True, then the information in mi always
-        takes precedence.
-        '''
-        if mi.title and mi.title != _('Unknown'):
-            self.title = mi.title
-
-        if mi.authors and mi.authors[0] != _('Unknown'):
-            self.authors = mi.authors
-
-        for attr in ('author_sort', 'title_sort', 'category',
-                     'publisher', 'series', 'series_index', 'rating',
-                     'isbn', 'application_id', 'manifest', 'spine', 'toc',
-                     'cover', 'guide', 'book_producer',
-                     'timestamp', 'lccn', 'lcc', 'ddc', 'pubdate', 'rights',
-                     'publication_type', 'uuid'):
-            if replace_metadata:
-                setattr(self, attr, getattr(mi, attr, 1.0 if \
-                        attr == 'series_index' else None))
-            elif hasattr(mi, attr):
-                val = getattr(mi, attr)
-                if val is not None:
-                    setattr(self, attr, val)
-
-        if replace_metadata:
-            self.tags = mi.tags
-        elif mi.tags:
-            self.tags += mi.tags
-        self.tags = list(set(self.tags))
-
-        if mi.author_sort_map:
-            self.author_sort_map.update(mi.author_sort_map)
-
-        if getattr(mi, 'cover_data', False):
-            other_cover = mi.cover_data[-1]
-            self_cover = self.cover_data[-1] if self.cover_data else ''
-            if not self_cover: self_cover = ''
-            if not other_cover: other_cover = ''
-            if len(other_cover) > len(self_cover):
-                self.cover_data = mi.cover_data
-
-        if replace_metadata:
-            self.comments = getattr(mi, 'comments', '')
-        else:
-            my_comments = getattr(self, 'comments', '')
-            other_comments = getattr(mi, 'comments', '')
-            if not my_comments:
-                my_comments = ''
-            if not other_comments:
-                other_comments = ''
-            if len(other_comments.strip()) > len(my_comments.strip()):
-                self.comments = other_comments
-
-        other_lang = getattr(mi, 'language', None)
-        if other_lang and other_lang.lower() != 'und':
-            self.language = other_lang
-
-
-    def format_series_index(self):
-        try:
-            x = float(self.series_index)
-        except ValueError:
-            x = 1
-        return fmt_sidx(x)
-
-    def authors_from_string(self, raw):
-        self.authors = string_to_authors(raw)
-
-    def format_authors(self):
-        return authors_to_string(self.authors)
-
-    def format_tags(self):
-        return u', '.join([unicode(t) for t in self.tags])
-
-    def format_rating(self):
-        return unicode(self.rating)
-
-    def __unicode__(self):
-        ans = []
-        def fmt(x, y):
-            ans.append(u'%-20s: %s'%(unicode(x), unicode(y)))
-
-        fmt('Title', self.title)
-        if self.title_sort:
-            fmt('Title sort', self.title_sort)
-        if self.authors:
-            fmt('Author(s)',  authors_to_string(self.authors) + \
-               ((' [' + self.author_sort + ']') if self.author_sort else ''))
-        if self.publisher:
-            fmt('Publisher', self.publisher)
-        if getattr(self, 'book_producer', False):
-            fmt('Book Producer', self.book_producer)
-        if self.category:
-            fmt('Category', self.category)
-        if self.comments:
-            fmt('Comments', self.comments)
-        if self.isbn:
-            fmt('ISBN', self.isbn)
-        if self.tags:
-            fmt('Tags', u', '.join([unicode(t) for t in self.tags]))
-        if self.series:
-            fmt('Series', self.series + ' #%s'%self.format_series_index())
-        if self.language:
-            fmt('Language', self.language)
-        if self.rating is not None:
-            fmt('Rating', self.rating)
-        if self.timestamp is not None:
-            fmt('Timestamp', isoformat(self.timestamp))
-        if self.pubdate is not None:
-            fmt('Published', isoformat(self.pubdate))
-        if self.rights is not None:
-            fmt('Rights', unicode(self.rights))
-        if self.lccn:
-            fmt('LCCN', unicode(self.lccn))
-        if self.lcc:
-            fmt('LCC', unicode(self.lcc))
-        if self.ddc:
-            fmt('DDC', unicode(self.ddc))
-
-        return u'\n'.join(ans)
-
-    def to_html(self):
-        ans = [(_('Title'), unicode(self.title))]
-        ans += [(_('Author(s)'), (authors_to_string(self.authors) if self.authors else _('Unknown')))]
-        ans += [(_('Publisher'), unicode(self.publisher))]
-        ans += [(_('Producer'), unicode(self.book_producer))]
-        ans += [(_('Comments'), unicode(self.comments))]
-        ans += [('ISBN', unicode(self.isbn))]
-        if self.lccn:
-            ans += [('LCCN', unicode(self.lccn))]
-        if self.lcc:
-            ans += [('LCC', unicode(self.lcc))]
-        if self.ddc:
-            ans += [('DDC', unicode(self.ddc))]
-        ans += [(_('Tags'), u', '.join([unicode(t) for t in self.tags]))]
-        if self.series:
-            ans += [(_('Series'), unicode(self.series)+ ' #%s'%self.format_series_index())]
-        ans += [(_('Language'), unicode(self.language))]
-        if self.timestamp is not None:
-            ans += [(_('Timestamp'), unicode(self.timestamp.isoformat(' ')))]
-        if self.pubdate is not None:
-            ans += [(_('Published'), unicode(self.pubdate.isoformat(' ')))]
-        if self.rights is not None:
-            ans += [(_('Rights'), unicode(self.rights))]
-        for i, x in enumerate(ans):
-            ans[i] = u'<tr><td><b>%s</b></td><td>%s</td></tr>'%x
-        return u'<table>%s</table>'%u'\n'.join(ans)
-
-    def __str__(self):
-        return self.__unicode__().encode('utf-8')
-
-    def __nonzero__(self):
-        return bool(self.title or self.author or self.comments or self.tags)
 
 def check_isbn10(isbn):
     try:

src/calibre/ebooks/metadata/book/__init__.py

@@ -11,48 +11,45 @@ an empty list/dictionary for complex types and (None, None) for cover_data
 '''
 
 SOCIAL_METADATA_FIELDS = frozenset([
-    'tags', # Ordered list
+    'tags',             # Ordered list
-    # A floating point number between 0 and 10
+    'rating',           # A floating point number between 0 and 10
-    'rating',
+    'comments',         # A simple HTML enabled string
-    # A simple HTML enabled string
+    'series',           # A simple string
-    'comments',
+    'series_index',     # A floating point number
-    # A simple string
-    'series',
-    # A floating point number
-    'series_index',
     # Of the form { scheme1:value1, scheme2:value2}
     # For example: {'isbn':'123456789', 'doi':'xxxx', ... }
     'classifiers',
-    'isbn', # Pseudo field for convenience, should get/set isbn classifier
+])
 
+'''
+The list of names that convert to classifiers when in get and set.
+'''
+
+TOP_LEVEL_CLASSIFIERS = frozenset([
+    'isbn',
 ])
 
 PUBLICATION_METADATA_FIELDS = frozenset([
-    # title must never be None. Should be _('Unknown')
+    'title',            # title must never be None. Should be _('Unknown')
-    'title',
     # Pseudo field that can be set, but if not set is auto generated
     # from title and languages
     'title_sort',
-    # Ordered list of authors. Must never be None, can be [_('Unknown')]
+    'authors',          # Ordered list. Must never be None, can be [_('Unknown')]
-    'authors',
+    'author_sort_map',  # Map of sort strings for each author
-    # Map of sort strings for each author
-    'author_sort_map',
     # Pseudo field that can be set, but if not set is auto generated
     # from authors and languages
     'author_sort',
     'book_producer',
-    # Dates and times must be timezone aware
+    'timestamp',        # Dates and times must be timezone aware
-    'timestamp',
     'pubdate',
     'rights',
     # So far only known publication type is periodical:calibre
     # If None, means book
     'publication_type',
-    # A UUID usually of type 4
+    'uuid',             # A UUID usually of type 4
-    'uuid',
+    'language',         # the primary language of this book
-    'languages', # ordered list
+    'languages',        # ordered list
-    # Simple string, no special semantics
+    'publisher',        # Simple string, no special semantics
-    'publisher',
     # Absolute path to image file encoded in filesystem_encoding
     'cover',
     # Of the form (format, data) where format is, for e.g. 'jpeg', 'png', 'gif'...
@@ -69,33 +66,56 @@ BOOK_STRUCTURE_FIELDS = frozenset([
     ])
 
 USER_METADATA_FIELDS = frozenset([
-    # A dict of a form to be specified
+    # A dict of dicts similar to field_metadata. Each field description dict
+    # also contains a value field with the key #value#.
     'user_metadata',
 ])
 
 DEVICE_METADATA_FIELDS = frozenset([
-    # Ordered list of strings
+    'device_collections',   # Ordered list of strings
-    'device_collections',
+    'lpath',                # Unicode, / separated
-    'lpath', # Unicode, / separated
+    'size',                 # In bytes
-    # In bytes
+    'mime',                 # Mimetype of the book file being represented
-    'size',
+
-    # Mimetype of the book file being represented
-    'mime',
 ])
 
 CALIBRE_METADATA_FIELDS = frozenset([
-    # An application id
+    'application_id',   # An application id, currently set to the db_id.
-    # Semantics to be defined. Is it a db key? a db name + key? A uuid?
+    # the calibre primary key of the item.
-    'application_id',
+    'db_id',            # the calibre primary key of the item.
+                        # TODO: NEWMETA: May want to remove once Sony's no longer use it
     ]
 )
 
+ALL_METADATA_FIELDS =      SOCIAL_METADATA_FIELDS.union(
+                           PUBLICATION_METADATA_FIELDS).union(
+                           BOOK_STRUCTURE_FIELDS).union(
+                           USER_METADATA_FIELDS).union(
+                           DEVICE_METADATA_FIELDS).union(
+                           CALIBRE_METADATA_FIELDS)
 
-SERIALIZABLE_FIELDS = SOCIAL_METADATA_FIELDS.union(
+# All fields except custom fields
-                      USER_METADATA_FIELDS).union(
+STANDARD_METADATA_FIELDS = SOCIAL_METADATA_FIELDS.union(
-                      PUBLICATION_METADATA_FIELDS).union(
+                           PUBLICATION_METADATA_FIELDS).union(
-                      CALIBRE_METADATA_FIELDS).union(
+                           BOOK_STRUCTURE_FIELDS).union(
-        frozenset(['lpath'])) # I don't think we need device_collections
+                           DEVICE_METADATA_FIELDS).union(
+                           CALIBRE_METADATA_FIELDS)
 
-# Serialization of covers/thumbnails will have to be handled carefully, maybe
+# Metadata fields that smart update should copy without special handling
-# as an option to the serializer class
+COPYABLE_METADATA_FIELDS = SOCIAL_METADATA_FIELDS.union(
+                           PUBLICATION_METADATA_FIELDS).union(
+                           BOOK_STRUCTURE_FIELDS).union(
+                           DEVICE_METADATA_FIELDS).union(
+                           CALIBRE_METADATA_FIELDS) - \
+                           frozenset(['title', 'title_sort', 'authors',
+                                      'author_sort', 'author_sort_map' 'comments',
+                                      'cover_data', 'tags', 'language', 'lpath',
+                                      'size'])
+
+SERIALIZABLE_FIELDS =      SOCIAL_METADATA_FIELDS.union(
+                           USER_METADATA_FIELDS).union(
+                           PUBLICATION_METADATA_FIELDS).union(
+                           CALIBRE_METADATA_FIELDS).union(
+                           DEVICE_METADATA_FIELDS) - \
+                           frozenset(['device_collections'])
+                           # device_collections is rebuilt when needed

src/calibre/ebooks/metadata/book/base.py

@@ -6,8 +6,15 @@ __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
 import copy
+import traceback
+
+from calibre import prints
+from calibre.ebooks.metadata.book import COPYABLE_METADATA_FIELDS
+from calibre.ebooks.metadata.book import STANDARD_METADATA_FIELDS
+from calibre.ebooks.metadata.book import TOP_LEVEL_CLASSIFIERS
+from calibre.utils.date import isoformat, format_date
+
 
-from calibre.ebooks.metadata.book import RESERVED_METADATA_FIELDS
 
 NULL_VALUES = {
                 'user_metadata': {},
@@ -19,103 +26,356 @@ NULL_VALUES = {
                 'author_sort_map': {},
                 'authors'      : [_('Unknown')],
                 'title'        : _('Unknown'),
+                'language'     : 'und'
 }
 
 class Metadata(object):
 
     '''
-    This class must expose a superset of the API of MetaInformation in terms
+    A class representing all the metadata for a book.
-    of attribute access and methods. Only the __init__ method is different.
-    MetaInformation will simply become a function that creates and fills in
-    the attributes of this class.
 
     Please keep the method based API of this class to a minimum. Every method
     becomes a reserved field name.
     '''
 
-    def __init__(self):
+    def __init__(self, title, authors=(_('Unknown'),), other=None):
+        '''
+        @param title: title or ``_('Unknown')``
+        @param authors: List of strings or []
+        @param other: None or a metadata object
+        '''
         object.__setattr__(self, '_data', copy.deepcopy(NULL_VALUES))
+        if other is not None:
+            self.smart_update(other)
+        else:
+            if title:
+                self.title = title
+            if authors:
+                #: List of strings or []
+                self.author = list(authors) if authors else []# Needed for backward compatibility
+                self.authors = list(authors) if authors else []
 
     def __getattribute__(self, field):
         _data = object.__getattribute__(self, '_data')
-        if field in RESERVED_METADATA_FIELDS:
+        if field in TOP_LEVEL_CLASSIFIERS:
+            return _data.get('classifiers').get(field, None)
+        if field in STANDARD_METADATA_FIELDS:
             return _data.get(field, None)
         try:
             return object.__getattribute__(self, field)
         except AttributeError:
             pass
         if field in _data['user_metadata'].iterkeys():
-            # TODO: getting user metadata values
+            return _data['user_metadata'][field]['#value#']
-            pass
         raise AttributeError(
                 'Metadata object has no attribute named: '+ repr(field))
 
-
+    def __setattr__(self, field, val, extra=None):
-    def __setattr__(self, field, val):
         _data = object.__getattribute__(self, '_data')
-        if field in RESERVED_METADATA_FIELDS:
+        if field in TOP_LEVEL_CLASSIFIERS:
-            if field != 'user_metadata':
+            _data['classifiers'].update({field: val})
-                if not val:
+        elif field in STANDARD_METADATA_FIELDS:
-                    val = NULL_VALUES[field]
+            if val is None:
-                _data[field] = val
+                val = NULL_VALUES.get(field, None)
-            else:
+            _data[field] = val
-                raise AttributeError('You cannot set user_metadata directly.')
         elif field in _data['user_metadata'].iterkeys():
-            # TODO: Setting custom column values
+            _data['user_metadata'][field]['#value#'] = val
-            pass
+            _data['user_metadata'][field]['#extra#'] = extra
         else:
             # You are allowed to stick arbitrary attributes onto this object as
-            # long as they dont conflict with global or user metadata names
+            # long as they don't conflict with global or user metadata names
             # Don't abuse this privilege
             self.__dict__[field] = val
 
+    def get(self, field, default=None):
+        if default is not None:
+            try:
+                return self.__getattribute__(field)
+            except AttributeError:
+                return default
+        return self.__getattribute__(field)
+
+    def get_extra(self, field):
+        _data = object.__getattribute__(self, '_data')
+        if field in _data['user_metadata'].iterkeys():
+            return _data['user_metadata'][field]['#extra#']
+        raise AttributeError(
+                'Metadata object has no attribute named: '+ repr(field))
+
+    def set(self, field, val, extra=None):
+        self.__setattr__(field, val, extra)
+
     @property
-    def user_metadata_names(self):
+    def user_metadata_keys(self):
         'The set of user metadata names this object knows about'
         _data = object.__getattribute__(self, '_data')
         return frozenset(_data['user_metadata'].iterkeys())
 
-    # Old MetaInformation API {{{
+    def get_all_user_metadata(self, make_copy):
-    def copy(self):
+        '''
-        pass
+        return a dict containing all the custom field metadata associated with
+        the book.
+        '''
+        _data = object.__getattribute__(self, '_data')
+        user_metadata = _data['user_metadata']
+        if not make_copy:
+            return user_metadata
+        res = {}
+        for k in user_metadata:
+            res[k] = copy.deepcopy(user_metadata[k])
+        return res
 
+    def get_user_metadata(self, field, make_copy):
+        '''
+        return field metadata from the object if it is there. Otherwise return
+        None. field is the key name, not the label. Return a copy if requested,
+        just in case the user wants to change values in the dict.
+        '''
+        _data = object.__getattribute__(self, '_data')
+        _data = _data['user_metadata']
+        if field in _data:
+            if make_copy:
+                return copy.deepcopy(_data[field])
+            return _data[field]
+        return None
+
+    def set_all_user_metadata(self, metadata):
+        '''
+        store custom field metadata into the object. Field is the key name
+        not the label
+        '''
+        if metadata is None:
+            traceback.print_stack()
+        else:
+            for key in metadata:
+                self.set_user_metadata(key, metadata[key])
+
+    def set_user_metadata(self, field, metadata):
+        '''
+        store custom field metadata for one column into the object. Field is
+        the key name not the label
+        '''
+        if field is not None:
+            if metadata is None:
+                traceback.print_stack()
+            metadata = copy.deepcopy(metadata)
+            if '#value#' not in metadata:
+                if metadata['datatype'] == 'text' and metadata['is_multiple']:
+                    metadata['#value#'] = []
+                else:
+                    metadata['#value#'] = None
+            _data = object.__getattribute__(self, '_data')
+            _data['user_metadata'][field] = metadata
+
+    def get_all_non_none_attributes(self):
+        '''
+        Return a dictionary containing all non-None metadata fields, including
+        the custom ones.
+        '''
+        result = {}
+        _data = object.__getattribute__(self, '_data')
+        for attr in STANDARD_METADATA_FIELDS:
+            v = _data.get(attr, None)
+            if v is not None:
+                result[attr] = v
+        for attr in _data['user_metadata'].iterkeys():
+            v = _data['user_metadata'][attr]['#value#']
+            if v is not None:
+                result[attr] = v
+                if _data['user_metadata'][attr]['datatype'] == 'series':
+                    result[attr+'_index'] = _data['user_metadata'][attr]['#extra#']
+        return result
+
+    # Old Metadata API {{{
     def print_all_attributes(self):
-        pass
+        for x in STANDARD_METADATA_FIELDS:
+            prints('%s:'%x, getattr(self, x, 'None'))
+        for x in self.user_metadata_keys:
+            meta = self.get_user_metadata(x, make_copy=False)
+            if meta is not None:
+                prints(x, meta)
+        prints('--------------')
 
     def smart_update(self, other, replace_metadata=False):
-        pass
+        '''
+        Merge the information in C{other} into self. In case of conflicts, the information
+        in C{other} takes precedence, unless the information in other is NULL.
+        '''
+        def copy_not_none(dest, src, attr):
+            v = getattr(src, attr, None)
+            if v is not None:
+                setattr(dest, attr, copy.deepcopy(v))
 
-    def format_series_index(self):
+        if other.title and other.title != _('Unknown'):
-        pass
+            self.title = other.title
+            if hasattr(other, 'title_sort'):
+                self.title_sort = other.title_sort
+
+        if other.authors and other.authors[0] != _('Unknown'):
+            self.authors = other.authors
+            if hasattr(other, 'author_sort_map'):
+                self.author_sort_map = other.author_sort_map
+            if hasattr(other, 'author_sort'):
+                self.author_sort = other.author_sort
+
+        if replace_metadata:
+            for attr in COPYABLE_METADATA_FIELDS:
+                setattr(self, attr, getattr(other, attr, 1.0 if \
+                        attr == 'series_index' else None))
+            self.tags = other.tags
+            self.cover_data = getattr(other, 'cover_data', '')
+            self.set_all_user_metadata(other.get_all_user_metadata(make_copy=True))
+            copy_not_none(self, other, 'lpath')
+            copy_not_none(self, other, 'size')
+            copy_not_none(self, other, 'comments')
+            # language is handled below
+        else:
+            for attr in COPYABLE_METADATA_FIELDS:
+                if hasattr(other, attr):
+                    copy_not_none(self, other, attr)
+                    val = getattr(other, attr)
+                    if val is not None:
+                        setattr(self, attr, copy.deepcopy(val))
+            if other.tags:
+                self.tags += list(set(self.tags + other.tags))
+            if getattr(other, 'cover_data', False):
+                other_cover = other.cover_data[-1]
+                self_cover = self.cover_data[-1] if self.cover_data else ''
+                if not self_cover: self_cover = ''
+                if not other_cover: other_cover = ''
+                if len(other_cover) > len(self_cover):
+                    self.cover_data = other.cover_data
+            if getattr(other, 'user_metadata_keys', None):
+                for x in other.user_metadata_keys:
+                    meta = other.get_user_metadata(x, make_copy=True)
+                    if meta is not None:
+                        self.set_user_metadata(x, meta) # get... did the deepcopy
+            my_comments = getattr(self, 'comments', '')
+            other_comments = getattr(other, 'comments', '')
+            if not my_comments:
+                my_comments = ''
+            if not other_comments:
+                other_comments = ''
+            if len(other_comments.strip()) > len(my_comments.strip()):
+                self.comments = other_comments
+        other_lang = getattr(other, 'language', None)
+        if other_lang and other_lang.lower() != 'und':
+            self.language = other_lang
+
+    def format_series_index(self, val=None):
+        from calibre.ebooks.metadata import fmt_sidx
+        v = self.series_index if val is None else val
+        try:
+            x = float(v)
+        except ValueError:
+            x = 1
+        return fmt_sidx(x)
 
     def authors_from_string(self, raw):
-        pass
+        from calibre.ebooks.metadata import string_to_authors
+        self.authors = string_to_authors(raw)
 
     def format_authors(self):
-        pass
+        from calibre.ebooks.metadata import authors_to_string
+        return authors_to_string(self.authors)
 
     def format_tags(self):
-        pass
+        return u', '.join([unicode(t) for t in self.tags])
 
     def format_rating(self):
         return unicode(self.rating)
 
+    def format_custom_field(self, key):
+        '''
+        returns the tuple (field_name, formatted_value)
+        '''
+        cmeta = self.get_user_metadata(key, make_copy=False)
+        name = unicode(cmeta['name'])
+        res = self.get(key, None)
+        if res is not None:
+            datatype = cmeta['datatype']
+            if datatype == 'text' and cmeta['is_multiple']:
+                res = u', '.join(res)
+            elif datatype == 'series':
+                res = res + ' [%s]'%self.format_series_index(val=self.get_extra(key))
+            elif datatype == 'datetime':
+                res = format_date(res, cmeta['display'].get('date_format','dd MMM yyyy'))
+            elif datatype == 'bool':
+                res = _('Yes') if res else _('No')
+        return (name, unicode(res))
+
     def __unicode__(self):
-        pass
+        from calibre.ebooks.metadata import authors_to_string
+        ans = []
+        def fmt(x, y):
+            ans.append(u'%-20s: %s'%(unicode(x), unicode(y)))
+
+        fmt('Title', self.title)
+        if self.title_sort:
+            fmt('Title sort', self.title_sort)
+        if self.authors:
+            fmt('Author(s)',  authors_to_string(self.authors) + \
+               ((' [' + self.author_sort + ']') if self.author_sort else ''))
+        if self.publisher:
+            fmt('Publisher', self.publisher)
+        if getattr(self, 'book_producer', False):
+            fmt('Book Producer', self.book_producer)
+        if self.comments:
+            fmt('Comments', self.comments)
+        if self.isbn:
+            fmt('ISBN', self.isbn)
+        if self.tags:
+            fmt('Tags', u', '.join([unicode(t) for t in self.tags]))
+        if self.series:
+            fmt('Series', self.series + ' #%s'%self.format_series_index())
+        if self.language:
+            fmt('Language', self.language)
+        if self.rating is not None:
+            fmt('Rating', self.rating)
+        if self.timestamp is not None:
+            fmt('Timestamp', isoformat(self.timestamp))
+        if self.pubdate is not None:
+            fmt('Published', isoformat(self.pubdate))
+        if self.rights is not None:
+            fmt('Rights', unicode(self.rights))
+        for key in self.user_metadata_keys:
+            val = self.get(key, None)
+            if val is not None:
+                (name, val) = self.format_custom_field(key)
+                fmt(name, unicode(val))
+        return u'\n'.join(ans)
 
     def to_html(self):
-        pass
+        from calibre.ebooks.metadata import authors_to_string
+        ans = [(_('Title'), unicode(self.title))]
+        ans += [(_('Author(s)'), (authors_to_string(self.authors) if self.authors else _('Unknown')))]
+        ans += [(_('Publisher'), unicode(self.publisher))]
+        ans += [(_('Producer'), unicode(self.book_producer))]
+        ans += [(_('Comments'), unicode(self.comments))]
+        ans += [('ISBN', unicode(self.isbn))]
+        ans += [(_('Tags'), u', '.join([unicode(t) for t in self.tags]))]
+        if self.series:
+            ans += [(_('Series'), unicode(self.series)+ ' #%s'%self.format_series_index())]
+        ans += [(_('Language'), unicode(self.language))]
+        if self.timestamp is not None:
+            ans += [(_('Timestamp'), unicode(self.timestamp.isoformat(' ')))]
+        if self.pubdate is not None:
+            ans += [(_('Published'), unicode(self.pubdate.isoformat(' ')))]
+        if self.rights is not None:
+            ans += [(_('Rights'), unicode(self.rights))]
+        for key in self.user_metadata_keys:
+            val = self.get(key, None)
+            if val is not None:
+                (name, val) = self.format_custom_field(key)
+                ans += [(name, val)]
+        for i, x in enumerate(ans):
+            ans[i] = u'<tr><td><b>%s</b></td><td>%s</td></tr>'%x
+        return u'<table>%s</table>'%u'\n'.join(ans)
 
     def __str__(self):
         return self.__unicode__().encode('utf-8')
 
     def __nonzero__(self):
-        return True
+        return bool(self.title or self.author or self.comments or self.tags)
 
     # }}}
-
-# We don't need reserved field names for this object any more. Lets just use a
-# protocol like the last char of a user field label should be _ when using this
-# object
-# So mi.tags returns the builtin tags and mi.tags_ returns the user tags
-

src/calibre/ebooks/metadata/book/json_codec.py

@@ -0,0 +1,124 @@
+'''
+Created on 4 Jun 2010
+
+@author: charles
+'''
+
+from base64 import b64encode, b64decode
+import json
+import traceback
+
+from calibre.ebooks.metadata.book import SERIALIZABLE_FIELDS
+from calibre.constants import filesystem_encoding, preferred_encoding
+from calibre.library.field_metadata import FieldMetadata
+from calibre.utils.date import parse_date, isoformat, UNDEFINED_DATE
+
+# Translate datetimes to and from strings. The string form is the datetime in
+# UTC. The returned date is also UTC
+def string_to_datetime(src):
+    if src == "None":
+        return None
+#    dt = strptime(src, '%d %m %Y %H:%M:%S', assume_utc=True, as_utc=True)
+#    if dt == UNDEFINED_DATE:
+#        return None
+    return parse_date(src)
+
+def datetime_to_string(dateval):
+    if dateval is None or dateval == UNDEFINED_DATE:
+        return "None"
+#    tt = date_to_utc(dateval).timetuple()
+#    res = "%02d %02d %04d %02d:%02d:%02d"%(tt.tm_mday, tt.tm_mon, tt.tm_year,
+#                                     tt.tm_hour, tt.tm_min, tt.tm_sec)
+    return isoformat(dateval)
+
+def encode_thumbnail(thumbnail):
+    '''
+    Encode the image part of a thumbnail, then return the 3 part tuple
+    '''
+    if thumbnail is None:
+        return None
+    return (thumbnail[0], thumbnail[1], b64encode(str(thumbnail[2])))
+
+def decode_thumbnail(tup):
+    '''
+    Decode an encoded thumbnail into its 3 component parts
+    '''
+    if tup is None:
+        return None
+    return (tup[0], tup[1], b64decode(tup[2]))
+
+class JsonCodec(object):
+
+    def __init__(self):
+        self.field_metadata = FieldMetadata()
+
+    def encode_to_file(self, file, booklist):
+        json.dump(self.encode_booklist_metadata(booklist), file, indent=2, encoding='utf-8')
+
+    def encode_booklist_metadata(self, booklist):
+        result = []
+        for book in booklist:
+            result.append(self.encode_book_metadata(book))
+        return result
+
+    def encode_book_metadata(self, book):
+        result = {}
+        for key in SERIALIZABLE_FIELDS:
+            result[key] = self.encode_metadata_attr(book, key)
+        return result
+
+    def encode_metadata_attr(self, book, key):
+        if key == 'user_metadata':
+            meta = book.get_all_user_metadata(make_copy=True)
+            for k in meta:
+                if meta[k]['datatype'] == 'datetime':
+                    meta[k]['#value#'] = datetime_to_string(meta[k]['#value#'])
+            return meta
+        if key in self.field_metadata:
+            datatype = self.field_metadata[key]['datatype']
+        else:
+            datatype = None
+        value = book.get(key)
+        if key == 'thumbnail':
+            return encode_thumbnail(value)
+        elif isinstance(value, str): # str includes bytes
+            enc = filesystem_encoding if key == 'lpath' else preferred_encoding
+            return value.decode(enc, 'replace')
+        elif isinstance(value, (list, tuple)):
+            return [x.decode(preferred_encoding, 'replace') if
+                    isinstance(x, str) else x for x in value]
+        elif datatype == 'datetime':
+            return datetime_to_string(value)
+        else:
+            return value
+
+    def decode_from_file(self, file, booklist, book_class, prefix):
+        js = []
+        try:
+            js = json.load(file, encoding='utf-8')
+            for item in js:
+                book = book_class(prefix, item.get('lpath', None))
+                for key in item.keys():
+                    meta = self.decode_metadata(key, item[key])
+                    if key == 'user_metadata':
+                        book.set_all_user_metadata(meta)
+                    else:
+                        setattr(book, key, meta)
+                booklist.append(book)
+        except:
+            print 'exception during JSON decoding'
+            traceback.print_exc()
+            booklist = []
+
+    def decode_metadata(self, key, value):
+        if key == 'user_metadata':
+            for k in value:
+                if value[k]['datatype'] == 'datetime':
+                    value[k]['#value#'] = string_to_datetime(value[k]['#value#'])
+            return value
+        elif key in self.field_metadata:
+            if self.field_metadata[key]['datatype'] == 'datetime':
+                return string_to_datetime(value)
+        if key == 'thumbnail':
+            return decode_thumbnail(value)
+        return value

src/calibre/ebooks/metadata/epub.py

@@ -164,7 +164,7 @@ def get_cover(opf, opf_path, stream, reader=None):
             return render_html_svg_workaround(cpage, default_log)
 
 def get_metadata(stream, extract_cover=True):
-    """ Return metadata as a :class:`MetaInformation` object """
+    """ Return metadata as a :class:`Metadata` object """
     stream.seek(0)
     reader = OCFZipReader(stream)
     mi = MetaInformation(reader.opf)

src/calibre/ebooks/metadata/fetch.py

@@ -29,7 +29,7 @@ class MetadataSource(Plugin): # {{{
     future use.
 
     The fetch method must store the results in `self.results` as a list of
-    :class:`MetaInformation` objects. If there is an error, it should be stored
+    :class:`Metadata` objects. If there is an error, it should be stored
     in `self.exception` and `self.tb` (for the traceback).
     '''
 

src/calibre/ebooks/metadata/isbndb.py

@@ -8,7 +8,7 @@ import sys, re
 from urllib import quote
 
 from calibre.utils.config import OptionParser
-from calibre.ebooks.metadata import MetaInformation
+from calibre.ebooks.metadata.book.base import Metadata
 from calibre.ebooks.BeautifulSoup import BeautifulStoneSoup
 from calibre import browser
 
@@ -42,10 +42,10 @@ def fetch_metadata(url, max=100, timeout=5.):
     return books
 
 
-class ISBNDBMetadata(MetaInformation):
+class ISBNDBMetadata(Metadata):
 
     def __init__(self, book):
-        MetaInformation.__init__(self, None, [])
+        Metadata.__init__(self, None, [])
 
         self.isbn = book.get('isbn13', book.get('isbn'))
         self.title = book.find('titlelong').string

src/calibre/ebooks/metadata/opf2.py

@@ -16,7 +16,8 @@ from lxml import etree
 from calibre.ebooks.chardet import xml_to_unicode
 from calibre.constants import __appname__, __version__, filesystem_encoding
 from calibre.ebooks.metadata.toc import TOC
-from calibre.ebooks.metadata import MetaInformation, string_to_authors
+from calibre.ebooks.metadata import string_to_authors, MetaInformation
+from calibre.ebooks.metadata.book.base import Metadata
 from calibre.utils.date import parse_date, isoformat
 from calibre.utils.localization import get_lang
 
@@ -926,16 +927,16 @@ class OPF(object):
                 setattr(self, attr, val)
 
 
-class OPFCreator(MetaInformation):
+class OPFCreator(Metadata):
 
-    def __init__(self, base_path, *args, **kwargs):
+    def __init__(self, base_path, other):
         '''
         Initialize.
         @param base_path: An absolute path to the directory in which this OPF file
         will eventually be. This is used by the L{create_manifest} method
         to convert paths to files into relative paths.
         '''
-        MetaInformation.__init__(self, *args, **kwargs)
+        Metadata.__init__(self, title='', other=other)
         self.base_path = os.path.abspath(base_path)
         if self.application_id is None:
             self.application_id = str(uuid.uuid4())
@@ -1187,7 +1188,7 @@ def metadata_to_opf(mi, as_string=True):
     factory(DC('contributor'), mi.book_producer, __appname__, 'bkp')
     if hasattr(mi.pubdate, 'isoformat'):
         factory(DC('date'), isoformat(mi.pubdate))
-    if mi.category:
+    if hasattr(mi, 'category') and mi.category:
         factory(DC('type'), mi.category)
     if mi.comments:
         factory(DC('description'), mi.comments)

src/calibre/gui2/book_details.py

@@ -28,6 +28,8 @@ WEIGHTS[_('Tags')] = 4
 
 def render_rows(data):
     keys = data.keys()
+    # First sort by name. The WEIGHTS sort will preserve this sub-order
+    keys.sort(cmp=lambda x, y: cmp(x.lower(), y.lower()))
     keys.sort(cmp=lambda x, y: cmp(WEIGHTS[x], WEIGHTS[y]))
     rows = []
     for key in keys:

src/calibre/gui2/library/models.py

@@ -323,7 +323,11 @@ class BooksModel(QAbstractTableModel): # {{{
             data[_('Series')] = \
                 _('Book <font face="serif">%s</font> of %s.')%\
                     (sidx, prepare_string_for_xml(series))
-
+        mi = self.db.get_metadata(idx)
+        for key in mi.user_metadata_keys:
+            name, val = mi.format_custom_field(key)
+            if val is not None:
+                data[name] = val
         return data
 
     def set_cache(self, idx):
@@ -372,7 +376,6 @@ class BooksModel(QAbstractTableModel): # {{{
         return ans
 
     def get_metadata(self, rows, rows_are_ids=False, full_metadata=False):
-        # Should this add the custom columns? It doesn't at the moment
         metadata, _full_metadata = [], []
         if not rows_are_ids:
             rows = [self.db.id(row.row()) for row in rows]
@@ -1053,7 +1056,7 @@ class DeviceBooksModel(BooksModel): # {{{
             if hasattr(cdata, 'image_path'):
                 img.load(cdata.image_path)
             else:
-                img.loadFromData(cdata)
+                img.loadFromData(cdata[2])
             if img.isNull():
                 img = self.default_image
             data['cover'] = img

src/calibre/gui2/preferences/save_template.py

@@ -8,10 +8,8 @@ __docformat__ = 'restructuredtext en'
 
 from PyQt4.Qt import QWidget, pyqtSignal
 
-from calibre.gui2 import error_dialog
 from calibre.gui2.preferences.save_template_ui import Ui_Form
-from calibre.library.save_to_disk import FORMAT_ARG_DESCS, \
+from calibre.library.save_to_disk import FORMAT_ARG_DESCS
-        preprocess_template
 
 class SaveTemplate(QWidget, Ui_Form):
 
@@ -41,18 +39,21 @@ class SaveTemplate(QWidget, Ui_Form):
         self.changed_signal.emit()
 
     def validate(self):
-        tmpl = preprocess_template(self.opt_template.text())
+        # TODO: NEWMETA: I haven't figured out how to get the custom columns
-        fa = {}
+        # into here, so for the moment make all templates valid.
-        for x in FORMAT_ARG_DESCS.keys():
-            fa[x]='random long string'
-        try:
-            tmpl.format(**fa)
-        except Exception, err:
-            error_dialog(self, _('Invalid template'),
-                    '<p>'+_('The template %s is invalid:')%tmpl + \
-                    '<br>'+str(err), show=True)
-            return False
         return True
+#        tmpl = preprocess_template(self.opt_template.text())
+#        fa = {}
+#        for x in FORMAT_ARG_DESCS.keys():
+#            fa[x]='random long string'
+#        try:
+#            tmpl.format(**fa)
+#        except Exception, err:
+#            error_dialog(self, _('Invalid template'),
+#                    '<p>'+_('The template %s is invalid:')%tmpl + \
+#                    '<br>'+str(err), show=True)
+#            return False
+#        return True
 
     def set_value(self, val):
         self.opt_template.set_value(val)

src/calibre/library/database2.py

@@ -521,15 +521,15 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         '''
         Convenience method to return metadata as a L{MetaInformation} object.
         '''
-        aum = self.authors(idx, index_is_id=index_is_id)
+        aut_list = self.authors_with_sort_strings(idx, index_is_id=index_is_id)
-        if aum: aum = [a.strip().replace('|', ',') for a in aum.split(',')]
+        aum = []
+        aus = {}
+        for (author, author_sort) in aut_list:
+            aum.append(author)
+            aus[author] = author_sort
         mi = MetaInformation(self.title(idx, index_is_id=index_is_id), aum)
         mi.author_sort = self.author_sort(idx, index_is_id=index_is_id)
-        if mi.authors:
+        mi.author_sort_map = aus
-            mi.author_sort_map = {}
-            for name, sort in zip(mi.authors, self.authors_sort_strings(idx,
-                index_is_id)):
-                mi.author_sort_map[name] = sort
         mi.comments    = self.comments(idx, index_is_id=index_is_id)
         mi.publisher   = self.publisher(idx, index_is_id=index_is_id)
         mi.timestamp   = self.timestamp(idx, index_is_id=index_is_id)
@@ -546,6 +546,13 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         mi.isbn = self.isbn(idx, index_is_id=index_is_id)
         id = idx if index_is_id else self.id(idx)
         mi.application_id = id
+        for key,meta in self.field_metadata.iteritems():
+            if meta['is_custom']:
+                mi.set_user_metadata(key, meta)
+                mi.set(key, val=self.get_custom(idx, label=meta['label'],
+                                                index_is_id=index_is_id),
+                            extra=self.get_custom_extra(idx, label=meta['label'],
+                                                        index_is_id=index_is_id))
         if get_cover:
             mi.cover = self.cover(id, index_is_id=True, as_path=True)
         return mi
@@ -1084,6 +1091,15 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         if getattr(mi, 'timestamp', None) is not None:
             doit(self.set_timestamp, id, mi.timestamp, notify=False)
         self.set_path(id, True)
+
+        user_mi = mi.get_all_user_metadata(make_copy=False)
+        for key in user_mi.iterkeys():
+            if key in self.field_metadata and \
+                    user_mi[key]['datatype'] == self.field_metadata[key]['datatype']:
+                doit(self.set_custom, id,
+                     val=mi.get(key),
+                     extra=mi.get_extra(key),
+                     label=user_mi[key]['label'])
         self.notify('metadata', [id])
 
     # Given a book, return the list of author sort strings for the book's authors
@@ -1099,6 +1115,19 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             result.append(sort)
         return result
 
+    # Given a book, return the map of author sort strings for the book's authors
+    def authors_with_sort_strings(self, id, index_is_id=False):
+        id = id if index_is_id else self.id(id)
+        aut_strings = self.conn.get('''
+                        SELECT authors.name, authors.sort
+                        FROM authors, books_authors_link as bl
+                        WHERE bl.book=? and authors.id=bl.author
+                        ORDER BY bl.id''', (id,))
+        result = []
+        for (author, sort,) in aut_strings:
+            result.append((author.replace('|', ','), sort))
+        return result
+
     # Given a book, return the author_sort string for authors of the book
     def author_sort_from_book(self, id, index_is_id=False):
         auts = self.authors_sort_strings(id, index_is_id)

src/calibre/library/save_to_disk.py

@@ -6,7 +6,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os, traceback, cStringIO, re
+import os, traceback, cStringIO, re, string
 
 from calibre.utils.config import Config, StringConfig, tweaks
 from calibre.utils.filenames import shorten_components_to, supports_long_names, \
@@ -14,6 +14,7 @@ from calibre.utils.filenames import shorten_components_to, supports_long_names,
 from calibre.ebooks.metadata.opf2 import metadata_to_opf
 from calibre.ebooks.metadata.meta import set_metadata
 from calibre.constants import preferred_encoding, filesystem_encoding
+from calibre.ebooks.metadata import fmt_sidx
 from calibre.ebooks.metadata import title_sort
 from calibre import strftime
 
@@ -97,29 +98,33 @@ def preprocess_template(template):
         template = template.decode(preferred_encoding, 'replace')
     return template
 
+class SafeFormat(string.Formatter):
+    '''
+    Provides a format function that substitutes '' for any missing value
+    '''
+    def get_value(self, key, args, kwargs):
+        try:
+            return kwargs[key]
+        except:
+            return ''
+safe_formatter = SafeFormat()
+
 def safe_format(x, format_args):
-    try:
+    ans = safe_formatter.vformat(x, [], format_args).strip()
-        ans = x.format(**format_args).strip()
+    return re.sub(r'\s+', ' ', ans)
-        return re.sub(r'\s+', ' ', ans)
-    except IndexError: # Thrown if user used [] and index is out of bounds
-        pass
-    except AttributeError: # Thrown if user used a non existing attribute
-        pass
-    return ''
 
 def get_components(template, mi, id, timefmt='%b %Y', length=250,
         sanitize_func=ascii_filename, replace_whitespace=False,
         to_lowercase=False):
     library_order = tweaks['save_template_title_series_sorting'] == 'library_order'
     tsfmt = title_sort if library_order else lambda x: x
-    format_args = dict(**FORMAT_ARGS)
+    format_args = FORMAT_ARGS.copy()
+    format_args.update(mi.get_all_non_none_attributes())
     if mi.title:
         format_args['title'] = tsfmt(mi.title)
     if mi.authors:
         format_args['authors'] = mi.format_authors()
         format_args['author'] = format_args['authors']
-    if mi.author_sort:
-        format_args['author_sort'] = mi.author_sort
     if mi.tags:
         format_args['tags'] = mi.format_tags()
         if format_args['tags'].startswith('/'):
@@ -132,15 +137,25 @@ def get_components(template, mi, id, timefmt='%b %Y', length=250,
         template = re.sub(r'\{series_index[^}]*?\}', '', template)
     if mi.rating is not None:
         format_args['rating'] = mi.format_rating()
-    if mi.isbn:
-        format_args['isbn'] = mi.isbn
-    if mi.publisher:
-        format_args['publisher'] = mi.publisher
     if hasattr(mi.timestamp, 'timetuple'):
         format_args['timestamp'] = strftime(timefmt, mi.timestamp.timetuple())
     if hasattr(mi.pubdate, 'timetuple'):
         format_args['pubdate'] = strftime(timefmt, mi.pubdate.timetuple())
     format_args['id'] = str(id)
+    # Now format the custom fields
+    custom_metadata = mi.get_all_user_metadata(make_copy=False)
+    for key in custom_metadata:
+        if key in format_args:
+            ## TODO: NEWMETA: should ratings be divided by 2? The standard rating isn't...
+            if custom_metadata[key]['datatype'] == 'series':
+                format_args[key] = tsfmt(format_args[key])
+                if key+'_index' in format_args:
+                    format_args[key+'_index'] = fmt_sidx(format_args[key+'_index'])
+            elif custom_metadata[key]['datatype'] == 'datetime':
+                format_args[key] = strftime(timefmt, format_args[key].timetuple())
+            elif custom_metadata[key]['datatype'] == 'bool':
+                format_args[key] = _('yes') if format_args[key] else _('no')
+
     components = [x.strip() for x in template.split('/') if x.strip()]
     components = [safe_format(x, format_args) for x in components]
     components = [sanitize_func(x) for x in components if x]

src/calibre/library/server/mobile.py

@@ -199,6 +199,10 @@ class MobileServer(object):
         CKEYS = [key for key in sorted(CFM.get_custom_fields(),
              cmp=lambda x,y: cmp(CFM[x]['name'].lower(),
                                  CFM[y]['name'].lower()))]
+        # This method uses its own book dict, not the Metadata dict. The loop
+        # below could be changed to use db.get_metadata instead of reading
+        # info directly from the record made by the view, but it doesn't seem
+        # worth it at the moment.
         books = []
         for record in items[(start-1):(start-1)+num]:
             book = {'formats':record[FM['formats']], 'size':record[FM['size']]}

src/calibre/library/server/utils.py

@@ -5,7 +5,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import time
+import time, sys
 
 import cherrypy
 
@@ -44,8 +44,8 @@ def strftime(fmt='%Y/%m/%d %H:%M:%S', dt=None):
     except:
         return _strftime(fmt, nowf().timetuple())
 
-def format_tag_string(tags, sep):
+def format_tag_string(tags, sep, ignore_max=False):
-    MAX = tweaks['max_content_server_tags_shown']
+    MAX = sys.maxint if ignore_max else tweaks['max_content_server_tags_shown']
     if tags:
         tlist = [t.strip() for t in tags.split(sep)]
     else:
@@ -53,5 +53,6 @@ def format_tag_string(tags, sep):
     tlist.sort(cmp=lambda x,y:cmp(x.lower(), y.lower()))
     if len(tlist) > MAX:
         tlist = tlist[:MAX]+['...']
-    return u'%s'%(', '.join(tlist)) if tlist else ''
+    return u'%s:&:%s'%(tweaks['max_content_server_tags_shown'],
+                     ', '.join(tlist)) if tlist else ''
 

src/calibre/library/server/xml.py

@@ -66,6 +66,10 @@ class XMLServer(object):
                 return x.decode(preferred_encoding, 'replace')
             return unicode(x)
 
+        # This method uses its own book dict, not the Metadata dict. The loop
+        # below could be changed to use db.get_metadata instead of reading
+        # info directly from the record made by the view, but it doesn't seem
+        # worth it at the moment.
         for record in items[start:start+num]:
             kwargs = {}
             aus = record[FM['authors']] if record[FM['authors']] else __builtin__._('Unknown')
@@ -85,7 +89,7 @@ class XMLServer(object):
                     'comments'):
                 y = record[FM[x]]
                 if x == 'tags':
-                    y = format_tag_string(y, ',')
+                    y = format_tag_string(y, ',', ignore_max=True)
                 kwargs[x] = serialize(y) if y else ''
 
             c = kwargs.pop('comments')
@@ -107,7 +111,9 @@ class XMLServer(object):
                     name = CFM[key]['name']
                     custcols.append(k)
                     if datatype == 'text' and CFM[key]['is_multiple']:
-                        kwargs[k] = concat(name, format_tag_string(val,'|'))
+                        kwargs[k] = concat('#T#'+name,
+                                           format_tag_string(val,'|',
+                                                             ignore_max=True))
                     elif datatype == 'series':
                         kwargs[k] = concat(name, '%s [%s]'%(val,
                             fmt_sidx(record[CFM.cc_series_index_column_for(key)])))
@@ -138,6 +144,3 @@ class XMLServer(object):
         return etree.tostring(ans, encoding='utf-8', pretty_print=True,
                 xml_declaration=True)
 
-
-
-