From 43adf4226a6d381cf121ad5871b9237af57c58af Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 13 Sep 2010 08:18:09 +0100 Subject: [PATCH 1/2] Rationalize how smart_update knows what to do. Introduced 2 more metadata groups. One is the list of attributes that smart_update is to process specially. The other is the list of attributes that are to be copied if not none (what you called SPECIAL_FIELDS). These two help keep the replace and merge branches in sync. --- src/calibre/ebooks/metadata/book/__init__.py | 17 ++++++++++----- src/calibre/ebooks/metadata/book/base.py | 23 ++++++++++++-------- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/src/calibre/ebooks/metadata/book/__init__.py b/src/calibre/ebooks/metadata/book/__init__.py index 84a88606f2..e087f8072d 100644 --- a/src/calibre/ebooks/metadata/book/__init__.py +++ b/src/calibre/ebooks/metadata/book/__init__.py @@ -101,16 +101,23 @@ STANDARD_METADATA_FIELDS = SOCIAL_METADATA_FIELDS.union( DEVICE_METADATA_FIELDS).union( CALIBRE_METADATA_FIELDS) +# Metadata fields that smart update must do special processing to copy. + +SC_FIELDS_NOT_COPIED = frozenset(['title', 'title_sort', 'authors', + 'author_sort', 'author_sort_map', + 'cover_data', 'tags', 'language']) + +# Metadata fields that smart update should copy only if the source is not None +SC_FIELDS_COPY_NOT_NULL = frozenset(['lpath', 'size', 'comments', 'thumbnail']) + # Metadata fields that smart update should copy without special handling -COPYABLE_METADATA_FIELDS = SOCIAL_METADATA_FIELDS.union( +SC_COPYABLE_FIELDS = SOCIAL_METADATA_FIELDS.union( PUBLICATION_METADATA_FIELDS).union( BOOK_STRUCTURE_FIELDS).union( DEVICE_METADATA_FIELDS).union( CALIBRE_METADATA_FIELDS) - \ - frozenset(['title', 'title_sort', 'authors', - 'author_sort', 'author_sort_map' 'comments', - 'cover_data', 'tags', 'language', 'lpath', - 'size', 'thumbnail']) + SC_FIELDS_NOT_COPIED.union( + SC_FIELDS_COPY_NOT_NULL) SERIALIZABLE_FIELDS = SOCIAL_METADATA_FIELDS.union( USER_METADATA_FIELDS).union( diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py index 7812f81180..8538ed886c 100644 --- a/src/calibre/ebooks/metadata/book/base.py +++ b/src/calibre/ebooks/metadata/book/base.py @@ -9,7 +9,8 @@ import copy import traceback from calibre import prints -from calibre.ebooks.metadata.book import COPYABLE_METADATA_FIELDS +from calibre.ebooks.metadata.book import SC_COPYABLE_FIELDS +from calibre.ebooks.metadata.book import SC_FIELDS_COPY_NOT_NULL from calibre.ebooks.metadata.book import STANDARD_METADATA_FIELDS from calibre.ebooks.metadata.book import TOP_LEVEL_CLASSIFIERS from calibre.utils.date import isoformat, format_date @@ -223,22 +224,23 @@ class Metadata(object): self.author_sort = other.author_sort if replace_metadata: - SPECIAL_FIELDS = frozenset(['lpath', 'size', 'comments', 'thumbnail']) - for attr in COPYABLE_METADATA_FIELDS: + # SPECIAL_FIELDS = frozenset(['lpath', 'size', 'comments', 'thumbnail']) + for attr in SC_COPYABLE_FIELDS: setattr(self, attr, getattr(other, attr, 1.0 if \ attr == 'series_index' else None)) self.tags = other.tags self.cover_data = getattr(other, 'cover_data', - NULL_VALUES['cover_data']) + NULL_VALUES['cover_data']) self.set_all_user_metadata(other.get_all_user_metadata(make_copy=True)) - for x in SPECIAL_FIELDS: + for x in SC_FIELDS_COPY_NOT_NULL: copy_not_none(self, other, x) # language is handled below else: - for attr in COPYABLE_METADATA_FIELDS: - if hasattr(other, attr): - copy_not_none(self, other, attr) - copy_not_none(self, other, 'thumbnail') + for attr in SC_COPYABLE_FIELDS: + copy_not_none(self, other, attr) + for x in SC_FIELDS_COPY_NOT_NULL: + copy_not_none(self, other, x) + if other.tags: # Case-insensitive but case preserving merging lotags = [t.lower() for t in other.tags] @@ -249,6 +251,7 @@ class Metadata(object): oidx = lotags.index(t) self.tags[sidx] = other.tags[oidx] self.tags += [t for t in other.tags if t.lower() in ot-st] + if getattr(other, 'cover_data', False): other_cover = other.cover_data[-1] self_cover = self.cover_data[-1] if self.cover_data else '' @@ -256,6 +259,7 @@ class Metadata(object): if not other_cover: other_cover = '' if len(other_cover) > len(self_cover): self.cover_data = other.cover_data + if getattr(other, 'user_metadata_keys', None): for x in other.user_metadata_keys: meta = other.get_user_metadata(x, make_copy=True) @@ -274,6 +278,7 @@ class Metadata(object): self_tags[sidx] = other.tags[oidx] self_tags += [t for t in other.tags if t.lower() in ot-st] setattr(self, x, self_tags) + my_comments = getattr(self, 'comments', '') other_comments = getattr(other, 'comments', '') if not my_comments: From a85be2ba3229f2e27221fb7f64d25a7e48977ab7 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 13 Sep 2010 11:59:45 +0100 Subject: [PATCH 2/2] Several changes: 1) allow use of unusual standard fields in get_collections. Format them appropriately 2) change metadata.book.base.format_field to handle standard fields. 3) add standard metadata access methods to metadata.book.base. --- src/calibre/devices/usbms/books.py | 18 +----- src/calibre/ebooks/metadata/book/base.py | 71 +++++++++++++++++++++--- src/calibre/gui2/library/models.py | 2 +- src/calibre/library/field_metadata.py | 17 ++++-- 4 files changed, 76 insertions(+), 32 deletions(-) diff --git a/src/calibre/devices/usbms/books.py b/src/calibre/devices/usbms/books.py index 3e13527bd0..2b19027df4 100644 --- a/src/calibre/devices/usbms/books.py +++ b/src/calibre/devices/usbms/books.py @@ -105,21 +105,7 @@ class CollectionsBookList(BookList): attr_name = '' elif attr_name != '': attr_name = '(%s)'%attr_name - - if attr not in cust_field_meta: - cat_name = '%s %s'%(category, attr_name) - else: - fm = cust_field_meta[attr] - if fm['datatype'] == 'bool': - if category: - cat_name = '%s %s'%(_('Yes'), attr_name) - else: - cat_name = '%s %s'%(_('No'), attr_name) - elif fm['datatype'] == 'datetime': - cat_name = '%s %s'%(format_date(category, - fm['display'].get('date_format','dd MMM yyyy')), attr_name) - else: - cat_name = '%s %s'%(category, attr_name) + cat_name = '%s %s'%(category, attr_name) return cat_name.strip() def get_collections(self, collection_attributes): @@ -156,7 +142,7 @@ class CollectionsBookList(BookList): cust_field_meta = book.get_all_user_metadata(make_copy=False) for attr in attrs: attr = attr.strip() - val = meta_vals.get(attr, None) + ign, val = book.format_field(attr, ignore_series_index=True) if not val: continue if isbytestring(val): val = val.decode(preferred_encoding, 'replace') diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py index 8538ed886c..6e0351353f 100644 --- a/src/calibre/ebooks/metadata/book/base.py +++ b/src/calibre/ebooks/metadata/book/base.py @@ -13,6 +13,7 @@ from calibre.ebooks.metadata.book import SC_COPYABLE_FIELDS from calibre.ebooks.metadata.book import SC_FIELDS_COPY_NOT_NULL from calibre.ebooks.metadata.book import STANDARD_METADATA_FIELDS from calibre.ebooks.metadata.book import TOP_LEVEL_CLASSIFIERS +from calibre.library.field_metadata import FieldMetadata from calibre.utils.date import isoformat, format_date @@ -30,6 +31,8 @@ NULL_VALUES = { 'language' : 'und' } +field_metadata = FieldMetadata() + class Metadata(object): ''' @@ -112,6 +115,31 @@ class Metadata(object): _data = object.__getattribute__(self, '_data') return frozenset(_data['user_metadata'].iterkeys()) + def get_standard_metadata(self, field, make_copy): + ''' + return field metadata from the field if it is there. Otherwise return + None. field is the key name, not the label. Return a copy if requested, + just in case the user wants to change values in the dict. + ''' + if field in field_metadata and field_metadata[field]['kind'] == 'field': + if make_copy: + return copy.deepcopy(field_metadata[field]) + return field_metadata[field] + return None + + def get_all_standard_metadata(self, make_copy): + ''' + return a dict containing all the standard field metadata associated with + the book. + ''' + if not make_copy: + return field_metadata + res = {} + for k in field_metadata: + if field_metadata[k]['kind'] == 'field': + res[k] = copy.deepcopy(field_metadata[k]) + return res + def get_all_user_metadata(self, make_copy): ''' return a dict containing all the custom field metadata associated with @@ -315,24 +343,49 @@ class Metadata(object): def format_rating(self): return unicode(self.rating) - def format_custom_field(self, key): + def format_field(self, key, ignore_series_index=False): + from calibre.ebooks.metadata import authors_to_string ''' returns the tuple (field_name, formatted_value) ''' - cmeta = self.get_user_metadata(key, make_copy=False) - name = unicode(cmeta['name']) - res = self.get(key, None) - if res is not None: + if key in self.user_metadata_keys: + res = self.get(key, None) + if res is None or res == '': + return (None, None) + cmeta = self.get_user_metadata(key, make_copy=False) + name = unicode(cmeta['name']) datatype = cmeta['datatype'] if datatype == 'text' and cmeta['is_multiple']: res = u', '.join(res) elif datatype == 'series': - res = res + ' [%s]'%self.format_series_index(val=self.get_extra(key)) + if not ignore_series_index: + res = res + \ + ' [%s]'%self.format_series_index(val=self.get_extra(key)) elif datatype == 'datetime': res = format_date(res, cmeta['display'].get('date_format','dd MMM yyyy')) elif datatype == 'bool': res = _('Yes') if res else _('No') - return (name, unicode(res)) + return (name, unicode(res)) + + if key in field_metadata and field_metadata[key]['kind'] == 'field': + res = self.get(key, None) + if res is None or res == '': + return (None, None) + fmeta = field_metadata[key] + name = unicode(fmeta['name']) + datatype = fmeta['datatype'] + if key == 'authors': + res = authors_to_string(res) + elif datatype == 'text' and fmeta['is_multiple']: + res = u', '.join(res) + elif datatype == 'series': + if not ignore_series_index: + res = res + ' [%s]'%self.format_series_index() + elif datatype == 'datetime': + res = format_date(res, fmeta['display'].get('date_format','dd MMM yyyy')) + return (name, unicode(res)) + + return (None, None) def __unicode__(self): from calibre.ebooks.metadata import authors_to_string @@ -371,7 +424,7 @@ class Metadata(object): for key in self.user_metadata_keys: val = self.get(key, None) if val is not None: - (name, val) = self.format_custom_field(key) + (name, val) = self.format_field(key) fmt(name, unicode(val)) return u'\n'.join(ans) @@ -396,7 +449,7 @@ class Metadata(object): for key in self.user_metadata_keys: val = self.get(key, None) if val is not None: - (name, val) = self.format_custom_field(key) + (name, val) = self.format_field(key) ans += [(name, val)] for i, x in enumerate(ans): ans[i] = u'%s%s'%x diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index 09a28fb04e..5fa514ae8a 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -320,7 +320,7 @@ class BooksModel(QAbstractTableModel): # {{{ (sidx, prepare_string_for_xml(series)) mi = self.db.get_metadata(idx) for key in mi.user_metadata_keys: - name, val = mi.format_custom_field(key) + name, val = mi.format_field(key) if val is not None: data[name] = val return data diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py index 276a6ba971..2773f573b2 100644 --- a/src/calibre/library/field_metadata.py +++ b/src/calibre/library/field_metadata.py @@ -5,6 +5,7 @@ Created on 25 May 2010 ''' from calibre.utils.ordered_dict import OrderedDict +from calibre.utils.config import tweaks class TagsIcons(dict): ''' @@ -213,7 +214,7 @@ class FieldMetadata(dict): 'datatype':'text', 'is_multiple':None, 'kind':'field', - 'name':None, + 'name':_('On Device'), 'search_terms':['ondevice'], 'is_custom':False, 'is_category':False}), @@ -231,7 +232,7 @@ class FieldMetadata(dict): 'datatype':'datetime', 'is_multiple':None, 'kind':'field', - 'name':None, + 'name':_('Published'), 'search_terms':['pubdate'], 'is_custom':False, 'is_category':False}), @@ -258,7 +259,7 @@ class FieldMetadata(dict): 'datatype':'float', 'is_multiple':None, 'kind':'field', - 'name':None, + 'name':_('Size (MB)'), 'search_terms':['size'], 'is_custom':False, 'is_category':False}), @@ -267,7 +268,7 @@ class FieldMetadata(dict): 'datatype':'datetime', 'is_multiple':None, 'kind':'field', - 'name':None, + 'name':_('Date'), 'search_terms':['date'], 'is_custom':False, 'is_category':False}), @@ -276,7 +277,7 @@ class FieldMetadata(dict): 'datatype':'text', 'is_multiple':None, 'kind':'field', - 'name':None, + 'name':_('Title'), 'search_terms':['title'], 'is_custom':False, 'is_category':False}), @@ -310,6 +311,10 @@ class FieldMetadata(dict): self._tb_cats[k]['display'] = {} self._tb_cats[k]['is_editable'] = True self._add_search_terms_to_map(k, v['search_terms']) + self._tb_cats['timestamp']['display'] = { + 'date_format': tweaks['gui_timestamp_display_format']} + self._tb_cats['pubdate']['display'] = { + 'date_format': tweaks['gui_pubdate_display_format']} self.custom_field_prefix = '#' self.get = self._tb_cats.get @@ -410,7 +415,7 @@ class FieldMetadata(dict): if datatype == 'series': key += '_index' self._tb_cats[key] = {'table':None, 'column':None, - 'datatype':'float', 'is_multiple':False, + 'datatype':'float', 'is_multiple':None, 'kind':'field', 'name':'', 'search_terms':[key], 'label':label+'_index', 'colnum':None, 'display':{},