From 788128627459035cf0e87fc6b0baa4da708cef3d Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sat, 18 Sep 2010 11:08:33 +0100 Subject: [PATCH 1/7] 1) add the composite field custom datatype 2) clean up content server code so it uses the new formatting facilities --- src/calibre/devices/usbms/books.py | 7 ++- src/calibre/ebooks/metadata/book/__init__.py | 7 ++- src/calibre/ebooks/metadata/book/base.py | 35 +++++++++--- src/calibre/gui2/library/models.py | 33 ++++++++++-- src/calibre/gui2/preferences/columns.py | 3 +- .../gui2/preferences/create_custom_column.py | 30 ++++++++--- .../gui2/preferences/create_custom_column.ui | 53 ++++++++++++++++++- src/calibre/library/custom_columns.py | 6 +-- src/calibre/library/database2.py | 1 + src/calibre/library/field_metadata.py | 2 +- src/calibre/library/server/mobile.py | 36 +++++-------- src/calibre/library/server/opds.py | 24 ++++----- src/calibre/library/server/xml.py | 40 ++++++-------- 13 files changed, 181 insertions(+), 96 deletions(-) diff --git a/src/calibre/devices/usbms/books.py b/src/calibre/devices/usbms/books.py index eab625f7be..13fcb90b49 100644 --- a/src/calibre/devices/usbms/books.py +++ b/src/calibre/devices/usbms/books.py @@ -137,7 +137,6 @@ class CollectionsBookList(BookList): # For existing books, modify the collections only if the user # specified 'on_connect' attrs = collection_attributes - meta_vals = book.get_all_non_none_attributes() for attr in attrs: attr = attr.strip() ign, val, orig_val, fm = book.format_field_extended(attr) @@ -166,7 +165,7 @@ class CollectionsBookList(BookList): continue if attr == 'series' or \ ('series' in collection_attributes and - meta_vals.get('series', None) == category): + book.get('series', None) == category): is_series = True cat_name = self.compute_category_name(attr, category, fm) if cat_name not in collections: @@ -177,10 +176,10 @@ class CollectionsBookList(BookList): collections_lpaths[cat_name].add(lpath) if is_series: collections[cat_name].append( - (book, meta_vals.get(attr+'_index', sys.maxint))) + (book, book.get(attr+'_index', sys.maxint))) else: collections[cat_name].append( - (book, meta_vals.get('title_sort', 'zzzz'))) + (book, book.get('title_sort', 'zzzz'))) # Sort collections result = {} for category, books in collections.items(): diff --git a/src/calibre/ebooks/metadata/book/__init__.py b/src/calibre/ebooks/metadata/book/__init__.py index e087f8072d..e6dff9110b 100644 --- a/src/calibre/ebooks/metadata/book/__init__.py +++ b/src/calibre/ebooks/metadata/book/__init__.py @@ -81,9 +81,8 @@ DEVICE_METADATA_FIELDS = frozenset([ CALIBRE_METADATA_FIELDS = frozenset([ 'application_id', # An application id, currently set to the db_id. - # the calibre primary key of the item. 'db_id', # the calibre primary key of the item. - # TODO: NEWMETA: May want to remove once Sony's no longer use it + 'formats', # list of formats (extensions) for this book ] ) @@ -124,5 +123,5 @@ SERIALIZABLE_FIELDS = SOCIAL_METADATA_FIELDS.union( PUBLICATION_METADATA_FIELDS).union( CALIBRE_METADATA_FIELDS).union( DEVICE_METADATA_FIELDS) - \ - frozenset(['device_collections']) - # device_collections is rebuilt when needed + frozenset(['device_collections', 'formats']) + # these are rebuilt when needed diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py index b252f518da..31485dfe1b 100644 --- a/src/calibre/ebooks/metadata/book/base.py +++ b/src/calibre/ebooks/metadata/book/base.py @@ -5,8 +5,7 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import copy -import traceback +import copy, re, string, traceback from calibre import prints from calibre.ebooks.metadata.book import SC_COPYABLE_FIELDS @@ -33,6 +32,23 @@ NULL_VALUES = { field_metadata = FieldMetadata() +class SafeFormat(string.Formatter): + ''' + Provides a format function that substitutes '' for any missing value + ''' + def get_value(self, key, args, mi): + ign, v = mi.format_field(key, series_with_index=False) + if v is None: + return '' + return v + +composite_formatter = SafeFormat() +compress_spaces = re.compile(r'\s+') + +def format_composite(x, mi): + ans = composite_formatter.vformat(x, [], mi).strip() + return compress_spaces.sub(' ', ans) + class Metadata(object): ''' @@ -343,18 +359,19 @@ class Metadata(object): def format_rating(self): return unicode(self.rating) - def format_field(self, key): - name, val, ign, ign = self.format_field_extended(key) + def format_field(self, key, series_with_index=True): + name, val, ign, ign = self.format_field_extended(key, series_with_index) return (name, val) - def format_field_extended(self, key): + def format_field_extended(self, key, series_with_index=True): from calibre.ebooks.metadata import authors_to_string ''' returns the tuple (field_name, formatted_value) ''' if key in self.user_metadata_keys: res = self.get(key, None) - if res is None or res == '': + cmeta = self.get_user_metadata(key, make_copy=False) + if cmeta['datatype'] != 'composite' and (res is None or res == ''): return (None, None, None, None) orig_res = res cmeta = self.get_user_metadata(key, make_copy=False) @@ -362,13 +379,15 @@ class Metadata(object): datatype = cmeta['datatype'] if datatype == 'text' and cmeta['is_multiple']: res = u', '.join(res) - elif datatype == 'series': + elif datatype == 'series' and series_with_index: res = res + \ ' [%s]'%self.format_series_index(val=self.get_extra(key)) elif datatype == 'datetime': res = format_date(res, cmeta['display'].get('date_format','dd MMM yyyy')) elif datatype == 'bool': res = _('Yes') if res else _('No') + elif datatype == 'composite': + res = format_composite(cmeta['display']['composite_template'], self) return (name, res, orig_res, cmeta) if key in field_metadata and field_metadata[key]['kind'] == 'field': @@ -383,7 +402,7 @@ class Metadata(object): res = authors_to_string(res) elif datatype == 'text' and fmeta['is_multiple']: res = u', '.join(res) - elif datatype == 'series': + elif datatype == 'series' and series_with_index: res = res + ' [%s]'%self.format_series_index() elif datatype == 'datetime': res = format_date(res, fmeta['display'].get('date_format','dd MMM yyyy')) diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index e9e688c93b..7839b89d7e 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -86,6 +86,7 @@ class BooksModel(QAbstractTableModel): # {{{ self.last_search = '' # The last search performed on this model self.column_map = [] self.headers = {} + self.metadata_cache = {} self.alignment_map = {} self.buffer_size = buffer self.cover_cache = None @@ -114,6 +115,16 @@ class BooksModel(QAbstractTableModel): # {{{ def clear_caches(self): if self.cover_cache: self.cover_cache.clear_cache() + self.metadata_cache = {} + + def get_cached_metadata(self, idx): + if idx not in self.metadata_cache: + self.metadata_cache[idx] = self.db.get_metadata(idx) + return self.metadata_cache[idx] + + def remove_cached_metadata(self, idx): + if idx in self.metadata_cache: + del self.metadata_cache[idx] def read_config(self): self.use_roman_numbers = config['use_roman_numerals_for_series_number'] @@ -146,6 +157,7 @@ class BooksModel(QAbstractTableModel): # {{{ elif col in self.custom_columns: self.headers[col] = self.custom_columns[col]['name'] + self.metadata_cache = {} self.build_data_convertors() self.reset() self.database_changed.emit(db) @@ -159,11 +171,13 @@ class BooksModel(QAbstractTableModel): # {{{ db.add_listener(refresh_cover) def refresh_ids(self, ids, current_row=-1): + self.metadata_cache = {} rows = self.db.refresh_ids(ids) if rows: self.refresh_rows(rows, current_row=current_row) def refresh_rows(self, rows, current_row=-1): + self.metadata_cache = {} for row in rows: if row == current_row: self.new_bookdisplay_data.emit( @@ -193,6 +207,7 @@ class BooksModel(QAbstractTableModel): # {{{ return ret def count_changed(self, *args): + self.metadata_cache = {} self.count_changed_signal.emit(self.db.count()) def row_indices(self, index): @@ -262,6 +277,7 @@ class BooksModel(QAbstractTableModel): # {{{ self.sorting_done.emit(self.db.index) def refresh(self, reset=True): + self.metadata_cache = {} self.db.refresh(field=None) self.resort(reset=reset) @@ -318,7 +334,7 @@ class BooksModel(QAbstractTableModel): # {{{ data[_('Series')] = \ _('Book %s of %s.')%\ (sidx, prepare_string_for_xml(series)) - mi = self.db.get_metadata(idx) + mi = self.get_cached_metadata(idx) for key in mi.user_metadata_keys: name, val = mi.format_field(key) if val is not None: @@ -327,6 +343,7 @@ class BooksModel(QAbstractTableModel): # {{{ def set_cache(self, idx): l, r = 0, self.count()-1 + self.remove_cached_metadata(idx) if self.cover_cache is not None: l = max(l, idx-self.buffer_size) r = min(r, idx+self.buffer_size) @@ -586,6 +603,10 @@ class BooksModel(QAbstractTableModel): # {{{ def number_type(r, idx=-1): return QVariant(self.db.data[r][idx]) + def composite_type(r, key=None): + mi = self.get_cached_metadata(r) + return QVariant(mi.format_field(key)[1]) + self.dc = { 'title' : functools.partial(text_type, idx=self.db.field_metadata['title']['rec_index'], mult=False), @@ -620,7 +641,8 @@ class BooksModel(QAbstractTableModel): # {{{ idx = self.custom_columns[col]['rec_index'] datatype = self.custom_columns[col]['datatype'] if datatype in ('text', 'comments'): - self.dc[col] = functools.partial(text_type, idx=idx, mult=self.custom_columns[col]['is_multiple']) + self.dc[col] = functools.partial(text_type, idx=idx, + mult=self.custom_columns[col]['is_multiple']) elif datatype in ('int', 'float'): self.dc[col] = functools.partial(number_type, idx=idx) elif datatype == 'datetime': @@ -628,13 +650,15 @@ class BooksModel(QAbstractTableModel): # {{{ elif datatype == 'bool': self.dc[col] = functools.partial(bool_type, idx=idx) self.dc_decorator[col] = functools.partial( - bool_type_decorator, idx=idx, - bool_cols_are_tristate=tweaks['bool_custom_columns_are_tristate'] == 'yes') + bool_type_decorator, idx=idx, + bool_cols_are_tristate=tweaks['bool_custom_columns_are_tristate'] == 'yes') elif datatype == 'rating': self.dc[col] = functools.partial(rating_type, idx=idx) elif datatype == 'series': self.dc[col] = functools.partial(series_type, idx=idx, siix=self.db.field_metadata.cc_series_index_column_for(col)) + elif datatype == 'composite': + self.dc[col] = functools.partial(composite_type, key=col) else: print 'What type is this?', col, datatype # build a index column to data converter map, to remove the string lookup in the data loop @@ -729,6 +753,7 @@ class BooksModel(QAbstractTableModel): # {{{ if role == Qt.EditRole: row, col = index.row(), index.column() column = self.column_map[col] + self.remove_cached_metadata(row) if self.is_custom_column(column): if not self.set_custom_column_data(row, column, value): return False diff --git a/src/calibre/gui2/preferences/columns.py b/src/calibre/gui2/preferences/columns.py index c1b9230f42..761a9880b1 100644 --- a/src/calibre/gui2/preferences/columns.py +++ b/src/calibre/gui2/preferences/columns.py @@ -155,7 +155,8 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): name=self.custcols[c]['name'], datatype=self.custcols[c]['datatype'], is_multiple=self.custcols[c]['is_multiple'], - display = self.custcols[c]['display']) + display = self.custcols[c]['display'], + editable = self.custcols[c]['editable']) must_restart = True elif '*deleteme' in self.custcols[c]: db.delete_custom_column(label=self.custcols[c]['label']) diff --git a/src/calibre/gui2/preferences/create_custom_column.py b/src/calibre/gui2/preferences/create_custom_column.py index e8ab8707e2..4b21301ccd 100644 --- a/src/calibre/gui2/preferences/create_custom_column.py +++ b/src/calibre/gui2/preferences/create_custom_column.py @@ -38,6 +38,8 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn): 'is_multiple':False}, 8:{'datatype':'bool', 'text':_('Yes/No'), 'is_multiple':False}, + 8:{'datatype':'composite', + 'text':_('Field built from other fields'), 'is_multiple':False}, } def __init__(self, parent, editing, standard_colheads, standard_colnames): @@ -86,6 +88,8 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn): if ct == 'datetime': if c['display'].get('date_format', None): self.date_format_box.setText(c['display'].get('date_format', '')) + elif ct == 'composite': + self.composite_box.setText(c['display'].get('composite_template', '')) self.datatype_changed() self.exec_() @@ -94,9 +98,10 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn): col_type = self.column_types[self.column_type_box.currentIndex()]['datatype'] except: col_type = None - df_visible = col_type == 'datetime' for x in ('box', 'default_label', 'label'): - getattr(self, 'date_format_'+x).setVisible(df_visible) + getattr(self, 'date_format_'+x).setVisible(col_type == 'datetime') + for x in ('box', 'default_label', 'label'): + getattr(self, 'composite_'+x).setVisible(col_type == 'composite') def accept(self): @@ -122,6 +127,7 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn): bad_col = True if bad_col: return self.simple_error('', _('The lookup name %s is already used')%col) + bad_head = False for t in self.parent.custcols: if self.parent.custcols[t]['name'] == col_heading: @@ -133,12 +139,20 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn): if bad_head: return self.simple_error('', _('The heading %s is already used')%col_heading) - date_format = {} + display_dict = {} if col_type == 'datetime': if self.date_format_box.text(): - date_format = {'date_format':unicode(self.date_format_box.text())} + display_dict = {'date_format':unicode(self.date_format_box.text())} else: - date_format = {'date_format': None} + display_dict = {'date_format': None} + + if col_type == 'composite': + if not self.composite_box.text(): + return self.simple_error('', _('You must enter a template for composite fields')%col_heading) + display_dict = {'composite_template':unicode(self.composite_box.text())} + is_editable = False + else: + is_editable = True db = self.parent.gui.library_view.model().db key = db.field_metadata.custom_field_prefix+col @@ -148,8 +162,8 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn): 'label':col, 'name':col_heading, 'datatype':col_type, - 'editable':True, - 'display':date_format, + 'editable':is_editable, + 'display':display_dict, 'normalized':None, 'colnum':None, 'is_multiple':is_multiple, @@ -164,7 +178,7 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn): item.setText(col_heading) self.parent.custcols[self.orig_column_name]['label'] = col self.parent.custcols[self.orig_column_name]['name'] = col_heading - self.parent.custcols[self.orig_column_name]['display'].update(date_format) + self.parent.custcols[self.orig_column_name]['display'].update(display_dict) self.parent.custcols[self.orig_column_name]['*edited'] = True self.parent.custcols[self.orig_column_name]['*must_restart'] = True QDialog.accept(self) diff --git a/src/calibre/gui2/preferences/create_custom_column.ui b/src/calibre/gui2/preferences/create_custom_column.ui index 5cb9494845..640becca8c 100644 --- a/src/calibre/gui2/preferences/create_custom_column.ui +++ b/src/calibre/gui2/preferences/create_custom_column.ui @@ -147,9 +147,59 @@ + + + + + + + 0 + 0 + + + + <p>Field template. Uses the same syntax as save templates. + + + + + + + Similar to save templates. For example, {title} {isbn} + + + Default: (nothing) + + + + + + + + + &Template + + + composite_box + + + + + + + Qt::Vertical + + + + 20 + 40 + + + + - + Qt::Horizontal @@ -184,6 +234,7 @@ column_heading_box column_type_box date_format_box + composite_box button_box diff --git a/src/calibre/library/custom_columns.py b/src/calibre/library/custom_columns.py index 4ba664dadc..d74024280e 100644 --- a/src/calibre/library/custom_columns.py +++ b/src/calibre/library/custom_columns.py @@ -18,7 +18,7 @@ from calibre.utils.date import parse_date class CustomColumns(object): CUSTOM_DATA_TYPES = frozenset(['rating', 'text', 'comments', 'datetime', - 'int', 'float', 'bool', 'series']) + 'int', 'float', 'bool', 'series', 'composite']) def custom_table_names(self, num): return 'custom_column_%d'%num, 'books_custom_column_%d_link'%num @@ -540,7 +540,7 @@ class CustomColumns(object): if datatype not in self.CUSTOM_DATA_TYPES: raise ValueError('%r is not a supported data type'%datatype) normalized = datatype not in ('datetime', 'comments', 'int', 'bool', - 'float') + 'float', 'composite') is_multiple = is_multiple and datatype in ('text',) num = self.conn.execute( ('INSERT INTO ' @@ -551,7 +551,7 @@ class CustomColumns(object): if datatype in ('rating', 'int'): dt = 'INT' - elif datatype in ('text', 'comments', 'series'): + elif datatype in ('text', 'comments', 'series', 'composite'): dt = 'TEXT' elif datatype in ('float',): dt = 'REAL' diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 9e9e75a26e..d06d217b76 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -538,6 +538,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): mi.pubdate = self.pubdate(idx, index_is_id=index_is_id) mi.uuid = self.uuid(idx, index_is_id=index_is_id) mi.title_sort = self.title_sort(idx, index_is_id=index_is_id) + mi.formats = self.formats(idx, index_is_id=index_is_id).split(',') tags = self.tags(idx, index_is_id=index_is_id) if tags: mi.tags = [i.strip() for i in tags.split(',')] diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py index 2773f573b2..dcdfcfd9d6 100644 --- a/src/calibre/library/field_metadata.py +++ b/src/calibre/library/field_metadata.py @@ -68,7 +68,7 @@ class FieldMetadata(dict): ''' VALID_DATA_TYPES = frozenset([None, 'rating', 'text', 'comments', 'datetime', - 'int', 'float', 'bool', 'series']) + 'int', 'float', 'bool', 'series', 'composite']) # Builtin metadata {{{ diff --git a/src/calibre/library/server/mobile.py b/src/calibre/library/server/mobile.py index ab5b39eed8..8e7c75b0ac 100644 --- a/src/calibre/library/server/mobile.py +++ b/src/calibre/library/server/mobile.py @@ -228,29 +228,19 @@ class MobileServer(object): for key in CKEYS: def concat(name, val): return '%s:#:%s'%(name, unicode(val)) - val = record[CFM[key]['rec_index']] - if val: - datatype = CFM[key]['datatype'] - if datatype in ['comments']: - continue - name = CFM[key]['name'] - if datatype == 'text' and CFM[key]['is_multiple']: - book[key] = concat(name, - format_tag_string(val, '|', - no_tag_count=True)) - elif datatype == 'series': - book[key] = concat(name, '%s [%s]'%(val, - fmt_sidx(record[CFM.cc_series_index_column_for(key)]))) - elif datatype == 'datetime': - book[key] = concat(name, - format_date(val, CFM[key]['display'].get('date_format','dd MMM yyyy'))) - elif datatype == 'bool': - if val: - book[key] = concat(name, __builtin__._('Yes')) - else: - book[key] = concat(name, __builtin__._('No')) - else: - book[key] = concat(name, val) + mi = self.db.get_metadata(record[CFM['id']['rec_index']], index_is_id=True) + name, val = mi.format_field(key) + if val is None: + continue + datatype = CFM[key]['datatype'] + if datatype in ['comments']: + continue + if datatype == 'text' and CFM[key]['is_multiple']: + book[key] = concat(name, + format_tag_string(val, ',', + no_tag_count=True)) + else: + book[key] = concat(name, val) updated = self.db.last_modified() diff --git a/src/calibre/library/server/opds.py b/src/calibre/library/server/opds.py index e495598a2f..0eb7379ac5 100644 --- a/src/calibre/library/server/opds.py +++ b/src/calibre/library/server/opds.py @@ -132,7 +132,8 @@ def CATALOG_GROUP_ENTRY(item, category, base_href, version, updated): link ) -def ACQUISITION_ENTRY(item, version, FM, updated, CFM, CKEYS): +def ACQUISITION_ENTRY(item, version, db, updated, CFM, CKEYS): + FM = db.FIELD_MAP title = item[FM['title']] if not title: title = _('Unknown') @@ -157,22 +158,16 @@ def ACQUISITION_ENTRY(item, version, FM, updated, CFM, CKEYS): (series, fmt_sidx(float(item[FM['series_index']])))) for key in CKEYS: - val = item[CFM[key]['rec_index']] + mi = db.get_metadata(item[CFM['id']['rec_index']], index_is_id=True) + name, val = mi.format_field(key) if val is not None: - name = CFM[key]['name'] datatype = CFM[key]['datatype'] if datatype == 'text' and CFM[key]['is_multiple']: - extra.append('%s: %s
'%(name, format_tag_string(val, '|', + extra.append('%s: %s
'%(name, format_tag_string(val, ',', ignore_max=True, no_tag_count=True))) - elif datatype == 'series': - extra.append('%s: %s [%s]
'%(name, val, - fmt_sidx(item[CFM.cc_series_index_column_for(key)]))) - elif datatype == 'datetime': - extra.append('%s: %s
'%(name, - format_date(val, CFM[key]['display'].get('date_format','dd MMM yyyy')))) else: - extra.append('%s: %s
' % (CFM[key]['name'], val)) + extra.append('%s: %s
'%(name, val)) comments = item[FM['comments']] if comments: comments = comments_to_html(comments) @@ -280,13 +275,14 @@ class NavFeed(Feed): class AcquisitionFeed(NavFeed): def __init__(self, updated, id_, items, offsets, page_url, up_url, version, - FM, CFM): + db): NavFeed.__init__(self, id_, updated, version, offsets, page_url, up_url) + CFM = db.field_metadata CKEYS = [key for key in sorted(CFM.get_custom_fields(), cmp=lambda x,y: cmp(CFM[x]['name'].lower(), CFM[y]['name'].lower()))] for item in items: - self.root.append(ACQUISITION_ENTRY(item, version, FM, updated, + self.root.append(ACQUISITION_ENTRY(item, version, db, updated, CFM, CKEYS)) class CategoryFeed(NavFeed): @@ -384,7 +380,7 @@ class OPDSServer(object): cherrypy.response.headers['Last-Modified'] = self.last_modified(updated) cherrypy.response.headers['Content-Type'] = 'application/atom+xml;profile=opds-catalog' return str(AcquisitionFeed(updated, id_, items, offsets, - page_url, up_url, version, self.db.FIELD_MAP, self.db.field_metadata)) + page_url, up_url, version, self.db)) def opds_search(self, query=None, version=0, offset=0): try: diff --git a/src/calibre/library/server/xml.py b/src/calibre/library/server/xml.py index 8715dda7d0..7f5bc31e70 100644 --- a/src/calibre/library/server/xml.py +++ b/src/calibre/library/server/xml.py @@ -102,31 +102,21 @@ class XMLServer(object): for key in CKEYS: def concat(name, val): return '%s:#:%s'%(name, unicode(val)) - val = record[CFM[key]['rec_index']] - if val: - datatype = CFM[key]['datatype'] - if datatype in ['comments']: - continue - k = str('CF_'+key[1:]) - name = CFM[key]['name'] - custcols.append(k) - if datatype == 'text' and CFM[key]['is_multiple']: - kwargs[k] = concat('#T#'+name, - format_tag_string(val,'|', - ignore_max=True)) - elif datatype == 'series': - kwargs[k] = concat(name, '%s [%s]'%(val, - fmt_sidx(record[CFM.cc_series_index_column_for(key)]))) - elif datatype == 'datetime': - kwargs[k] = concat(name, - format_date(val, CFM[key]['display'].get('date_format','dd MMM yyyy'))) - elif datatype == 'bool': - if val: - kwargs[k] = concat(name, __builtin__._('Yes')) - else: - kwargs[k] = concat(name, __builtin__._('No')) - else: - kwargs[k] = concat(name, val) + mi = self.db.get_metadata(record[CFM['id']['rec_index']], index_is_id=True) + name, val = mi.format_field(key) + if not val: + continue + datatype = CFM[key]['datatype'] + if datatype in ['comments']: + continue + k = str('CF_'+key[1:]) + name = CFM[key]['name'] + custcols.append(k) + if datatype == 'text' and CFM[key]['is_multiple']: + kwargs[k] = concat('#T#'+name, format_tag_string(val,',', + ignore_max=True)) + else: + kwargs[k] = concat(name, val) kwargs['custcols'] = ','.join(custcols) books.append(E.book(c, **kwargs)) From 83fc5b2cc0452533cdcdc342d8ce21e3ab5501a4 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sat, 18 Sep 2010 11:38:51 +0100 Subject: [PATCH 2/7] Small cleanup of composite field code. --- src/calibre/ebooks/metadata/book/base.py | 12 ++++++++---- src/calibre/gui2/library/models.py | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py index 31485dfe1b..ce6e2ee78d 100644 --- a/src/calibre/ebooks/metadata/book/base.py +++ b/src/calibre/ebooks/metadata/book/base.py @@ -46,7 +46,10 @@ composite_formatter = SafeFormat() compress_spaces = re.compile(r'\s+') def format_composite(x, mi): - ans = composite_formatter.vformat(x, [], mi).strip() + try: + ans = composite_formatter.vformat(x, [], mi).strip() + except: + ans = x return compress_spaces.sub(' ', ans) class Metadata(object): @@ -86,7 +89,10 @@ class Metadata(object): except AttributeError: pass if field in _data['user_metadata'].iterkeys(): - return _data['user_metadata'][field]['#value#'] + d = _data['user_metadata'][field] + if d['datatype'] != 'composite': + return d['#value#'] + return format_composite(d['display']['composite_template'], self) raise AttributeError( 'Metadata object has no attribute named: '+ repr(field)) @@ -386,8 +392,6 @@ class Metadata(object): res = format_date(res, cmeta['display'].get('date_format','dd MMM yyyy')) elif datatype == 'bool': res = _('Yes') if res else _('No') - elif datatype == 'composite': - res = format_composite(cmeta['display']['composite_template'], self) return (name, res, orig_res, cmeta) if key in field_metadata and field_metadata[key]['kind'] == 'field': diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index 7839b89d7e..2a116f6f3d 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -605,7 +605,7 @@ class BooksModel(QAbstractTableModel): # {{{ def composite_type(r, key=None): mi = self.get_cached_metadata(r) - return QVariant(mi.format_field(key)[1]) + return QVariant(mi.get(key, '')) self.dc = { 'title' : functools.partial(text_type, From c59545a96a968488221f494fcee0baccab642a63 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sat, 18 Sep 2010 13:45:01 +0100 Subject: [PATCH 3/7] Change composites to use the cache correctly, so that searches & sorts used. In the process, remove the metadata cache from models.py. Fix some bugs introduced by composite columns: 1) no edit widget in bulk_metadata edit 2) explicitly do not make a delegate in views.py --- src/calibre/gui2/custom_column_widgets.py | 2 ++ src/calibre/gui2/library/models.py | 28 ++--------------------- src/calibre/gui2/library/views.py | 3 +++ src/calibre/library/caches.py | 20 +++++++++++++++- src/calibre/library/database2.py | 11 ++++----- 5 files changed, 31 insertions(+), 33 deletions(-) diff --git a/src/calibre/gui2/custom_column_widgets.py b/src/calibre/gui2/custom_column_widgets.py index 67ab94d29a..d16233be1a 100644 --- a/src/calibre/gui2/custom_column_widgets.py +++ b/src/calibre/gui2/custom_column_widgets.py @@ -348,6 +348,8 @@ def populate_metadata_page(layout, db, book_id, bulk=False, two_column=False, pa ans = [] column = row = comments_row = 0 for col in cols: + if not x[col]['editable']: + continue dt = x[col]['datatype'] if dt == 'comments': continue diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index 2a116f6f3d..be1bf9bc2d 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -86,7 +86,6 @@ class BooksModel(QAbstractTableModel): # {{{ self.last_search = '' # The last search performed on this model self.column_map = [] self.headers = {} - self.metadata_cache = {} self.alignment_map = {} self.buffer_size = buffer self.cover_cache = None @@ -115,16 +114,6 @@ class BooksModel(QAbstractTableModel): # {{{ def clear_caches(self): if self.cover_cache: self.cover_cache.clear_cache() - self.metadata_cache = {} - - def get_cached_metadata(self, idx): - if idx not in self.metadata_cache: - self.metadata_cache[idx] = self.db.get_metadata(idx) - return self.metadata_cache[idx] - - def remove_cached_metadata(self, idx): - if idx in self.metadata_cache: - del self.metadata_cache[idx] def read_config(self): self.use_roman_numbers = config['use_roman_numerals_for_series_number'] @@ -157,7 +146,6 @@ class BooksModel(QAbstractTableModel): # {{{ elif col in self.custom_columns: self.headers[col] = self.custom_columns[col]['name'] - self.metadata_cache = {} self.build_data_convertors() self.reset() self.database_changed.emit(db) @@ -171,13 +159,11 @@ class BooksModel(QAbstractTableModel): # {{{ db.add_listener(refresh_cover) def refresh_ids(self, ids, current_row=-1): - self.metadata_cache = {} rows = self.db.refresh_ids(ids) if rows: self.refresh_rows(rows, current_row=current_row) def refresh_rows(self, rows, current_row=-1): - self.metadata_cache = {} for row in rows: if row == current_row: self.new_bookdisplay_data.emit( @@ -207,7 +193,6 @@ class BooksModel(QAbstractTableModel): # {{{ return ret def count_changed(self, *args): - self.metadata_cache = {} self.count_changed_signal.emit(self.db.count()) def row_indices(self, index): @@ -277,7 +262,6 @@ class BooksModel(QAbstractTableModel): # {{{ self.sorting_done.emit(self.db.index) def refresh(self, reset=True): - self.metadata_cache = {} self.db.refresh(field=None) self.resort(reset=reset) @@ -334,7 +318,7 @@ class BooksModel(QAbstractTableModel): # {{{ data[_('Series')] = \ _('Book %s of %s.')%\ (sidx, prepare_string_for_xml(series)) - mi = self.get_cached_metadata(idx) + mi = self.db.get_metadata(idx) for key in mi.user_metadata_keys: name, val = mi.format_field(key) if val is not None: @@ -343,7 +327,6 @@ class BooksModel(QAbstractTableModel): # {{{ def set_cache(self, idx): l, r = 0, self.count()-1 - self.remove_cached_metadata(idx) if self.cover_cache is not None: l = max(l, idx-self.buffer_size) r = min(r, idx+self.buffer_size) @@ -603,10 +586,6 @@ class BooksModel(QAbstractTableModel): # {{{ def number_type(r, idx=-1): return QVariant(self.db.data[r][idx]) - def composite_type(r, key=None): - mi = self.get_cached_metadata(r) - return QVariant(mi.get(key, '')) - self.dc = { 'title' : functools.partial(text_type, idx=self.db.field_metadata['title']['rec_index'], mult=False), @@ -640,7 +619,7 @@ class BooksModel(QAbstractTableModel): # {{{ for col in self.custom_columns: idx = self.custom_columns[col]['rec_index'] datatype = self.custom_columns[col]['datatype'] - if datatype in ('text', 'comments'): + if datatype in ('text', 'comments', 'composite'): self.dc[col] = functools.partial(text_type, idx=idx, mult=self.custom_columns[col]['is_multiple']) elif datatype in ('int', 'float'): @@ -657,8 +636,6 @@ class BooksModel(QAbstractTableModel): # {{{ elif datatype == 'series': self.dc[col] = functools.partial(series_type, idx=idx, siix=self.db.field_metadata.cc_series_index_column_for(col)) - elif datatype == 'composite': - self.dc[col] = functools.partial(composite_type, key=col) else: print 'What type is this?', col, datatype # build a index column to data converter map, to remove the string lookup in the data loop @@ -753,7 +730,6 @@ class BooksModel(QAbstractTableModel): # {{{ if role == Qt.EditRole: row, col = index.row(), index.column() column = self.column_map[col] - self.remove_cached_metadata(row) if self.is_custom_column(column): if not self.set_custom_column_data(row, column, value): return False diff --git a/src/calibre/gui2/library/views.py b/src/calibre/gui2/library/views.py index d67d286aeb..9951edf21b 100644 --- a/src/calibre/gui2/library/views.py +++ b/src/calibre/gui2/library/views.py @@ -391,6 +391,9 @@ class BooksView(QTableView): # {{{ self.setItemDelegateForColumn(cm.index(colhead), self.cc_bool_delegate) elif cc['datatype'] == 'rating': self.setItemDelegateForColumn(cm.index(colhead), self.rating_delegate) + elif cc['datatype'] == 'composite': + pass + # no delegate for composite columns, as they are not editable else: dattr = colhead+'_delegate' delegate = colhead if hasattr(self, dattr) else 'text' diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 4f795ab733..a013d23cb9 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -121,6 +121,11 @@ class ResultCache(SearchQueryParser): self.build_date_relop_dict() self.build_numeric_relop_dict() + self.composites = [] + for key in field_metadata: + if field_metadata[key]['datatype'] == 'composite': + self.composites.append((key, field_metadata[key]['rec_index'])) + def __getitem__(self, row): return self._data[self._map_filtered[row]] @@ -372,7 +377,7 @@ class ResultCache(SearchQueryParser): if len(self.field_metadata[x]['search_terms']): db_col[x] = self.field_metadata[x]['rec_index'] if self.field_metadata[x]['datatype'] not in \ - ['text', 'comments', 'series']: + ['composite', 'text', 'comments', 'series']: exclude_fields.append(db_col[x]) col_datatype[db_col[x]] = self.field_metadata[x]['datatype'] is_multiple_cols[db_col[x]] = self.field_metadata[x]['is_multiple'] @@ -534,6 +539,10 @@ class ResultCache(SearchQueryParser): self._data[id] = db.conn.get('SELECT * from meta2 WHERE id=?', (id,))[0] self._data[id].append(db.has_cover(id, index_is_id=True)) self._data[id].append(db.book_on_device_string(id)) + if len(self.composites) > 0: + mi = db.get_metadata(id, index_is_id=True) + for k,c in self.composites: + self._data[id][c] = mi.format_field(k)[1] except IndexError: return None try: @@ -550,6 +559,10 @@ class ResultCache(SearchQueryParser): self._data[id] = db.conn.get('SELECT * from meta2 WHERE id=?', (id,))[0] self._data[id].append(db.has_cover(id, index_is_id=True)) self._data[id].append(db.book_on_device_string(id)) + if len(self.composites) > 0: + mi = db.get_metadata(id, index_is_id=True) + for k,c in self.composites: + self._data[id][c] = mi.format_field(k)[1] self._map[0:0] = ids self._map_filtered[0:0] = ids @@ -575,6 +588,11 @@ class ResultCache(SearchQueryParser): if item is not None: item.append(db.has_cover(item[0], index_is_id=True)) item.append(db.book_on_device_string(item[0])) + if len(self.composites) > 0: + mi = db.get_metadata(item[0], index_is_id=True) + for k,c in self.composites: + item[c] = mi.format_field(k)[1] + self._map = [i[0] for i in self._data if i is not None] if field is not None: self.sort(field, ascending) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index d06d217b76..d51a8a62c0 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -323,12 +323,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.has_id = self.data.has_id self.count = self.data.count - self.refresh_ondevice = functools.partial(self.data.refresh_ondevice, self) - - self.refresh() - self.last_update_check = self.last_modified() - - for prop in ('author_sort', 'authors', 'comment', 'comments', 'isbn', 'publisher', 'rating', 'series', 'series_index', 'tags', 'title', 'timestamp', 'uuid', 'pubdate', 'ondevice'): @@ -337,6 +331,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): setattr(self, 'title_sort', functools.partial(self.get_property, loc=self.FIELD_MAP['sort'])) + self.refresh_ondevice = functools.partial(self.data.refresh_ondevice, self) + self.refresh() + self.last_update_check = self.last_modified() + + def initialize_database(self): metadata_sqlite = open(P('metadata_sqlite.sql'), 'rb').read() self.conn.executescript(metadata_sqlite) From ed7597ae5f142998c3444f1ad941725fa4d21b0d Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sat, 18 Sep 2010 19:40:44 +0100 Subject: [PATCH 4/7] Playing with search & replace. Added 'global' template values to the replace expression. Also fixed some problems with exceptions, and problems with case-insensitive matching in the history boxes. --- src/calibre/ebooks/metadata/book/base.py | 9 +++ src/calibre/gui2/dialogs/metadata_bulk.py | 68 +++++++++++++++++++---- 2 files changed, 66 insertions(+), 11 deletions(-) diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py index ce6e2ee78d..1eae2e5326 100644 --- a/src/calibre/ebooks/metadata/book/base.py +++ b/src/calibre/ebooks/metadata/book/base.py @@ -12,6 +12,7 @@ from calibre.ebooks.metadata.book import SC_COPYABLE_FIELDS from calibre.ebooks.metadata.book import SC_FIELDS_COPY_NOT_NULL from calibre.ebooks.metadata.book import STANDARD_METADATA_FIELDS from calibre.ebooks.metadata.book import TOP_LEVEL_CLASSIFIERS +from calibre.ebooks.metadata.book import ALL_METADATA_FIELDS from calibre.library.field_metadata import FieldMetadata from calibre.utils.date import isoformat, format_date @@ -131,6 +132,14 @@ class Metadata(object): def set(self, field, val, extra=None): self.__setattr__(field, val, extra) + @property + def all_keys(self): + ''' + All attribute keys known by this instance, even if their value is None + ''' + _data = object.__getattribute__(self, '_data') + return frozenset(ALL_METADATA_FIELDS.union(_data['user_metadata'].iterkeys())) + @property def user_metadata_keys(self): 'The set of user metadata names this object knows about' diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index b7d1d0c54b..1fb889757f 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -4,15 +4,15 @@ __copyright__ = '2008, Kovid Goyal ' '''Dialog to edit metadata in bulk''' from threading import Thread -import re +import re, string -from PyQt4.Qt import QDialog, QGridLayout +from PyQt4.Qt import Qt, QDialog, QGridLayout from PyQt4 import QtGui from calibre.gui2.dialogs.metadata_bulk_ui import Ui_MetadataBulkDialog from calibre.gui2.dialogs.tag_editor import TagEditor from calibre.ebooks.metadata import string_to_authors, \ - authors_to_string + authors_to_string, MetaInformation from calibre.gui2.custom_column_widgets import populate_metadata_page from calibre.gui2.dialogs.progress import BlockingBusy from calibre.gui2 import error_dialog, Dispatcher @@ -99,6 +99,26 @@ class Worker(Thread): self.callback() +class SafeFormat(string.Formatter): + ''' + Provides a format function that substitutes '' for any missing value + ''' + def get_value(self, key, args, vals): + v = vals.get(key, None) + if v is None: + return '' + if isinstance(v, (tuple, list)): + v = ','.join(v) + return v + +composite_formatter = SafeFormat() + +def format_composite(x, mi): + try: + ans = composite_formatter.vformat(x, [], mi).strip() + except: + ans = x + return ans class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog): @@ -163,7 +183,7 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog): self.s_r_number_of_books = min(7, len(self.ids)) for i in range(1,self.s_r_number_of_books+1): w = QtGui.QLabel(self.tabWidgetPage3) - w.setText(_('Book %d:'%i)) + w.setText(_('Book %d:')%i) self.gridLayout1.addWidget(w, i+offset, 0, 1, 1) w = QtGui.QLineEdit(self.tabWidgetPage3) w.setReadOnly(True) @@ -205,6 +225,10 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog): self.test_text.editTextChanged[str].connect(self.s_r_paint_results) self.central_widget.setCurrentIndex(0) + self.search_for.completer().setCaseSensitivity(Qt.CaseSensitive) + self.replace_with.completer().setCaseSensitivity(Qt.CaseSensitive) + + def s_r_field_changed(self, txt): txt = unicode(txt) for i in range(0, self.s_r_number_of_books): @@ -220,6 +244,8 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog): if val: val.sort(cmp=lambda x,y: cmp(x.lower(), y.lower())) val = val[0] + if txt == 'authors': + val = val.replace('|', ',') else: val = '' else: @@ -239,37 +265,55 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog): for i in range(0,self.s_r_number_of_books): getattr(self, 'book_%d_result'%(i+1)).setText('') + field_match_re = re.compile(r'(^|[^\\])(\\g<)([^>]+)(>)') + def s_r_func(self, match): - rf = self.s_r_functions[unicode(self.replace_func.currentText())] - rv = unicode(self.replace_with.text()) - val = match.expand(rv) - return rf(val) + rfunc = self.s_r_functions[unicode(self.replace_func.currentText())] + rtext = unicode(self.replace_with.text()) + mi_data = self.mi.get_all_non_none_attributes() + + def fm_func(m): + try: + if m.group(3) not in self.mi.all_keys: return m.group(0) + else: return '%s{%s}'%(m.group(1), m.group(3)) + except: + import traceback + traceback.print_exc() + return m.group(0) + + rtext = re.sub(self.field_match_re, fm_func, rtext) + rtext = match.expand(rtext) + rtext = format_composite(rtext, mi_data) + return rfunc(rtext) def s_r_paint_results(self, txt): self.s_r_error = None self.s_r_set_colors() try: self.s_r_obj = re.compile(unicode(self.search_for.text())) - except re.error as e: + except Exception as e: self.s_r_obj = None self.s_r_error = e self.s_r_set_colors() return try: + self.mi = MetaInformation(None, None) self.test_result.setText(self.s_r_obj.sub(self.s_r_func, unicode(self.test_text.text()))) - except re.error as e: + except Exception as e: self.s_r_error = e self.s_r_set_colors() return for i in range(0,self.s_r_number_of_books): + id = self.ids[i] + self.mi = self.db.get_metadata(id, index_is_id=True) wt = getattr(self, 'book_%d_text'%(i+1)) wr = getattr(self, 'book_%d_result'%(i+1)) try: wr.setText(self.s_r_obj.sub(self.s_r_func, unicode(wt.text()))) - except re.error as e: + except Exception as e: self.s_r_error = e self.s_r_set_colors() break @@ -303,6 +347,8 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog): # The standard tags and authors values want to be lists. # All custom columns are to be strings val = fm['is_multiple'].join(val) + elif field == 'authors': + val = [v.replace('|', ',') for v in val] else: val = apply_pattern(val) From 7eaf417bb10e9d87038b47941c524ea9aa121ad2 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sun, 19 Sep 2010 07:47:03 +0100 Subject: [PATCH 5/7] Fix content server tags display problem --- resources/content_server/gui.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/resources/content_server/gui.js b/resources/content_server/gui.js index afc21137e1..bd0743a854 100644 --- a/resources/content_server/gui.js +++ b/resources/content_server/gui.js @@ -84,7 +84,10 @@ function render_book(book) { } title += '' title += '