From ea29f4b683ada1c41593ff90664cfa146008be5f Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Thu, 23 Sep 2010 14:36:47 +0100 Subject: [PATCH] Changes: 1) complete rewrite of composite field processing -- creation of of formatter class in utils -- change template validator (prefs/save_template.py) to use new formatting class -- change save_to_disk to use new formatting class -- change Metadata class to use new formatting class -- Check for mutually recursive composite fields -- change caches.py to use the 'get' interface (now the right one) for composites 2) Add template validation to the base deviceconfig plugin. It checks if the display widget has a 'validate' method, and if so, it calls it. 3) Change models.py so that composite templates can be edited on the library display. -- back out the changes that set 'editable = False' 4) Fix problem in models.py where book info view was not being updated when a field is changed on library display 5) Changed save_to_disk to permit slashes in field specifications. Did this by splitting the template after template processing. This gives us basic variable folder structures Example: Simple example: we want the folder structure series/series_index - title. If series does not exist, then the title should be in the top folder. Template: {series:||/}{series_index:|| - }{title} 6) Change syntax for extended templates -- prefixes and suffixes have moved to the end of the field specification. Syntax: {series:|prefix value|suffix value} You can put a standard python format specification between the : and the first |. Either zero or two bars must be present. 7) Addition of some built-in functions to template processing. These appear in the format part. Syntax: {title:uppercase()|prefix value|suffix value} Functions apply to the value of the field in the format specification. The functions available are: -- uppercase(), lowercase(), titlecase(), capitalise() -- ifempty(text) If the field is empty, replace it with text. -- shorten(from start, center string, from end) Replace the field with a shortened version. The shortened version is found by joining the field's first 'from start' characters, the center string, and the field's last 'from end' characters. Example: assume that the title is 'Values of beta will give rise to dom'. The field specification {title:shorten(6,---,6)} will produce the result 'Values---to dom' -- lookup(key if field not empty, key if field empty) Replace the value of 'field' with the value of another field. The first field key (lookup name) is used if 'field' is not empty. The second field key is used if field is empty. This, coupled with composite fields and the change to save_to_disk above, facilitates complex variable folder trees on devices. Example: If a book has a series, then we want the folder structure series/series index - title.fmt. If the book does not have a series, then we want the folder structure genre/author_sort/title.fmt. If the book has no genre, use 'Unknown'. To accomplish this, we: 1) create a composite field named AA containing '{series:||}/{series_index} - {title'. 2) create a composite field named BB containing '{#genre:ifempty(Unknown)}/{author_sort}/{title} 3) set the save template to '{series:lookup(AA,BB)} --- src/calibre/ebooks/metadata/book/base.py | 50 ++++---- src/calibre/gui2/custom_column_widgets.py | 2 + .../gui2/device_drivers/configwidget.py | 15 +++ src/calibre/gui2/library/delegates.py | 28 ++++- src/calibre/gui2/library/models.py | 12 +- src/calibre/gui2/library/views.py | 6 +- src/calibre/gui2/preferences/columns.py | 3 +- .../gui2/preferences/create_custom_column.py | 4 - src/calibre/gui2/preferences/plugins.py | 6 +- src/calibre/gui2/preferences/save_template.py | 14 +-- src/calibre/library/caches.py | 2 +- src/calibre/library/save_to_disk.py | 29 +---- src/calibre/utils/formatter.py | 113 ++++++++++++++++++ 13 files changed, 210 insertions(+), 74 deletions(-) create mode 100644 src/calibre/utils/formatter.py diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py index a2b2790ed9..16819cbd39 100644 --- a/src/calibre/ebooks/metadata/book/base.py +++ b/src/calibre/ebooks/metadata/book/base.py @@ -5,7 +5,7 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import copy, re, string, traceback +import copy, re, traceback from calibre import prints from calibre.ebooks.metadata.book import SC_COPYABLE_FIELDS @@ -15,6 +15,7 @@ from calibre.ebooks.metadata.book import TOP_LEVEL_CLASSIFIERS from calibre.ebooks.metadata.book import ALL_METADATA_FIELDS from calibre.library.field_metadata import FieldMetadata from calibre.utils.date import isoformat, format_date +from calibre.utils.formatter import TemplateFormatter NULL_VALUES = { @@ -32,33 +33,19 @@ NULL_VALUES = { field_metadata = FieldMetadata() -class SafeFormat(string.Formatter): - ''' - Provides a format function that substitutes '' for any missing value - ''' +class SafeFormat(TemplateFormatter): def get_value(self, key, args, mi): - from calibre.library.save_to_disk import explode_string_template_value try: - prefix, key, suffix = explode_string_template_value(key) ign, v = mi.format_field(key, series_with_index=False) if v is None: return '' if v == '': return '' - return prefix + v + suffix + return v except: return key composite_formatter = SafeFormat() -compress_spaces = re.compile(r'\s+') - -def format_composite(x, mi): - try: - ans = composite_formatter.vformat(x, [], mi).strip() - except: - traceback.print_exc() - ans = x - return compress_spaces.sub(' ', ans) class Metadata(object): @@ -75,7 +62,9 @@ class Metadata(object): @param authors: List of strings or [] @param other: None or a metadata object ''' - object.__setattr__(self, '_data', copy.deepcopy(NULL_VALUES)) + _data = copy.deepcopy(NULL_VALUES) + object.__setattr__(self, '_data', _data) + _data['_curseq'] = _data['_compseq'] = 0 if other is not None: self.smart_update(other) else: @@ -98,14 +87,28 @@ class Metadata(object): pass if field in _data['user_metadata'].iterkeys(): d = _data['user_metadata'][field] - if d['datatype'] != 'composite': - return d['#value#'] - return format_composite(d['display']['composite_template'], self) + val = d['#value#'] + if d['datatype'] != 'composite' or \ + (_data['_curseq'] == _data['_compseq'] and val is not None): + return val + # Data in the structure has changed. Recompute the composite fields + _data['_compseq'] = _data['_curseq'] + for ck in _data['user_metadata']: + cf = _data['user_metadata'][ck] + if cf['datatype'] != 'composite': + continue + cf['#value#'] = 'RECURSIVE_COMPOSITE FIELD ' + field + cf['#value#'] = composite_formatter.safe_format( + d['display']['composite_template'], + self, _('TEMPLATE ERROR')).strip() + return d['#value#'] + raise AttributeError( 'Metadata object has no attribute named: '+ repr(field)) def __setattr__(self, field, val, extra=None): _data = object.__getattribute__(self, '_data') + _data['_curseq'] += 1 if field in TOP_LEVEL_CLASSIFIERS: _data['classifiers'].update({field: val}) elif field in STANDARD_METADATA_FIELDS: @@ -193,7 +196,7 @@ class Metadata(object): if v is not None: result[attr] = v for attr in _data['user_metadata'].iterkeys(): - v = _data['user_metadata'][attr]['#value#'] + v = self.get(attr, None) if v is not None: result[attr] = v if _data['user_metadata'][attr]['datatype'] == 'series': @@ -466,9 +469,6 @@ class Metadata(object): return (None, None, None, None) - def expand_template(self, template): - return format_composite(template, self) - def __unicode__(self): from calibre.ebooks.metadata import authors_to_string ans = [] diff --git a/src/calibre/gui2/custom_column_widgets.py b/src/calibre/gui2/custom_column_widgets.py index d16233be1a..90abfc2474 100644 --- a/src/calibre/gui2/custom_column_widgets.py +++ b/src/calibre/gui2/custom_column_widgets.py @@ -351,6 +351,8 @@ def populate_metadata_page(layout, db, book_id, bulk=False, two_column=False, pa if not x[col]['editable']: continue dt = x[col]['datatype'] + if dt == 'composite': + continue if dt == 'comments': continue w = widget_factory(dt, col) diff --git a/src/calibre/gui2/device_drivers/configwidget.py b/src/calibre/gui2/device_drivers/configwidget.py index 3d9c9ab2ee..1d6c84ef7c 100644 --- a/src/calibre/gui2/device_drivers/configwidget.py +++ b/src/calibre/gui2/device_drivers/configwidget.py @@ -6,7 +6,9 @@ __docformat__ = 'restructuredtext en' from PyQt4.Qt import QWidget, QListWidgetItem, Qt, QVariant, SIGNAL +from calibre.gui2 import error_dialog from calibre.gui2.device_drivers.configwidget_ui import Ui_ConfigWidget +from calibre.utils.formatter import validation_formatter class ConfigWidget(QWidget, Ui_ConfigWidget): @@ -77,3 +79,16 @@ class ConfigWidget(QWidget, Ui_ConfigWidget): def use_author_sort(self): return self.opt_use_author_sort.isChecked() + + def validate(self): + print 'here in validate' + tmpl = unicode(self.opt_save_template.text()) + try: + validation_formatter.validate(tmpl) + return True + except Exception, err: + error_dialog(self, _('Invalid template'), + '

'+_('The template %s is invalid:')%tmpl + \ + '
'+str(err), show=True) + + return False diff --git a/src/calibre/gui2/library/delegates.py b/src/calibre/gui2/library/delegates.py index bf233b1175..ceb1cf14a8 100644 --- a/src/calibre/gui2/library/delegates.py +++ b/src/calibre/gui2/library/delegates.py @@ -15,10 +15,11 @@ from PyQt4.Qt import QColor, Qt, QModelIndex, QSize, \ QStyledItemDelegate, QCompleter, \ QComboBox -from calibre.gui2 import UNDEFINED_QDATE +from calibre.gui2 import UNDEFINED_QDATE, error_dialog from calibre.gui2.widgets import EnLineEdit, TagsLineEdit from calibre.utils.date import now, format_date from calibre.utils.config import tweaks +from calibre.utils.formatter import validation_formatter from calibre.gui2.dialogs.comments_dialog import CommentsDialog class RatingDelegate(QStyledItemDelegate): # {{{ @@ -303,6 +304,31 @@ class CcBoolDelegate(QStyledItemDelegate): # {{{ val = 2 if val is None else 1 if not val else 0 editor.setCurrentIndex(val) +class CcTemplateDelegate(QStyledItemDelegate): # {{{ + def __init__(self, parent): + ''' + Delegate for custom_column bool data. + ''' + QStyledItemDelegate.__init__(self, parent) + + def createEditor(self, parent, option, index): + return EnLineEdit(parent) + + def setModelData(self, editor, model, index): + val = unicode(editor.text()) + try: + validation_formatter.validate(val) + except Exception, err: + error_dialog(self.parent(), _('Invalid template'), + '

'+_('The template %s is invalid:')%val + \ + '
'+str(err), show=True) + model.setData(index, QVariant(val), Qt.EditRole) + + def setEditorData(self, editor, index): + m = index.model() + val = m.custom_columns[m.column_map[index.column()]]['display']['composite_template'] + editor.setText(val) + # }}} diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index 4b1e974b12..fe64a33c47 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -696,7 +696,8 @@ class BooksModel(QAbstractTableModel): # {{{ return flags def set_custom_column_data(self, row, colhead, value): - typ = self.custom_columns[colhead]['datatype'] + cc = self.custom_columns[colhead] + typ = cc['datatype'] label=self.db.field_metadata.key_to_label(colhead) s_index = None if typ in ('text', 'comments'): @@ -722,6 +723,14 @@ class BooksModel(QAbstractTableModel): # {{{ val = qt_to_dt(val, as_utc=False) elif typ == 'series': val, s_index = parse_series_string(self.db, label, value.toString()) + elif typ == 'composite': + tmpl = unicode(value.toString()).strip() + disp = cc['display'] + disp['composite_template'] = tmpl + self.db.set_custom_column_metadata(cc['colnum'], display = disp) + self.refresh(reset=True) + return True + self.db.set_custom(self.db.id(row), val, extra=s_index, label=label, num=None, append=False, notify=True) return True @@ -768,6 +777,7 @@ class BooksModel(QAbstractTableModel): # {{{ self.db.set_pubdate(id, qt_to_dt(val, as_utc=False)) else: self.db.set(row, column, val) + self.refresh_rows([row], row) self.dataChanged.emit(index, index) return True diff --git a/src/calibre/gui2/library/views.py b/src/calibre/gui2/library/views.py index d3ead429cf..b113866ecc 100644 --- a/src/calibre/gui2/library/views.py +++ b/src/calibre/gui2/library/views.py @@ -13,7 +13,7 @@ from PyQt4.Qt import QTableView, Qt, QAbstractItemView, QMenu, pyqtSignal, \ from calibre.gui2.library.delegates import RatingDelegate, PubDateDelegate, \ TextDelegate, DateDelegate, TagsDelegate, CcTextDelegate, \ - CcBoolDelegate, CcCommentsDelegate, CcDateDelegate + CcBoolDelegate, CcCommentsDelegate, CcDateDelegate, CcTemplateDelegate from calibre.gui2.library.models import BooksModel, DeviceBooksModel from calibre.utils.config import tweaks, prefs from calibre.gui2 import error_dialog, gprefs @@ -47,6 +47,7 @@ class BooksView(QTableView): # {{{ self.cc_text_delegate = CcTextDelegate(self) self.cc_bool_delegate = CcBoolDelegate(self) self.cc_comments_delegate = CcCommentsDelegate(self) + self.cc_template_delegate = CcTemplateDelegate(self) self.display_parent = parent self._model = modelcls(self) self.setModel(self._model) @@ -392,8 +393,7 @@ class BooksView(QTableView): # {{{ elif cc['datatype'] == 'rating': self.setItemDelegateForColumn(cm.index(colhead), self.rating_delegate) elif cc['datatype'] == 'composite': - pass - # no delegate for composite columns, as they are not editable + self.setItemDelegateForColumn(cm.index(colhead), self.cc_template_delegate) else: dattr = colhead+'_delegate' delegate = colhead if hasattr(self, dattr) else 'text' diff --git a/src/calibre/gui2/preferences/columns.py b/src/calibre/gui2/preferences/columns.py index 761a9880b1..c1b9230f42 100644 --- a/src/calibre/gui2/preferences/columns.py +++ b/src/calibre/gui2/preferences/columns.py @@ -155,8 +155,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): name=self.custcols[c]['name'], datatype=self.custcols[c]['datatype'], is_multiple=self.custcols[c]['is_multiple'], - display = self.custcols[c]['display'], - editable = self.custcols[c]['editable']) + display = self.custcols[c]['display']) must_restart = True elif '*deleteme' in self.custcols[c]: db.delete_custom_column(label=self.custcols[c]['label']) diff --git a/src/calibre/gui2/preferences/create_custom_column.py b/src/calibre/gui2/preferences/create_custom_column.py index e88949a23c..bec21270df 100644 --- a/src/calibre/gui2/preferences/create_custom_column.py +++ b/src/calibre/gui2/preferences/create_custom_column.py @@ -156,9 +156,6 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn): return self.simple_error('', _('You must enter a template for' ' composite columns')) display_dict = {'composite_template':unicode(self.composite_box.text())} - is_editable = False - else: - is_editable = True db = self.parent.gui.library_view.model().db key = db.field_metadata.custom_field_prefix+col @@ -168,7 +165,6 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn): 'label':col, 'name':col_heading, 'datatype':col_type, - 'editable':is_editable, 'display':display_dict, 'normalized':None, 'colnum':None, diff --git a/src/calibre/gui2/preferences/plugins.py b/src/calibre/gui2/preferences/plugins.py index a26553db1c..388227e438 100644 --- a/src/calibre/gui2/preferences/plugins.py +++ b/src/calibre/gui2/preferences/plugins.py @@ -199,7 +199,11 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): config_dialog.exec_() if config_dialog.result() == QDialog.Accepted: - plugin.save_settings(config_widget) + if hasattr(config_widget, 'validate'): + if config_widget.validate(): + plugin.save_settings(config_widget) + else: + plugin.save_settings(config_widget) self._plugin_model.refresh_plugin(plugin) else: help_text = plugin.customization_help(gui=True) diff --git a/src/calibre/gui2/preferences/save_template.py b/src/calibre/gui2/preferences/save_template.py index 0f48893b69..5b3f0321b2 100644 --- a/src/calibre/gui2/preferences/save_template.py +++ b/src/calibre/gui2/preferences/save_template.py @@ -13,17 +13,8 @@ from PyQt4.Qt import QWidget, pyqtSignal from calibre.gui2 import error_dialog from calibre.gui2.preferences.save_template_ui import Ui_Form from calibre.library.save_to_disk import FORMAT_ARG_DESCS, preprocess_template +from calibre.utils.formatter import validation_formatter -class ValidateFormat(string.Formatter): - ''' - Provides a format function that substitutes '' for any missing value - ''' - def get_value(self, key, args, kwargs): - return 'this is some text that should be long enough' - -validate_formatter = ValidateFormat() -def validate_format(x, format_args): - return validate_formatter.vformat(x, [], format_args).strip() class SaveTemplate(QWidget, Ui_Form): @@ -62,9 +53,8 @@ class SaveTemplate(QWidget, Ui_Form): custom fields, because they may or may not exist. ''' tmpl = preprocess_template(self.opt_template.text()) - fa = {} try: - validate_format(tmpl, fa) + validation_formatter.validate(tmpl) except Exception, err: error_dialog(self, _('Invalid template'), '

'+_('The template %s is invalid:')%tmpl + \ diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 42feb6f8fa..7849eecb2e 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -546,7 +546,7 @@ class ResultCache(SearchQueryParser): if len(self.composites) > 0: mi = db.get_metadata(id, index_is_id=True) for k,c in self.composites: - self._data[id][c] = mi.format_field(k)[1] + self._data[id][c] = mi.get(k, None) except IndexError: return None try: diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py index 90e5413389..a0f739e4c2 100644 --- a/src/calibre/library/save_to_disk.py +++ b/src/calibre/library/save_to_disk.py @@ -9,6 +9,7 @@ __docformat__ = 'restructuredtext en' import os, traceback, cStringIO, re, string from calibre.utils.config import Config, StringConfig, tweaks +from calibre.utils.formatter import TemplateFormatter from calibre.utils.filenames import shorten_components_to, supports_long_names, \ ascii_filename, sanitize_file_name from calibre.ebooks.metadata.opf2 import metadata_to_opf @@ -101,40 +102,20 @@ def preprocess_template(template): template = template.decode(preferred_encoding, 'replace') return template -template_value_re = re.compile(r'^([^\|]*(?=\|))(?:\|?)([^\|]*)(?:\|?)((?<=\|).*?)$', - flags= re.UNICODE) - -def explode_string_template_value(key): - try: - matches = template_value_re.match(key) - if matches.lastindex != 3: - return key - return matches.groups() - except: - return '', key, '' - -class SafeFormat(string.Formatter): +class SafeFormat(TemplateFormatter): ''' Provides a format function that substitutes '' for any missing value ''' def get_value(self, key, args, kwargs): try: - prefix, key, suffix = explode_string_template_value(key) if kwargs[key]: - return prefix + unicode(kwargs[key]) + suffix + return kwargs[key] return '' except: return '' safe_formatter = SafeFormat() -def safe_format(x, format_args): - try: - ans = safe_formatter.vformat(x, [], format_args).strip() - except: - ans = '' - return re.sub(r'\s+', ' ', ans) - def get_components(template, mi, id, timefmt='%b %Y', length=250, sanitize_func=ascii_filename, replace_whitespace=False, to_lowercase=False): @@ -178,8 +159,8 @@ def get_components(template, mi, id, timefmt='%b %Y', length=250, elif custom_metadata[key]['datatype'] == 'bool': format_args[key] = _('yes') if format_args[key] else _('no') - components = [x.strip() for x in template.split('/') if x.strip()] - components = [safe_format(x, format_args) for x in components] + components = safe_formatter.safe_format(template, format_args, '') + components = [x.strip() for x in components.split('/') if x.strip()] components = [sanitize_func(x) for x in components if x] if not components: components = [str(id)] diff --git a/src/calibre/utils/formatter.py b/src/calibre/utils/formatter.py new file mode 100644 index 0000000000..f9ef4e0846 --- /dev/null +++ b/src/calibre/utils/formatter.py @@ -0,0 +1,113 @@ +''' +Created on 23 Sep 2010 + +@author: charles +''' + +import re, string + +def _lookup(val, mi, field_if_set, field_not_set): + if hasattr(mi, 'format_field'): + if val: + return mi.format_field(field_if_set.strip())[1] + else: + return mi.format_field(field_not_set.strip())[1] + else: + if val: + return mi.get(field_if_set.strip(), '') + else: + return mi.get(field_not_set.strip(), '') + +def _ifempty(val, mi, value_if_empty): + if val: + return val + else: + return value_if_empty + +def _shorten(val, mi, leading, center_string, trailing): + l = int(leading) + t = int(trailing) + if len(val) > l + len(center_string) + t: + return val[0:l] + center_string + val[-t:] + else: + return val + +class TemplateFormatter(string.Formatter): + ''' + Provides a format function that substitutes '' for any missing value + ''' + + functions = { + 'uppercase' : (0, lambda x: x.upper()), + 'lowercase' : (0, lambda x: x.lower()), + 'titlecase' : (0, lambda x: x.title()), + 'capitalize' : (0, lambda x: x.capitalize()), + 'ifempty' : (1, _ifempty), + 'lookup' : (2, _lookup), + 'shorten' : (3, _shorten), + } + + def get_value(self, key, args, mi): + raise Exception('get_value must be implemented in the subclass') + + format_string_re = re.compile(r'^(.*)\|(.*)\|(.*)$') + + def _explode_format_string(self, fmt): + try: + matches = self.format_string_re.match(fmt) + if matches is None or matches.lastindex != 3: + return fmt, '', '' + return matches.groups() + except: + import traceback + traceback.print_exc() + return fmt, '', '' + + def format_field(self, val, fmt): + fmt, prefix, suffix = self._explode_format_string(fmt) + + p = fmt.find('(') + if p >= 0 and fmt[-1] == ')' and fmt[0:p] in self.functions: + field = fmt[0:p] + func = self.functions[field] + args = fmt[p+1:-1].split(',') + if (func[0] == 0 and (len(args) != 1 or args[0])) or \ + (func[0] > 0 and func[0] != len(args)): + raise Exception ('Incorrect number of arguments for function '+ fmt[0:p]) + if func[0] == 0: + val = func[1](val, self.mi) + else: + val = func[1](val, self.mi, *args) + else: + val = string.Formatter.format_field(self, val, fmt) + if not val: + return '' + return prefix + val + suffix + + compress_spaces = re.compile(r'\s+') + + def vformat(self, fmt, args, kwargs): + self.mi = kwargs + ans = string.Formatter.vformat(self, fmt, args, kwargs) + return self.compress_spaces.sub(' ', ans).strip() + + def safe_format(self, fmt, kwargs, error_value): + try: + ans = self.vformat(fmt, [], kwargs).strip() + except: + ans = error_value + return ans + +class ValidateFormat(TemplateFormatter): + ''' + Provides a format function that substitutes '' for any missing value + ''' + def get_value(self, key, args, kwargs): + return 'this is some text that should be long enough' + + def validate(self, x): + return self.vformat(x, [], {}) + +validation_formatter = ValidateFormat() + +