Make templating language more capable

src/calibre/ebooks/metadata/book/base.py

@@ -5,7 +5,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import copy, re, string, traceback
+import copy, traceback
 
 from calibre import prints
 from calibre.ebooks.metadata.book import SC_COPYABLE_FIELDS
@@ -15,6 +15,7 @@ from calibre.ebooks.metadata.book import TOP_LEVEL_CLASSIFIERS
 from calibre.ebooks.metadata.book import ALL_METADATA_FIELDS
 from calibre.library.field_metadata import FieldMetadata
 from calibre.utils.date import isoformat, format_date
+from calibre.utils.formatter import TemplateFormatter
 
 
 NULL_VALUES = {
@@ -32,33 +33,19 @@ NULL_VALUES = {
 
 field_metadata = FieldMetadata()
 
-class SafeFormat(string.Formatter):
+class SafeFormat(TemplateFormatter):
-    '''
-    Provides a format function that substitutes '' for any missing value
-    '''
     def get_value(self, key, args, mi):
-        from calibre.library.save_to_disk import explode_string_template_value
         try:
-            prefix, key, suffix = explode_string_template_value(key)
+            ign, v = mi.format_field(key.lower(), series_with_index=False)
-            ign, v = mi.format_field(key, series_with_index=False)
             if v is None:
                 return ''
             if v == '':
                 return ''
-            return prefix + v + suffix
+            return v
         except:
             return key
 
 composite_formatter = SafeFormat()
-compress_spaces = re.compile(r'\s+')
-
-def format_composite(x, mi):
-    try:
-        ans = composite_formatter.vformat(x, [], mi).strip()
-    except:
-        traceback.print_exc()
-        ans = x
-    return compress_spaces.sub(' ', ans)
 
 class Metadata(object):
 
@@ -75,7 +62,9 @@ class Metadata(object):
         @param authors: List of strings or []
         @param other: None or a metadata object
         '''
-        object.__setattr__(self, '_data', copy.deepcopy(NULL_VALUES))
+        _data = copy.deepcopy(NULL_VALUES)
+        object.__setattr__(self, '_data', _data)
+        _data['_curseq'] = _data['_compseq'] = 0
         if other is not None:
             self.smart_update(other)
         else:
@@ -98,14 +87,28 @@ class Metadata(object):
             pass
         if field in _data['user_metadata'].iterkeys():
             d = _data['user_metadata'][field]
-            if d['datatype'] != 'composite':
+            val = d['#value#']
-                return d['#value#']
+            if d['datatype'] != 'composite' or \
-            return format_composite(d['display']['composite_template'], self)
+                    (_data['_curseq'] == _data['_compseq'] and val is not None):
+                return val
+            # Data in the structure has changed. Recompute the composite fields
+            _data['_compseq'] = _data['_curseq']
+            for ck in _data['user_metadata']:
+                cf = _data['user_metadata'][ck]
+                if cf['datatype'] != 'composite':
+                    continue
+                cf['#value#'] = 'RECURSIVE_COMPOSITE FIELD ' + field
+                cf['#value#'] = composite_formatter.safe_format(
+                                            cf['display']['composite_template'],
+                                            self, _('TEMPLATE ERROR')).strip()
+            return d['#value#']
+
         raise AttributeError(
                 'Metadata object has no attribute named: '+ repr(field))
 
     def __setattr__(self, field, val, extra=None):
         _data = object.__getattribute__(self, '_data')
+        _data['_curseq'] += 1
         if field in TOP_LEVEL_CLASSIFIERS:
             _data['classifiers'].update({field: val})
         elif field in STANDARD_METADATA_FIELDS:
@@ -193,7 +196,7 @@ class Metadata(object):
             if v is not None:
                 result[attr] = v
         for attr in _data['user_metadata'].iterkeys():
-            v = _data['user_metadata'][attr]['#value#']
+            v = self.get(attr, None)
             if v is not None:
                 result[attr] = v
                 if _data['user_metadata'][attr]['datatype'] == 'series':
@@ -466,9 +469,6 @@ class Metadata(object):
 
         return (None, None, None, None)
 
-    def expand_template(self, template):
-        return format_composite(template, self)
-
     def __unicode__(self):
         from calibre.ebooks.metadata import authors_to_string
         ans = []

src/calibre/gui2/custom_column_widgets.py

@@ -351,6 +351,8 @@ def populate_metadata_page(layout, db, book_id, bulk=False, two_column=False, pa
         if not x[col]['editable']:
             continue
         dt = x[col]['datatype']
+        if dt == 'composite':
+            continue
         if dt == 'comments':
             continue
         w = widget_factory(dt, col)

src/calibre/gui2/device_drivers/configwidget.py

@@ -6,7 +6,9 @@ __docformat__ = 'restructuredtext en'
 
 from PyQt4.Qt import QWidget, QListWidgetItem, Qt, QVariant, SIGNAL
 
+from calibre.gui2 import error_dialog
 from calibre.gui2.device_drivers.configwidget_ui import Ui_ConfigWidget
+from calibre.utils.formatter import validation_formatter
 
 class ConfigWidget(QWidget, Ui_ConfigWidget):
 
@@ -77,3 +79,15 @@ class ConfigWidget(QWidget, Ui_ConfigWidget):
 
     def use_author_sort(self):
         return self.opt_use_author_sort.isChecked()
+
+    def validate(self):
+        tmpl = unicode(self.opt_save_template.text())
+        try:
+            validation_formatter.validate(tmpl)
+            return True
+        except Exception, err:
+            error_dialog(self, _('Invalid template'),
+                    '<p>'+_('The template %s is invalid:')%tmpl + \
+                    '<br>'+unicode(err), show=True)
+
+            return False

src/calibre/gui2/library/delegates.py

@@ -15,10 +15,11 @@ from PyQt4.Qt import QColor, Qt, QModelIndex, QSize, \
                      QStyledItemDelegate, QCompleter, \
                      QComboBox
 
-from calibre.gui2 import UNDEFINED_QDATE
+from calibre.gui2 import UNDEFINED_QDATE, error_dialog
 from calibre.gui2.widgets import EnLineEdit, TagsLineEdit
 from calibre.utils.date import now, format_date
 from calibre.utils.config import tweaks
+from calibre.utils.formatter import validation_formatter
 from calibre.gui2.dialogs.comments_dialog import CommentsDialog
 
 class RatingDelegate(QStyledItemDelegate): # {{{
@@ -303,6 +304,33 @@ class CcBoolDelegate(QStyledItemDelegate): # {{{
             val = 2 if val is None else 1 if not val else 0
         editor.setCurrentIndex(val)
 
+# }}}
+
+class CcTemplateDelegate(QStyledItemDelegate): # {{{
+    def __init__(self, parent):
+        '''
+        Delegate for custom_column bool data.
+        '''
+        QStyledItemDelegate.__init__(self, parent)
+
+    def createEditor(self, parent, option, index):
+        return EnLineEdit(parent)
+
+    def setModelData(self, editor, model, index):
+        val = unicode(editor.text())
+        try:
+            validation_formatter.validate(val)
+        except Exception, err:
+            error_dialog(self.parent(), _('Invalid template'),
+                    '<p>'+_('The template %s is invalid:')%val + \
+                    '<br>'+str(err), show=True)
+        model.setData(index, QVariant(val), Qt.EditRole)
+
+    def setEditorData(self, editor, index):
+        m = index.model()
+        val = m.custom_columns[m.column_map[index.column()]]['display']['composite_template']
+        editor.setText(val)
+
 
 # }}}
 

src/calibre/gui2/library/models.py

@@ -696,7 +696,8 @@ class BooksModel(QAbstractTableModel): # {{{
         return flags
 
     def set_custom_column_data(self, row, colhead, value):
-        typ = self.custom_columns[colhead]['datatype']
+        cc = self.custom_columns[colhead]
+        typ = cc['datatype']
         label=self.db.field_metadata.key_to_label(colhead)
         s_index = None
         if typ in ('text', 'comments'):
@@ -722,6 +723,14 @@ class BooksModel(QAbstractTableModel): # {{{
                 val = qt_to_dt(val, as_utc=False)
         elif typ == 'series':
             val, s_index = parse_series_string(self.db, label, value.toString())
+        elif typ == 'composite':
+            tmpl = unicode(value.toString()).strip()
+            disp = cc['display']
+            disp['composite_template'] = tmpl
+            self.db.set_custom_column_metadata(cc['colnum'], display = disp)
+            self.refresh(reset=True)
+            return True
+
         self.db.set_custom(self.db.id(row), val, extra=s_index,
                            label=label, num=None, append=False, notify=True)
         return True
@@ -768,6 +777,7 @@ class BooksModel(QAbstractTableModel): # {{{
                     self.db.set_pubdate(id, qt_to_dt(val, as_utc=False))
                 else:
                     self.db.set(row, column, val)
+            self.refresh_rows([row], row)
             self.dataChanged.emit(index, index)
         return True
 

src/calibre/gui2/library/views.py

@@ -13,7 +13,7 @@ from PyQt4.Qt import QTableView, Qt, QAbstractItemView, QMenu, pyqtSignal, \
 
 from calibre.gui2.library.delegates import RatingDelegate, PubDateDelegate, \
     TextDelegate, DateDelegate, TagsDelegate, CcTextDelegate, \
-    CcBoolDelegate, CcCommentsDelegate, CcDateDelegate
+    CcBoolDelegate, CcCommentsDelegate, CcDateDelegate, CcTemplateDelegate
 from calibre.gui2.library.models import BooksModel, DeviceBooksModel
 from calibre.utils.config import tweaks, prefs
 from calibre.gui2 import error_dialog, gprefs
@@ -47,6 +47,7 @@ class BooksView(QTableView): # {{{
         self.cc_text_delegate = CcTextDelegate(self)
         self.cc_bool_delegate = CcBoolDelegate(self)
         self.cc_comments_delegate = CcCommentsDelegate(self)
+        self.cc_template_delegate = CcTemplateDelegate(self)
         self.display_parent = parent
         self._model = modelcls(self)
         self.setModel(self._model)
@@ -392,8 +393,7 @@ class BooksView(QTableView): # {{{
                 elif cc['datatype'] == 'rating':
                     self.setItemDelegateForColumn(cm.index(colhead), self.rating_delegate)
                 elif cc['datatype'] == 'composite':
-                    pass
+                    self.setItemDelegateForColumn(cm.index(colhead), self.cc_template_delegate)
-                    # no delegate for composite columns, as they are not editable
             else:
                 dattr = colhead+'_delegate'
                 delegate = colhead if hasattr(self, dattr) else 'text'

src/calibre/gui2/preferences/columns.py

@@ -155,8 +155,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
                                 name=self.custcols[c]['name'],
                                 datatype=self.custcols[c]['datatype'],
                                 is_multiple=self.custcols[c]['is_multiple'],
-                                display = self.custcols[c]['display'],
+                                display = self.custcols[c]['display'])
-                                editable = self.custcols[c]['editable'])
                 must_restart = True
             elif '*deleteme' in self.custcols[c]:
                 db.delete_custom_column(label=self.custcols[c]['label'])

src/calibre/gui2/preferences/create_custom_column.py

@@ -156,9 +156,6 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
                 return self.simple_error('', _('You must enter a template for'
                     ' composite columns'))
             display_dict = {'composite_template':unicode(self.composite_box.text())}
-            is_editable = False
-        else:
-            is_editable = True
 
         db = self.parent.gui.library_view.model().db
         key = db.field_metadata.custom_field_prefix+col
@@ -168,7 +165,6 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
                     'label':col,
                     'name':col_heading,
                     'datatype':col_type,
-                    'editable':is_editable,
                     'display':display_dict,
                     'normalized':None,
                     'colnum':None,

src/calibre/gui2/preferences/plugins.py

@@ -199,7 +199,11 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
                     config_dialog.exec_()
 
                     if config_dialog.result() == QDialog.Accepted:
-                        plugin.save_settings(config_widget)
+                        if hasattr(config_widget, 'validate'):
+                            if config_widget.validate():
+                                plugin.save_settings(config_widget)
+                        else:
+                            plugin.save_settings(config_widget)
                         self._plugin_model.refresh_plugin(plugin)
                 else:
                     help_text = plugin.customization_help(gui=True)

src/calibre/gui2/preferences/save_template.py

@@ -6,24 +6,13 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import string
-
 from PyQt4.Qt import QWidget, pyqtSignal
 
 from calibre.gui2 import error_dialog
 from calibre.gui2.preferences.save_template_ui import Ui_Form
 from calibre.library.save_to_disk import FORMAT_ARG_DESCS, preprocess_template
+from calibre.utils.formatter import validation_formatter
 
-class ValidateFormat(string.Formatter):
-    '''
-    Provides a format function that substitutes '' for any missing value
-    '''
-    def get_value(self, key, args, kwargs):
-            return 'this is some text that should be long enough'
-
-validate_formatter = ValidateFormat()
-def validate_format(x, format_args):
-    return validate_formatter.vformat(x, [], format_args).strip()
 
 class SaveTemplate(QWidget, Ui_Form):
 
@@ -62,9 +51,8 @@ class SaveTemplate(QWidget, Ui_Form):
         custom fields, because they may or may not exist.
         '''
         tmpl = preprocess_template(self.opt_template.text())
-        fa = {}
         try:
-            validate_format(tmpl, fa)
+            validation_formatter.validate(tmpl)
         except Exception, err:
             error_dialog(self, _('Invalid template'),
                     '<p>'+_('The template %s is invalid:')%tmpl + \

src/calibre/library/caches.py

@@ -546,7 +546,7 @@ class ResultCache(SearchQueryParser):
                 if len(self.composites) > 0:
                     mi = db.get_metadata(id, index_is_id=True)
                     for k,c in self.composites:
-                        self._data[id][c] = mi.format_field(k)[1]
+                        self._data[id][c] = mi.get(k, None)
             except IndexError:
                 return None
         try:

src/calibre/library/database2.py

@@ -734,7 +734,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         except:
             return None
         if not verify_formats:
-            return formats
+            return ','.join(formats)
         ans = []
         for format in formats:
             if self.format_abspath(id, format, index_is_id=True) is not None:

src/calibre/library/save_to_disk.py

@@ -6,9 +6,10 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os, traceback, cStringIO, re, string
+import os, traceback, cStringIO, re
 
 from calibre.utils.config import Config, StringConfig, tweaks
+from calibre.utils.formatter import TemplateFormatter
 from calibre.utils.filenames import shorten_components_to, supports_long_names, \
                                     ascii_filename, sanitize_file_name
 from calibre.ebooks.metadata.opf2 import metadata_to_opf
@@ -101,40 +102,20 @@ def preprocess_template(template):
         template = template.decode(preferred_encoding, 'replace')
     return template
 
-template_value_re = re.compile(r'^([^\|]*(?=\|))(?:\|?)([^\|]*)(?:\|?)((?<=\|).*?)$',
+class SafeFormat(TemplateFormatter):
-                               flags= re.UNICODE)
-
-def explode_string_template_value(key):
-    try:
-        matches = template_value_re.match(key)
-        if matches.lastindex != 3:
-            return key
-        return matches.groups()
-    except:
-        return '', key, ''
-
-class SafeFormat(string.Formatter):
     '''
     Provides a format function that substitutes '' for any missing value
     '''
     def get_value(self, key, args, kwargs):
         try:
-            prefix, key, suffix = explode_string_template_value(key)
+            if kwargs[key.lower()]:
-            if kwargs[key]:
+                return kwargs[key.lower()]
-                return prefix + unicode(kwargs[key]) + suffix
             return ''
         except:
             return ''
 
 safe_formatter = SafeFormat()
 
-def safe_format(x, format_args):
-    try:
-        ans = safe_formatter.vformat(x, [], format_args).strip()
-    except:
-        ans = ''
-    return re.sub(r'\s+', ' ', ans)
-
 def get_components(template, mi, id, timefmt='%b %Y', length=250,
         sanitize_func=ascii_filename, replace_whitespace=False,
         to_lowercase=False):
@@ -178,8 +159,8 @@ def get_components(template, mi, id, timefmt='%b %Y', length=250,
             elif custom_metadata[key]['datatype'] == 'bool':
                 format_args[key] = _('yes') if format_args[key] else _('no')
 
-    components = [x.strip() for x in template.split('/') if x.strip()]
+    components = safe_formatter.safe_format(template, format_args, '')
-    components = [safe_format(x, format_args) for x in components]
+    components = [x.strip() for x in components.split('/') if x.strip()]
     components = [sanitize_func(x) for x in components if x]
     if not components:
         components = [str(id)]

src/calibre/utils/formatter.py

@@ -0,0 +1,115 @@
+'''
+Created on 23 Sep 2010
+
+@author: charles
+'''
+
+import re, string
+
+def _lookup(val, mi, field_if_set, field_not_set):
+    if hasattr(mi, 'format_field'):
+        if val:
+            return mi.format_field(field_if_set.strip())[1]
+        else:
+            return mi.format_field(field_not_set.strip())[1]
+    else:
+        if val:
+            return mi.get(field_if_set.strip(), '')
+        else:
+            return mi.get(field_not_set.strip(), '')
+
+def _ifempty(val, mi, value_if_empty):
+    if val:
+        return val
+    else:
+        return value_if_empty
+
+def _shorten(val, mi, leading, center_string, trailing):
+    l = int(leading)
+    t = int(trailing)
+    if len(val) > l + len(center_string) + t:
+        return val[0:l] + center_string + val[-t:]
+    else:
+        return val
+
+class TemplateFormatter(string.Formatter):
+    '''
+    Provides a format function that substitutes '' for any missing value
+    '''
+
+    functions = {
+                    'uppercase'     : (0, lambda x: x.upper()),
+                    'lowercase'     : (0, lambda x: x.lower()),
+                    'titlecase'     : (0, lambda x: x.title()),
+                    'capitalize'    : (0, lambda x: x.capitalize()),
+                    'ifempty'       : (1, _ifempty),
+                    'lookup'        : (2, _lookup),
+                    'shorten'       : (3, _shorten),
+        }
+
+    format_string_re = re.compile(r'^(.*)\|(.*)\|(.*)$')
+    compress_spaces = re.compile(r'\s+')
+
+    def get_value(self, key, args, mi):
+        raise Exception('get_value must be implemented in the subclass')
+
+
+    def _explode_format_string(self, fmt):
+        try:
+            matches = self.format_string_re.match(fmt)
+            if matches is None or matches.lastindex != 3:
+                return fmt, '', ''
+            return matches.groups()
+        except:
+            import traceback
+            traceback.print_exc()
+            return fmt, '', ''
+
+    def format_field(self, val, fmt):
+        # Handle conditional text
+        fmt, prefix, suffix = self._explode_format_string(fmt)
+
+        # Handle functions
+        p = fmt.find('(')
+        if p >= 0 and fmt[-1] == ')' and fmt[0:p] in self.functions:
+            field = fmt[0:p]
+            func = self.functions[field]
+            args = fmt[p+1:-1].split(',')
+            if (func[0] == 0 and (len(args) != 1 or args[0])) or \
+                    (func[0] > 0 and func[0] != len(args)):
+                raise ValueError('Incorrect number of arguments for function '+ fmt[0:p])
+            if func[0] == 0:
+                val = func[1](val, self.mi)
+            else:
+                val = func[1](val, self.mi, *args)
+        else:
+            val = string.Formatter.format_field(self, val, fmt)
+        if not val:
+            return ''
+        return prefix + val + suffix
+
+    def vformat(self, fmt, args, kwargs):
+        self.mi = kwargs
+        ans = string.Formatter.vformat(self, fmt, args, kwargs)
+        return self.compress_spaces.sub(' ', ans).strip()
+
+    def safe_format(self, fmt, kwargs, error_value):
+        try:
+            ans = self.vformat(fmt, [], kwargs).strip()
+        except:
+            ans = error_value
+        return ans
+
+class ValidateFormat(TemplateFormatter):
+    '''
+    Provides a format function that substitutes '' for any missing value
+    '''
+    def get_value(self, key, args, kwargs):
+        return 'this is some text that should be long enough'
+
+    def validate(self, x):
+        return self.vformat(x, [], {})
+
+validation_formatter = ValidateFormat()
+
+