1) change plugboards to templates (pass one)

2) fix recursion detection in base.py
3) fix lack of refresh in model when editing custom fields on the GUI
4) change the name of the plugboard eval function in base.py
5) move recursion detection base code to formatter

src/calibre/ebooks/metadata/book/base.py

@@ -37,7 +37,13 @@ class SafeFormat(TemplateFormatter):
 
     def get_value(self, key, args, kwargs):
         try:
-            ign, v = self.book.format_field(key.lower(), series_with_index=False)
+            b = self.book.get_user_metadata(key, False)
+            if b and b['datatype'] == 'int' and self.book.get(key, 0) == 0:
+                v = ''
+            elif b and b['datatype'] == 'float' and b.get(key, 0.0) == 0.0:
+                v = ''
+            else:
+                ign, v = self.book.format_field(key.lower(), series_with_index=False)
             if v is None:
                 return ''
             if v == '':
@@ -65,7 +71,6 @@ class Metadata(object):
         '''
         _data = copy.deepcopy(NULL_VALUES)
         object.__setattr__(self, '_data', _data)
-        _data['_curseq'] = _data['_compseq'] = 0
         if other is not None:
             self.smart_update(other)
         else:
@@ -94,29 +99,22 @@ class Metadata(object):
         if field in _data['user_metadata'].iterkeys():
             d = _data['user_metadata'][field]
             val = d['#value#']
-            if d['datatype'] != 'composite' or \
+            if d['datatype'] != 'composite':
-                    (_data['_curseq'] == _data['_compseq'] and val is not None):
                 return val
-            # Data in the structure has changed. Recompute the composite fields
+            if val is None:
-            _data['_compseq'] = _data['_curseq']
+                d['#value#'] = 'RECURSIVE_COMPOSITE FIELD (Metadata) ' + field
-            for ck in _data['user_metadata']:
+                val = d['#value#'] = composite_formatter.safe_format(
-                cf = _data['user_metadata'][ck]
+                                            d['display']['composite_template'],
-                if cf['datatype'] != 'composite':
-                    continue
-                cf['#value#'] = 'RECURSIVE_COMPOSITE FIELD ' + field
-                cf['#value#'] = composite_formatter.safe_format(
-                                            cf['display']['composite_template'],
                                             self,
                                             _('TEMPLATE ERROR'),
                                             self).strip()
-            return d['#value#']
+            return val
 
         raise AttributeError(
                 'Metadata object has no attribute named: '+ repr(field))
 
     def __setattr__(self, field, val, extra=None):
         _data = object.__getattribute__(self, '_data')
-        _data['_curseq'] += 1
         if field in TOP_LEVEL_CLASSIFIERS:
             _data['classifiers'].update({field: val})
         elif field in STANDARD_METADATA_FIELDS:
@@ -124,7 +122,10 @@ class Metadata(object):
                 val = NULL_VALUES.get(field, None)
             _data[field] = val
         elif field in _data['user_metadata'].iterkeys():
-            _data['user_metadata'][field]['#value#'] = val
+            if _data['user_metadata'][field]['datatype'] == 'composite':
+                _data['user_metadata'][field]['#value#'] = None
+            else:
+                _data['user_metadata'][field]['#value#'] = val
             _data['user_metadata'][field]['#extra#'] = extra
         else:
             # You are allowed to stick arbitrary attributes onto this object as
@@ -294,28 +295,24 @@ class Metadata(object):
             _data = object.__getattribute__(self, '_data')
             _data['user_metadata'][field] = metadata
 
-    def copy_specific_attributes(self, other, attrs):
+    def template_to_attribute(self, other, attrs):
         '''
-        Takes a dict {src:dest, src:dest} and copys other[src] to self[dest].
+        Takes a dict {src:dest, src:dest}, evaluates the template in the context
-        This is on a best-efforts basis. Some assignments can make no sense.
+        of other, then copies the result to self[dest]. This is on a best-
+        efforts basis. Some assignments can make no sense.
         '''
         if not attrs:
             return
         for src in attrs:
             try:
-                sfm = other.metadata_for_field(src)
+                val = composite_formatter.safe_format\
+                    (src, other, 'PLUGBOARD TEMPLATE ERROR', other)
                 dfm = self.metadata_for_field(attrs[src])
                 if dfm['is_multiple']:
-                    if sfm['is_multiple']:
+                    self.set(attrs[src],
-                        self.set(attrs[src], other.get(src))
+                        [f.strip() for f in val.split(',') if f.strip()])
-                    else:
-                        self.set(attrs[src],
-                            [f.strip() for f in other.get(src).split(',')
-                                            if f.strip()])
-                elif sfm['is_multiple']:
-                    self.set(attrs[src], ','.join(other.get(src)))
                 else:
-                    self.set(attrs[src], other.get(src))
+                    self.set(attrs[src], val)
             except:
                 traceback.print_exc()
                 pass

src/calibre/gui2/device.py

@@ -349,7 +349,7 @@ class DeviceManager(Thread): # {{{
                             with open(f, 'r+b') as stream:
                                 if cpb:
                                     newmi = mi.deepcopy()
-                                    newmi.copy_specific_attributes(mi, cpb)
+                                    newmi.template_to_attribute(mi, cpb)
                                 else:
                                     newmi = mi
                                 set_metadata(stream, newmi, stream_type=ext)

src/calibre/gui2/library/models.py

@@ -750,8 +750,10 @@ class BooksModel(QAbstractTableModel): # {{{
             self.refresh(reset=True)
             return True
 
-        self.db.set_custom(self.db.id(row), val, extra=s_index,
+        id = self.db.id(row)
+        self.db.set_custom(id, val, extra=s_index,
                            label=label, num=None, append=False, notify=True)
+        self.refresh_ids([id], current_row=row)
         return True
 
     def setData(self, index, value, role):

src/calibre/gui2/preferences/plugboard.py

@@ -56,18 +56,19 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         self.formats.insert(1, plugboard_any_format_value)
         self.new_format.addItems(self.formats)
 
-        self.fields = ['']
+        self.source_fields = ['']
-        for f in self.db.all_field_keys():
+        for f in self.db.custom_field_keys():
-            if self.db.field_metadata[f].get('rec_index', None) is not None and\
+            if self.db.field_metadata[f]['datatype'] == 'composite':
-                    self.db.field_metadata[f]['datatype'] is not None and \
+                self.source_fields.append(f)
-                    self.db.field_metadata[f]['search_terms']:
+        self.source_fields.sort(cmp=field_cmp)
-                self.fields.append(f)
+
-        self.fields.sort(cmp=field_cmp)
+        self.dest_fields = ['', 'authors', 'author_sort', 'publisher',
+                            'tags', 'title']
 
         self.source_widgets = []
         self.dest_widgets = []
         for i in range(0, 10):
-            w = QtGui.QComboBox(self)
+            w = QtGui.QLineEdit(self)
             self.source_widgets.append(w)
             self.fields_layout.addWidget(w, 5+i, 0, 1, 1)
             w = QtGui.QComboBox(self)
@@ -101,14 +102,13 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         self.ok_button.setEnabled(True)
         self.del_button.setEnabled(True)
         for w in self.source_widgets:
-            w.addItems(self.fields)
+            w.clear()
         for w in self.dest_widgets:
-            w.addItems(self.fields)
+            w.addItems(self.dest_fields)
 
     def set_field(self, i, src, dst):
-        idx = self.fields.index(src)
+        self.source_widgets[i].setText(src)
-        self.source_widgets[i].setCurrentIndex(idx)
+        idx = self.dest_fields.index(dst)
-        idx = self.fields.index(dst)
         self.dest_widgets[i].setCurrentIndex(idx)
 
     def edit_device_changed(self, txt):
@@ -216,11 +216,11 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
     def ok_clicked(self):
         pb = {}
         for i in range(0, len(self.source_widgets)):
-            s = self.source_widgets[i].currentIndex()
+            s = unicode(self.source_widgets[i].text())
-            if s != 0:
+            if s:
                 d = self.dest_widgets[i].currentIndex()
                 if d != 0:
-                    pb[self.fields[s]] = self.fields[d]
+                    pb[s] = self.dest_fields[d]
         if len(pb) == 0:
             if self.current_format in self.current_plugboards:
                 fpb = self.current_plugboards[self.current_format]
@@ -266,9 +266,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
                 if d not in self.current_plugboards[f]:
                     continue
                 ops = []
-                for op in self.fields:
+                for op in self.current_plugboards[f][d]:
-                    if op not in self.current_plugboards[f][d]:
-                        continue
                     ops.append(op + '->' + self.current_plugboards[f][d][op])
                 txt += '%s:%s [%s]\n'%(f, d, ', '.join(ops))
         self.existing_plugboards.setPlainText(txt)

src/calibre/gui2/preferences/plugboard.ui

@@ -87,6 +87,9 @@
        <property name="lineWrapMode">
         <enum>QPlainTextEdit::NoWrap</enum>
        </property>
+       <property name="readOnly">
+        <bool>true</bool>
+       </property>
       </widget>
      </item>
      <item row="4" column="0">
@@ -109,7 +112,7 @@
      <item row="0" column="0">
       <widget class="QLabel" name="label_2">
        <property name="text">
-        <string>Source field</string>
+        <string>Source template</string>
        </property>
        <property name="alignment">
         <set>Qt::AlignCenter</set>

src/calibre/library/caches.py

@@ -672,7 +672,7 @@ class ResultCache(SearchQueryParser): # {{{
             if len(self.composites) > 0:
                 mi = db.get_metadata(id, index_is_id=True)
                 for k,c in self.composites:
-                    self._data[id][c] = mi.format_field(k)[1]
+                    self._data[id][c] = mi.get(k)
         self._map[0:0] = ids
         self._map_filtered[0:0] = ids
 
@@ -702,7 +702,7 @@ class ResultCache(SearchQueryParser): # {{{
                 if len(self.composites) > 0:
                     mi = db.get_metadata(item[0], index_is_id=True)
                     for k,c in self.composites:
-                        item[c] = mi.format_field(k)[1]
+                        item[c] = mi.get(k)
 
         self._map = [i[0] for i in self._data if i is not None]
         if field is not None:

src/calibre/library/save_to_disk.py

@@ -112,8 +112,6 @@ class SafeFormat(TemplateFormatter):
     Provides a format function that substitutes '' for any missing value
     '''
 
-    composite_values = {}
-
     def get_value(self, key, args, kwargs):
         try:
             b = self.book.get_user_metadata(key, False)
@@ -131,11 +129,6 @@ class SafeFormat(TemplateFormatter):
         except:
             return ''
 
-    def safe_format(self, fmt, kwargs, error_value, book, sanitize=None):
-        self.composite_values = {}
-        return TemplateFormatter.safe_format(self, fmt, kwargs, error_value,
-                                             book, sanitize)
-
 safe_formatter = SafeFormat()
 
 def get_components(template, mi, id, timefmt='%b %Y', length=250,
@@ -279,7 +272,7 @@ def save_book_to_disk(id, db, root, opts, length):
             try:
                 if cpb:
                     newmi = mi.deepcopy()
-                    newmi.copy_specific_attributes(mi, cpb)
+                    newmi.template_to_attribute(mi, cpb)
                 else:
                     newmi = mi
                 set_metadata(stream, newmi, fmt)

src/calibre/utils/formatter.py

@@ -11,6 +11,10 @@ class TemplateFormatter(string.Formatter):
     Provides a format function that substitutes '' for any missing value
     '''
 
+    # Dict to do recursion detection. It is up the the individual get_value
+    # method to use it. It is cleared when starting to format a template
+    composite_values = {}
+
     def __init__(self):
         string.Formatter.__init__(self)
         self.book = None
@@ -114,6 +118,7 @@ class TemplateFormatter(string.Formatter):
         self.kwargs = kwargs
         self.book = book
         self.sanitize = sanitize
+        self.composite_values = {}
         try:
             ans = self.vformat(fmt, [], kwargs).strip()
         except: