Exporting books: Add support for metadata 'plugboard'. Fix regression in bulk edit performance

src/calibre/customize/builtins.py

@@ -797,6 +797,17 @@ class Sending(PreferencesPlugin):
     description = _('Control how calibre transfers files to your '
             'ebook reader')
 
+class Plugboard(PreferencesPlugin):
+    name = 'Plugboard'
+    icon = I('plugboard.png')
+    gui_name = _('Metadata plugboards')
+    category = 'Import/Export'
+    gui_category = _('Import/Export')
+    category_order = 3
+    name_order = 4
+    config_widget = 'calibre.gui2.preferences.plugboard'
+    description = _('Change metadata fields before saving/sending')
+
 class Email(PreferencesPlugin):
     name = 'Email'
     icon = I('mail.png')
@@ -857,8 +868,8 @@ class Misc(PreferencesPlugin):
     description = _('Miscellaneous advanced configuration')
 
 plugins += [LookAndFeel, Behavior, Columns, Toolbar, InputOptions,
-        CommonOptions, OutputOptions, Adding, Saving, Sending, Email, Server,
+        CommonOptions, OutputOptions, Adding, Saving, Sending, Plugboard,
-        Plugins, Tweaks, Misc]
+        Email, Server, Plugins, Tweaks, Misc]
 
 #}}}
 

src/calibre/ebooks/metadata/book/base.py

@@ -37,7 +37,13 @@ class SafeFormat(TemplateFormatter):
 
     def get_value(self, key, args, kwargs):
         try:
-            ign, v = self.book.format_field(key.lower(), series_with_index=False)
+            b = self.book.get_user_metadata(key, False)
+            if b and b['datatype'] == 'int' and self.book.get(key, 0) == 0:
+                v = ''
+            elif b and b['datatype'] == 'float' and b.get(key, 0.0) == 0.0:
+                v = ''
+            else:
+                ign, v = self.book.format_field(key.lower(), series_with_index=False)
             if v is None:
                 return ''
             if v == '':
@@ -65,7 +71,6 @@ class Metadata(object):
         '''
         _data = copy.deepcopy(NULL_VALUES)
         object.__setattr__(self, '_data', _data)
-        _data['_curseq'] = _data['_compseq'] = 0
         if other is not None:
             self.smart_update(other)
         else:
@@ -94,29 +99,22 @@ class Metadata(object):
         if field in _data['user_metadata'].iterkeys():
             d = _data['user_metadata'][field]
             val = d['#value#']
-            if d['datatype'] != 'composite' or \
+            if d['datatype'] != 'composite':
-                    (_data['_curseq'] == _data['_compseq'] and val is not None):
                 return val
-            # Data in the structure has changed. Recompute the composite fields
+            if val is None:
-            _data['_compseq'] = _data['_curseq']
+                d['#value#'] = 'RECURSIVE_COMPOSITE FIELD (Metadata) ' + field
-            for ck in _data['user_metadata']:
+                val = d['#value#'] = composite_formatter.safe_format(
-                cf = _data['user_metadata'][ck]
+                                            d['display']['composite_template'],
-                if cf['datatype'] != 'composite':
-                    continue
-                cf['#value#'] = 'RECURSIVE_COMPOSITE FIELD ' + field
-                cf['#value#'] = composite_formatter.safe_format(
-                                            cf['display']['composite_template'],
                                             self,
                                             _('TEMPLATE ERROR'),
                                             self).strip()
-            return d['#value#']
+            return val
 
         raise AttributeError(
                 'Metadata object has no attribute named: '+ repr(field))
 
     def __setattr__(self, field, val, extra=None):
         _data = object.__getattribute__(self, '_data')
-        _data['_curseq'] += 1
         if field in TOP_LEVEL_CLASSIFIERS:
             _data['classifiers'].update({field: val})
         elif field in STANDARD_METADATA_FIELDS:
@@ -124,7 +122,10 @@ class Metadata(object):
                 val = NULL_VALUES.get(field, None)
             _data[field] = val
         elif field in _data['user_metadata'].iterkeys():
-            _data['user_metadata'][field]['#value#'] = val
+            if _data['user_metadata'][field]['datatype'] == 'composite':
+                _data['user_metadata'][field]['#value#'] = None
+            else:
+                _data['user_metadata'][field]['#value#'] = val
             _data['user_metadata'][field]['#extra#'] = extra
         else:
             # You are allowed to stick arbitrary attributes onto this object as
@@ -182,7 +183,7 @@ class Metadata(object):
         return metadata describing a standard or custom field.
         '''
         if key not in self.custom_field_keys():
-            return self.get_standard_metadata(self, key, make_copy=False)
+            return self.get_standard_metadata(key, make_copy=False)
         return self.get_user_metadata(key, make_copy=False)
 
     def all_non_none_fields(self):
@@ -294,6 +295,28 @@ class Metadata(object):
             _data = object.__getattribute__(self, '_data')
             _data['user_metadata'][field] = metadata
 
+    def template_to_attribute(self, other, attrs):
+        '''
+        Takes a dict {src:dest, src:dest}, evaluates the template in the context
+        of other, then copies the result to self[dest]. This is on a best-
+        efforts basis. Some assignments can make no sense.
+        '''
+        if not attrs:
+            return
+        for src in attrs:
+            try:
+                val = composite_formatter.safe_format\
+                    (src, other, 'PLUGBOARD TEMPLATE ERROR', other)
+                dfm = self.metadata_for_field(attrs[src])
+                if dfm and dfm['is_multiple']:
+                    self.set(attrs[src],
+                        [f.strip() for f in val.split(',') if f.strip()])
+                else:
+                    self.set(attrs[src], val)
+            except:
+                traceback.print_exc()
+                pass
+
     # Old Metadata API {{{
     def print_all_attributes(self):
         for x in STANDARD_METADATA_FIELDS:

src/calibre/gui2/actions/choose_library.py

@@ -14,7 +14,7 @@ from calibre import isbytestring
 from calibre.constants import filesystem_encoding
 from calibre.utils.config import prefs
 from calibre.gui2 import gprefs, warning_dialog, Dispatcher, error_dialog, \
-    question_dialog
+    question_dialog, info_dialog
 from calibre.gui2.actions import InterfaceAction
 
 class LibraryUsageStats(object):
@@ -115,6 +115,14 @@ class ChooseLibraryAction(InterfaceAction):
                     type=Qt.QueuedConnection)
             self.choose_menu.addAction(ac)
 
+        self.rename_separator = self.choose_menu.addSeparator()
+
+        self.create_action(spec=(_('Library backup status...'), 'lt.png', None,
+            None), attr='action_backup_status')
+        self.action_backup_status.triggered.connect(self.backup_status,
+                type=Qt.QueuedConnection)
+        self.choose_menu.addAction(self.action_backup_status)
+
     def library_name(self):
         db = self.gui.library_view.model().db
         path = db.library_path
@@ -206,6 +214,16 @@ class ChooseLibraryAction(InterfaceAction):
         self.stats.remove(location)
         self.build_menus()
 
+    def backup_status(self, location):
+        dirty_text = 'no'
+        try:
+            dirty_text = \
+                  unicode(self.gui.library_view.model().db.dirty_queue_length())
+        except:
+            dirty_text = _('none')
+        info_dialog(self.gui, _('Backup status'), '<p>'+
+                _('Book metadata files remaining to be written: %s') % dirty_text,
+                show=True)
 
     def switch_requested(self, location):
         if not self.change_library_allowed():

src/calibre/gui2/device.py

@@ -34,6 +34,8 @@ from calibre.ebooks.metadata.meta import set_metadata
 from calibre.constants import DEBUG
 from calibre.utils.config import prefs, tweaks
 from calibre.utils.magick.draw import thumbnail
+from calibre.library.save_to_disk import plugboard_any_device_value, \
+                                         plugboard_any_format_value
 # }}}
 
 class DeviceJob(BaseJob): # {{{
@@ -317,19 +319,40 @@ class DeviceManager(Thread): # {{{
                                args=[booklist, on_card],
                         description=_('Send collections to device'))
 
-    def _upload_books(self, files, names, on_card=None, metadata=None):
+    def _upload_books(self, files, names, on_card=None, metadata=None, plugboards=None):
         '''Upload books to device: '''
         if metadata and files and len(metadata) == len(files):
             for f, mi in zip(files, metadata):
                 if isinstance(f, unicode):
                     ext = f.rpartition('.')[-1].lower()
+                    dev_name = self.connected_device.__class__.__name__
+                    cpb = None
+                    if ext in plugboards:
+                        cpb = plugboards[ext]
+                    elif plugboard_any_format_value in plugboards:
+                        cpb = plugboards[plugboard_any_format_value]
+                    if cpb is not None:
+                        if dev_name in cpb:
+                            cpb = cpb[dev_name]
+                        elif plugboard_any_device_value in plugboards[ext]:
+                            cpb = cpb[plugboard_any_device_value]
+                        else:
+                            cpb = None
+
+                    if DEBUG:
+                        prints('Using plugboard', ext, dev_name, cpb)
                     if ext:
                         try:
                             if DEBUG:
                                 prints('Setting metadata in:', mi.title, 'at:',
                                         f, file=sys.__stdout__)
                             with open(f, 'r+b') as stream:
-                                set_metadata(stream, mi, stream_type=ext)
+                                if cpb:
+                                    newmi = mi.deepcopy()
+                                    newmi.template_to_attribute(mi, cpb)
+                                else:
+                                    newmi = mi
+                                set_metadata(stream, newmi, stream_type=ext)
                         except:
                             if DEBUG:
                                 prints(traceback.format_exc(), file=sys.__stdout__)
@@ -338,12 +361,12 @@ class DeviceManager(Thread): # {{{
                                         metadata=metadata, end_session=False)
 
     def upload_books(self, done, files, names, on_card=None, titles=None,
-                     metadata=None):
+                     metadata=None, plugboards=None):
         desc = _('Upload %d books to device')%len(names)
         if titles:
             desc += u':' + u', '.join(titles)
         return self.create_job(self._upload_books, done, args=[files, names],
-                kwargs={'on_card':on_card,'metadata':metadata}, description=desc)
+                kwargs={'on_card':on_card,'metadata':metadata,'plugboards':plugboards}, description=desc)
 
     def add_books_to_metadata(self, locations, metadata, booklists):
         self.device.add_books_to_metadata(locations, metadata, booklists)
@@ -1257,10 +1280,11 @@ class DeviceMixin(object): # {{{
         :param files: List of either paths to files or file like objects
         '''
         titles = [i.title for i in metadata]
+        plugboards = self.library_view.model().db.prefs.get('plugboards', {})
         job = self.device_manager.upload_books(
                 Dispatcher(self.books_uploaded),
                 files, names, on_card=on_card,
-                metadata=metadata, titles=titles
+                metadata=metadata, titles=titles, plugboards=plugboards
               )
         self.upload_memory[job] = (metadata, on_card, memory, files)
 

src/calibre/gui2/dialogs/metadata_bulk.py

@@ -3,42 +3,110 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 
 '''Dialog to edit metadata in bulk'''
 
-from threading import Thread
+import re
-import re, string
 
-from PyQt4.Qt import Qt, QDialog, QGridLayout
+from PyQt4.Qt import Qt, QDialog, QGridLayout, QVBoxLayout, QFont, QLabel, \
+                     pyqtSignal
 from PyQt4 import QtGui
 
 from calibre.gui2.dialogs.metadata_bulk_ui import Ui_MetadataBulkDialog
 from calibre.gui2.dialogs.tag_editor import TagEditor
 from calibre.ebooks.metadata import string_to_authors, authors_to_string
 from calibre.gui2.custom_column_widgets import populate_metadata_page
-from calibre.gui2.dialogs.progress import BlockingBusy
+from calibre.gui2 import error_dialog
-from calibre.gui2 import error_dialog, Dispatcher
+from calibre.gui2.progress_indicator import ProgressIndicator
 from calibre.utils.config import dynamic
 
-class Worker(Thread):
+class MyBlockingBusy(QDialog):
+
+    do_one_signal = pyqtSignal()
+
+    phases = ['',
+              _('Title/Author'),
+              _('Standard metadata'),
+              _('Custom metadata'),
+              _('Search/Replace'),
+    ]
+
+    def __init__(self, msg, args, db, ids, cc_widgets, s_r_func,
+                 parent=None, window_title=_('Working')):
+        QDialog.__init__(self, parent)
+
+        self._layout = QVBoxLayout()
+        self.setLayout(self._layout)
+        self.msg_text = msg
+        self.msg = QLabel(msg+'        ') # Ensure dialog is wide enough
+        #self.msg.setWordWrap(True)
+        self.font = QFont()
+        self.font.setPointSize(self.font.pointSize() + 8)
+        self.msg.setFont(self.font)
+        self.pi = ProgressIndicator(self)
+        self.pi.setDisplaySize(100)
+        self._layout.addWidget(self.pi, 0, Qt.AlignHCenter)
+        self._layout.addSpacing(15)
+        self._layout.addWidget(self.msg, 0, Qt.AlignHCenter)
+        self.setWindowTitle(window_title)
+        self.resize(self.sizeHint())
+        self.start()
 
-    def __init__(self, args, db, ids, cc_widgets, callback):
-        Thread.__init__(self)
         self.args = args
         self.db = db
         self.ids = ids
         self.error = None
-        self.callback = callback
         self.cc_widgets = cc_widgets
+        self.s_r_func = s_r_func
+        self.do_one_signal.connect(self.do_one_safe, Qt.QueuedConnection)
 
-    def doit(self):
+    def start(self):
+        self.pi.startAnimation()
+
+    def stop(self):
+        self.pi.stopAnimation()
+
+    def accept(self):
+        self.stop()
+        return QDialog.accept(self)
+
+    def exec_(self):
+        self.current_index = 0
+        self.current_phase = 1
+        self.do_one_signal.emit()
+        return QDialog.exec_(self)
+
+    def do_one_safe(self):
+        try:
+            if self.current_index >= len(self.ids):
+                self.current_phase += 1
+                self.current_index = 0
+                if self.current_phase > 4:
+                    self.db.commit()
+                    return self.accept()
+            id = self.ids[self.current_index]
+            percent = int((self.current_index*100)/float(len(self.ids)))
+            self.msg.setText(self.msg_text.format(self.phases[self.current_phase],
+                                        percent))
+            self.do_one(id)
+        except Exception, err:
+            import traceback
+            try:
+                err = unicode(err)
+            except:
+                err = repr(err)
+            self.error = (err, traceback.format_exc())
+            return self.accept()
+
+    def do_one(self, id):
         remove, add, au, aus, do_aus, rating, pub, do_series, \
             do_autonumber, do_remove_format, remove_format, do_swap_ta, \
             do_remove_conv, do_auto_author, series, do_series_restart, \
             series_start_value, do_title_case, clear_series = self.args
 
+
         # first loop: do author and title. These will commit at the end of each
         # operation, because each operation modifies the file system. We want to
         # try hard to keep the DB and the file system in sync, even in the face
         # of exceptions or forced exits.
-        for id in self.ids:
+        if self.current_phase == 1:
             title_set = False
             if do_swap_ta:
                 title = self.db.title(id, index_is_id=True)
@@ -58,9 +126,8 @@ class Worker(Thread):
                 self.db.set_title(id, title.title(), notify=False)
             if au:
                 self.db.set_authors(id, string_to_authors(au), notify=False)
-
+        elif self.current_phase == 2:
-        # All of these just affect the DB, so we can tolerate a total rollback
+            # All of these just affect the DB, so we can tolerate a total rollback
-        for id in self.ids:
             if do_auto_author:
                 x = self.db.author_sort_from_book(id, index_is_id=True)
                 if x:
@@ -93,37 +160,20 @@ class Worker(Thread):
 
             if do_remove_conv:
                 self.db.delete_conversion_options(id, 'PIPE', commit=False)
-        self.db.commit()
+        elif self.current_phase == 3:
+            # both of these are fast enough to just do them all
+            for w in self.cc_widgets:
+                w.commit(self.ids)
+            self.db.bulk_modify_tags(self.ids, add=add, remove=remove,
+                                         notify=False)
+            self.current_index = len(self.ids)
+        elif self.current_phase == 4:
+            self.s_r_func(id)
+            self.current_index = len(self.ids)
+        # do the next one
+        self.current_index += 1
+        self.do_one_signal.emit()
 
-        for w in self.cc_widgets:
-            w.commit(self.ids)
-        self.db.bulk_modify_tags(self.ids, add=add, remove=remove,
-                notify=False)
-
-    def run(self):
-        try:
-            self.doit()
-        except Exception, err:
-            import traceback
-            try:
-                err = unicode(err)
-            except:
-                err = repr(err)
-            self.error = (err, traceback.format_exc())
-
-        self.callback()
-
-class SafeFormat(string.Formatter):
-    '''
-    Provides a format function that substitutes '' for any missing value
-    '''
-    def get_value(self, key, args, vals):
-        v = vals.get(key, None)
-        if v is None:
-            return ''
-        if isinstance(v, (tuple, list)):
-            v = ','.join(v)
-        return v
 
 class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
 
@@ -452,7 +502,7 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
                 self.s_r_set_colors()
                 break
 
-    def do_search_replace(self):
+    def do_search_replace(self, id):
         source = unicode(self.search_field.currentText())
         if not source or not self.s_r_obj:
             return
@@ -461,48 +511,45 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
             dest = source
         dfm = self.db.field_metadata[dest]
 
-        for id in self.ids:
+        mi = self.db.get_metadata(id, index_is_id=True,)
-            mi = self.db.get_metadata(id, index_is_id=True,)
+        val = mi.get(source)
-            val = mi.get(source)
+        if val is None:
-            if val is None:
+            return
-                continue
+        val = self.s_r_do_regexp(mi)
-            val = self.s_r_do_regexp(mi)
+        val = self.s_r_do_destination(mi, val)
-            val = self.s_r_do_destination(mi, val)
+        if dfm['is_multiple']:
-            if dfm['is_multiple']:
-                if dfm['is_custom']:
-                    # The standard tags and authors values want to be lists.
-                    # All custom columns are to be strings
-                    val = dfm['is_multiple'].join(val)
-                if dest == 'authors' and len(val) == 0:
-                    error_dialog(self, _('Search/replace invalid'),
-                                 _('Authors cannot be set to the empty string. '
-                                   'Book title %s not processed')%mi.title,
-                                 show=True)
-                    continue
-            else:
-                val = self.s_r_replace_mode_separator().join(val)
-                if dest == 'title' and len(val) == 0:
-                    error_dialog(self, _('Search/replace invalid'),
-                                 _('Title cannot be set to the empty string. '
-                                   'Book title %s not processed')%mi.title,
-                                 show=True)
-                    continue
-
             if dfm['is_custom']:
-                extra = self.db.get_custom_extra(id, label=dfm['label'], index_is_id=True)
+                # The standard tags and authors values want to be lists.
-                self.db.set_custom(id, val, label=dfm['label'], extra=extra,
+                # All custom columns are to be strings
-                                   commit=False)
+                val = dfm['is_multiple'].join(val)
+            if dest == 'authors' and len(val) == 0:
+                error_dialog(self, _('Search/replace invalid'),
+                             _('Authors cannot be set to the empty string. '
+                               'Book title %s not processed')%mi.title,
+                             show=True)
+                return
+        else:
+            val = self.s_r_replace_mode_separator().join(val)
+            if dest == 'title' and len(val) == 0:
+                error_dialog(self, _('Search/replace invalid'),
+                             _('Title cannot be set to the empty string. '
+                               'Book title %s not processed')%mi.title,
+                             show=True)
+                return
+
+        if dfm['is_custom']:
+            extra = self.db.get_custom_extra(id, label=dfm['label'], index_is_id=True)
+            self.db.set_custom(id, val, label=dfm['label'], extra=extra,
+                               commit=False)
+        else:
+            if dest == 'comments':
+                setter = self.db.set_comment
             else:
-                if dest == 'comments':
+                setter = getattr(self.db, 'set_'+dest)
-                    setter = self.db.set_comment
+            if dest in ['title', 'authors']:
-                else:
+                setter(id, val, notify=False)
-                    setter = getattr(self.db, 'set_'+dest)
+            else:
-                if dest in ['title', 'authors']:
+                setter(id, val, notify=False, commit=False)
-                    setter(id, val, notify=False)
-                else:
-                    setter(id, val, notify=False, commit=False)
-        self.db.commit()
-        dynamic['s_r_search_mode'] = self.search_mode.currentIndex()
 
     def create_custom_column_editors(self):
         w = self.central_widget.widget(1)
@@ -525,11 +572,11 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
 
     def initalize_authors(self):
         all_authors = self.db.all_authors()
-        all_authors.sort(cmp=lambda x, y : cmp(x[1], y[1]))
+        all_authors.sort(cmp=lambda x, y : cmp(x[1].lower(), y[1].lower()))
 
         for i in all_authors:
             id, name = i
-            name = authors_to_string([name.strip().replace('|', ',') for n in name.split(',')])
+            name = name.strip().replace('|', ',')
             self.authors.addItem(name)
         self.authors.setEditText('')
 
@@ -613,28 +660,25 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
                 do_remove_conv, do_auto_author, series, do_series_restart,
                 series_start_value, do_title_case, clear_series)
 
-        bb = BlockingBusy(_('Applying changes to %d books. This may take a while.')
+        bb = MyBlockingBusy(_('Applying changes to %d books.\nPhase {0} {1}%%.')
-                %len(self.ids), parent=self)
+                %len(self.ids), args, self.db, self.ids,
-        self.worker = Worker(args, self.db, self.ids,
                 getattr(self, 'custom_column_widgets', []),
-                Dispatcher(bb.accept, parent=bb))
+                self.do_search_replace, parent=self)
 
         # The metadata backup thread causes database commits
         # which can slow down bulk editing of large numbers of books
         self.model.stop_metadata_backup()
         try:
-            self.worker.start()
             bb.exec_()
         finally:
             self.model.start_metadata_backup()
 
-        if self.worker.error is not None:
+        if bb.error is not None:
             return error_dialog(self, _('Failed'),
-                    self.worker.error[0], det_msg=self.worker.error[1],
+                    bb.error[0], det_msg=bb.error[1],
                     show=True)
 
-        self.do_search_replace()
+        dynamic['s_r_search_mode'] = self.search_mode.currentIndex()
-
         self.db.clean()
         return QDialog.accept(self)
 

src/calibre/gui2/library/models.py

@@ -750,8 +750,10 @@ class BooksModel(QAbstractTableModel): # {{{
             self.refresh(reset=True)
             return True
 
-        self.db.set_custom(self.db.id(row), val, extra=s_index,
+        id = self.db.id(row)
+        self.db.set_custom(id, val, extra=s_index,
                            label=label, num=None, append=False, notify=True)
+        self.refresh_ids([id], current_row=row)
         return True
 
     def setData(self, index, value, role):

src/calibre/gui2/preferences/plugboard.py

@@ -0,0 +1,307 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+__license__   = 'GPL v3'
+__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from PyQt4 import QtGui
+
+from calibre.gui2 import error_dialog
+from calibre.gui2.preferences import ConfigWidgetBase, test_widget
+from calibre.gui2.preferences.plugboard_ui import Ui_Form
+from calibre.customize.ui import metadata_writers, device_plugins
+from calibre.library.save_to_disk import plugboard_any_format_value, \
+                        plugboard_any_device_value, plugboard_save_to_disk_value
+from calibre.utils.formatter import validation_formatter
+
+class ConfigWidget(ConfigWidgetBase, Ui_Form):
+
+    def genesis(self, gui):
+        self.gui = gui
+        self.db = gui.library_view.model().db
+        self.current_plugboards = self.db.prefs.get('plugboards',{})
+        self.current_device = None
+        self.current_format = None
+
+    def initialize(self):
+        def field_cmp(x, y):
+            if x.startswith('#'):
+                if y.startswith('#'):
+                    return cmp(x.lower(), y.lower())
+                else:
+                    return 1
+            elif y.startswith('#'):
+                return -1
+            else:
+                return cmp(x.lower(), y.lower())
+
+        ConfigWidgetBase.initialize(self)
+
+        self.devices = ['']
+        for device in device_plugins():
+            n = device.__class__.__name__
+            if n.startswith('FOLDER_DEVICE'):
+                n = 'FOLDER_DEVICE'
+            self.devices.append(n)
+        self.devices.sort(cmp=lambda x, y: cmp(x.lower(), y.lower()))
+        self.devices.insert(1, plugboard_save_to_disk_value)
+        self.devices.insert(2, plugboard_any_device_value)
+        self.new_device.addItems(self.devices)
+
+        self.formats = ['']
+        for w in metadata_writers():
+            for f in w.file_types:
+                self.formats.append(f)
+        self.formats.sort()
+        self.formats.insert(1, plugboard_any_format_value)
+        self.new_format.addItems(self.formats)
+
+        self.source_fields = ['']
+        for f in self.db.custom_field_keys():
+            if self.db.field_metadata[f]['datatype'] == 'composite':
+                self.source_fields.append(f)
+        self.source_fields.sort(cmp=field_cmp)
+
+        self.dest_fields = ['',
+                            'authors', 'author_sort', 'language', 'publisher',
+                            'tags', 'title', 'title_sort']
+
+        self.source_widgets = []
+        self.dest_widgets = []
+        for i in range(0, len(self.dest_fields)-1):
+            w = QtGui.QLineEdit(self)
+            self.source_widgets.append(w)
+            self.fields_layout.addWidget(w, 5+i, 0, 1, 1)
+            w = QtGui.QComboBox(self)
+            self.dest_widgets.append(w)
+            self.fields_layout.addWidget(w, 5+i, 1, 1, 1)
+
+        self.edit_device.currentIndexChanged[str].connect(self.edit_device_changed)
+        self.edit_format.currentIndexChanged[str].connect(self.edit_format_changed)
+        self.new_device.currentIndexChanged[str].connect(self.new_device_changed)
+        self.new_format.currentIndexChanged[str].connect(self.new_format_changed)
+        self.ok_button.clicked.connect(self.ok_clicked)
+        self.del_button.clicked.connect(self.del_clicked)
+
+        self.refill_all_boxes()
+
+    def clear_fields(self, edit_boxes=False, new_boxes=False):
+        self.ok_button.setEnabled(False)
+        self.del_button.setEnabled(False)
+        for w in self.source_widgets:
+            w.clear()
+        for w in self.dest_widgets:
+            w.clear()
+        if edit_boxes:
+            self.edit_device.setCurrentIndex(0)
+            self.edit_format.setCurrentIndex(0)
+        if new_boxes:
+            self.new_device.setCurrentIndex(0)
+            self.new_format.setCurrentIndex(0)
+
+    def set_fields(self):
+        self.ok_button.setEnabled(True)
+        self.del_button.setEnabled(True)
+        for w in self.source_widgets:
+            w.clear()
+        for w in self.dest_widgets:
+            w.addItems(self.dest_fields)
+
+    def set_field(self, i, src, dst):
+        self.source_widgets[i].setText(src)
+        idx = self.dest_fields.index(dst)
+        self.dest_widgets[i].setCurrentIndex(idx)
+
+    def edit_device_changed(self, txt):
+        if txt == '':
+            self.current_device = None
+            return
+        self.clear_fields(new_boxes=True)
+        self.current_device = unicode(txt)
+        fpb = self.current_plugboards.get(self.current_format, None)
+        if fpb is None:
+            print 'edit_device_changed: none format!'
+            return
+        dpb = fpb.get(self.current_device, None)
+        if dpb is None:
+            print 'edit_device_changed: none device!'
+            return
+        self.set_fields()
+        for i,src in enumerate(dpb):
+            self.set_field(i, src, dpb[src])
+        self.ok_button.setEnabled(True)
+        self.del_button.setEnabled(True)
+
+    def edit_format_changed(self, txt):
+        if txt == '':
+            self.edit_device.setCurrentIndex(0)
+            self.current_format = None
+            self.current_device = None
+            return
+        self.clear_fields(new_boxes=True)
+        txt = unicode(txt)
+        fpb = self.current_plugboards.get(txt, None)
+        if fpb is None:
+            print 'edit_format_changed: none editable format!'
+            return
+        self.current_format = txt
+        devices = ['']
+        for d in fpb:
+            devices.append(d)
+        self.edit_device.clear()
+        self.edit_device.addItems(devices)
+        self.edit_device.setCurrentIndex(0)
+
+    def new_device_changed(self, txt):
+        if txt == '':
+            self.current_device = None
+            return
+        self.clear_fields(edit_boxes=True)
+        self.current_device = unicode(txt)
+        error = False
+        if self.current_format == plugboard_any_format_value:
+            # user specified any format.
+            for f in self.current_plugboards:
+                devs = set(self.current_plugboards[f])
+                print 'check', self.current_format, devs
+                if self.current_device != plugboard_save_to_disk_value and \
+                        plugboard_any_device_value in devs:
+                    # specific format/any device in list. conflict.
+                    # note: any device does not match save_to_disk
+                    error = True
+                    break
+                if self.current_device in devs:
+                    # specific format/current device in list. conflict
+                    error = True
+                    break
+                if self.current_device == plugboard_any_device_value:
+                    # any device and a specific device already there. conflict
+                    error = True
+                    break
+        else:
+            # user specified specific format.
+            for f in self.current_plugboards:
+                devs = set(self.current_plugboards[f])
+                if f == plugboard_any_format_value and \
+                                self.current_device in devs:
+                    # any format/same device in list. conflict.
+                    error = True
+                    break
+                if f == self.current_format and self.current_device in devs:
+                    # current format/current device in list. conflict
+                    error = True
+                    break
+                if f == self.current_format and plugboard_any_device_value in devs:
+                    # current format/any device in list. conflict
+                    error = True
+                    break
+
+        if error:
+            error_dialog(self, '',
+                     _('That format and device already has a plugboard or '
+                       'conflicts with another plugboard.'),
+                     show=True)
+            self.new_device.setCurrentIndex(0)
+            return
+        self.set_fields()
+
+    def new_format_changed(self, txt):
+        if txt == '':
+            self.current_format = None
+            self.current_device = None
+            return
+        self.clear_fields(edit_boxes=True)
+        self.current_format = unicode(txt)
+        self.new_device.setCurrentIndex(0)
+
+    def ok_clicked(self):
+        pb = {}
+        for i in range(0, len(self.source_widgets)):
+            s = unicode(self.source_widgets[i].text())
+            if s:
+                d = self.dest_widgets[i].currentIndex()
+                if d != 0:
+                    try:
+                        validation_formatter.validate(s)
+                    except Exception, err:
+                        error_dialog(self, _('Invalid template'),
+                                '<p>'+_('The template %s is invalid:')%s + \
+                                '<br>'+str(err), show=True)
+                        return
+                    pb[s] = self.dest_fields[d]
+                else:
+                    error_dialog(self, _('Invalid destination'),
+                            '<p>'+_('The destination field cannot be blank'),
+                            show=True)
+                    return
+
+        if len(pb) == 0:
+            if self.current_format in self.current_plugboards:
+                fpb = self.current_plugboards[self.current_format]
+                if self.current_device in fpb:
+                    del fpb[self.current_device]
+                if len(fpb) == 0:
+                    del self.current_plugboards[self.current_format]
+        else:
+            if self.current_format not in self.current_plugboards:
+                self.current_plugboards[self.current_format] = {}
+            fpb = self.current_plugboards[self.current_format]
+            fpb[self.current_device] = pb
+        self.changed_signal.emit()
+        self.refill_all_boxes()
+
+    def del_clicked(self):
+        if self.current_format in self.current_plugboards:
+            fpb = self.current_plugboards[self.current_format]
+            if self.current_device in fpb:
+                del fpb[self.current_device]
+            if len(fpb) == 0:
+                del self.current_plugboards[self.current_format]
+        self.changed_signal.emit()
+        self.refill_all_boxes()
+
+    def refill_all_boxes(self):
+        self.current_device = None
+        self.current_format = None
+        self.clear_fields(new_boxes=True)
+        self.edit_format.clear()
+        self.edit_format.addItem('')
+        for format in self.current_plugboards:
+            self.edit_format.addItem(format)
+        self.edit_format.setCurrentIndex(0)
+        self.edit_device.clear()
+        self.ok_button.setEnabled(False)
+        self.del_button.setEnabled(False)
+        txt = ''
+        for f in self.formats:
+            if f not in self.current_plugboards:
+                continue
+            for d in self.devices:
+                if d not in self.current_plugboards[f]:
+                    continue
+                ops = []
+                for op in self.current_plugboards[f][d]:
+                    ops.append(op + '->' + self.current_plugboards[f][d][op])
+                txt += '%s:%s [%s]\n'%(f, d, ', '.join(ops))
+        self.existing_plugboards.setPlainText(txt)
+
+    def restore_defaults(self):
+        ConfigWidgetBase.restore_defaults(self)
+        self.current_plugboards = {}
+        self.refill_all_boxes()
+        self.changed_signal.emit()
+
+    def commit(self):
+        self.db.prefs.set('plugboards', self.current_plugboards)
+        return ConfigWidgetBase.commit(self)
+
+    def refresh_gui(self, gui):
+        pass
+
+
+if __name__ == '__main__':
+    from PyQt4.Qt import QApplication
+    app = QApplication([])
+    test_widget('Import/Export', 'plugboards')
+

src/calibre/gui2/preferences/plugboard.ui

@@ -0,0 +1,224 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <class>Form</class>
+ <widget class="QWidget" name="Form">
+  <property name="geometry">
+   <rect>
+    <x>0</x>
+    <y>0</y>
+    <width>707</width>
+    <height>340</height>
+   </rect>
+  </property>
+  <property name="windowTitle">
+   <string>Form</string>
+  </property>
+  <layout class="QGridLayout" name="gridLayout">
+   <item row="0" column="0" colspan="2">
+    <widget class="QLabel" name="label">
+     <property name="text">
+      <string>Here you can change the metadata calibre uses to update a book when saving to disk or sending to device.
+
+Use this dialog to define a 'plugboard' for for a format (or all formats) and a device (or all devices). The plugboard spefies what template is connected to what field. The template is used to compute a value, and that value is assigned to the connected field.
+
+Often templates will contain simple references to composite columns, but this is not necessary. You can use any template in a source box that you can use elsewhere in calibre.
+
+One possible use for a plugboard is to alter the title to contain series informaton. Another would be to change the author sort, something that mobi users might do to force it to use the ';' that the kindle requires. A third would be to specify the language.</string>
+     </property>
+     <property name="textFormat">
+      <enum>Qt::PlainText</enum>
+     </property>
+     <property name="wordWrap">
+      <bool>true</bool>
+     </property>
+    </widget>
+   </item>
+   <item row="1" column="0" colspan="2">
+    <widget class="Line" name="line">
+     <property name="orientation">
+      <enum>Qt::Horizontal</enum>
+     </property>
+    </widget>
+   </item>
+   <item row="2" column="0">
+    <layout class="QGridLayout" name="gridLayout_2">
+     <item row="0" column="1">
+      <widget class="QLabel" name="label_6">
+       <property name="text">
+        <string>Format (choose first)</string>
+       </property>
+       <property name="alignment">
+        <set>Qt::AlignCenter</set>
+       </property>
+      </widget>
+     </item>
+     <item row="0" column="2">
+      <widget class="QLabel" name="label_7">
+       <property name="text">
+        <string>Device (choose second)</string>
+       </property>
+       <property name="alignment">
+        <set>Qt::AlignCenter</set>
+       </property>
+      </widget>
+     </item>
+     <item row="1" column="0">
+      <widget class="QLabel" name="label_5">
+       <property name="text">
+        <string>Add new plugboard</string>
+       </property>
+      </widget>
+     </item>
+     <item row="1" column="1">
+      <widget class="QComboBox" name="new_format"/>
+     </item>
+     <item row="1" column="2">
+      <widget class="QComboBox" name="new_device"/>
+     </item>
+     <item row="2" column="0">
+      <widget class="QLabel" name="label_4">
+       <property name="text">
+        <string>Edit existing plugboard</string>
+       </property>
+      </widget>
+     </item>
+     <item row="2" column="1">
+      <widget class="QComboBox" name="edit_format"/>
+     </item>
+     <item row="2" column="2">
+      <widget class="QComboBox" name="edit_device"/>
+     </item>
+     <item row="3" column="0">
+      <widget class="QLabel" name="label_41">
+       <property name="text">
+        <string>Existing plugboards</string>
+       </property>
+       <property name="alignment">
+        <set>Qt::AlignLeading|Qt::AlignLeft|Qt::AlignTop</set>
+       </property>
+      </widget>
+     </item>
+     <item row="3" column="1" colspan="2">
+      <widget class="QPlainTextEdit" name="existing_plugboards">
+       <property name="lineWrapMode">
+        <enum>QPlainTextEdit::NoWrap</enum>
+       </property>
+       <property name="readOnly">
+        <bool>true</bool>
+       </property>
+      </widget>
+     </item>
+     <item row="4" column="0">
+      <spacer name="verticalSpacer">
+       <property name="orientation">
+        <enum>Qt::Vertical</enum>
+       </property>
+       <property name="sizeHint" stdset="0">
+        <size>
+         <width>20</width>
+         <height>40</height>
+        </size>
+       </property>
+      </spacer>
+     </item>
+    </layout>
+   </item>
+   <item row="2" column="1">
+    <layout class="QGridLayout" name="fields_layout">
+     <item row="0" column="0">
+      <widget class="QLabel" name="label_2">
+       <property name="text">
+        <string>Source template</string>
+       </property>
+       <property name="alignment">
+        <set>Qt::AlignCenter</set>
+       </property>
+      </widget>
+     </item>
+     <item row="0" column="1">
+      <widget class="QLabel" name="label_3">
+       <property name="text">
+        <string>Destination field</string>
+       </property>
+       <property name="alignment">
+        <set>Qt::AlignCenter</set>
+       </property>
+      </widget>
+     </item>
+     <item row="21" column="0">
+      <spacer name="verticalSpacer_2">
+       <property name="orientation">
+        <enum>Qt::Vertical</enum>
+       </property>
+       <property name="sizeHint" stdset="0">
+        <size>
+         <width>20</width>
+         <height>40</height>
+        </size>
+       </property>
+      </spacer>
+     </item>
+     <item row="19" column="0">
+      <layout class="QHBoxLayout" name="horizontalLayout">
+       <item>
+        <spacer name="horizontalSpacer_2">
+         <property name="orientation">
+          <enum>Qt::Horizontal</enum>
+         </property>
+         <property name="sizeHint" stdset="0">
+          <size>
+           <width>40</width>
+           <height>20</height>
+          </size>
+         </property>
+        </spacer>
+       </item>
+       <item>
+        <widget class="QPushButton" name="ok_button">
+         <property name="text">
+          <string>Save plugboard</string>
+         </property>
+        </widget>
+       </item>
+       <item>
+        <spacer name="horizontalSpacer">
+         <property name="orientation">
+          <enum>Qt::Horizontal</enum>
+         </property>
+         <property name="sizeHint" stdset="0">
+          <size>
+           <width>40</width>
+           <height>20</height>
+          </size>
+         </property>
+        </spacer>
+       </item>
+       <item>
+        <widget class="QPushButton" name="del_button">
+         <property name="text">
+          <string>Delete plugboard</string>
+         </property>
+        </widget>
+       </item>
+       <item>
+        <spacer name="horizontalSpacer_3">
+         <property name="orientation">
+          <enum>Qt::Horizontal</enum>
+         </property>
+         <property name="sizeHint" stdset="0">
+          <size>
+           <width>40</width>
+           <height>20</height>
+          </size>
+         </property>
+        </spacer>
+       </item>
+      </layout>
+     </item>
+    </layout>
+   </item>
+  </layout>
+ </widget>
+ <resources/>
+ <connections/>
+</ui>

src/calibre/library/caches.py

@@ -138,25 +138,46 @@ class CoverCache(Thread): # {{{
     def run(self):
         while self.keep_running:
             try:
-                time.sleep(0.050) # Limit 20/second to not overwhelm the GUI
+                # The GUI puts the same ID into the queue many times. The code
+                # below emptys the queue, building a set of unique values. When
+                # the queue is empty, do the work
+                ids = set()
                 id_ = self.load_queue.get(True, 2)
+                ids.add(id_)
+                try:
+                    while True:
+                        # Give the gui some time to put values into the queue
+                        id_ = self.load_queue.get(True, 0.5)
+                        ids.add(id_)
+                except Empty:
+                    pass
+                except:
+                    # Happens during shutdown
+                    break
             except Empty:
                 continue
             except:
                 #Happens during interpreter shutdown
                 break
-            try:
+            if not self.keep_running:
-                img = self._image_for_id(id_)
-            except:
-                import traceback
-                traceback.print_exc()
-                continue
-            try:
-                with self.lock:
-                    self.cache[id_] = img
-            except:
-                # Happens during interpreter shutdown
                 break
+            for id_ in ids:
+                time.sleep(0.050) # Limit 20/second to not overwhelm the GUI
+                try:
+                    img = self._image_for_id(id_)
+                except:
+                    try:
+                        traceback.print_exc()
+                    except:
+                        # happens during shutdown
+                        break
+                    continue
+                try:
+                    with self.lock:
+                        self.cache[id_] = img
+                except:
+                    # Happens during interpreter shutdown
+                    break
 
     def set_cache(self, ids):
         with self.lock:
@@ -660,7 +681,7 @@ class ResultCache(SearchQueryParser): # {{{
             if len(self.composites) > 0:
                 mi = db.get_metadata(id, index_is_id=True)
                 for k,c in self.composites:
-                    self._data[id][c] = mi.format_field(k)[1]
+                    self._data[id][c] = mi.get(k)
         self._map[0:0] = ids
         self._map_filtered[0:0] = ids
 
@@ -690,7 +711,7 @@ class ResultCache(SearchQueryParser): # {{{
                 if len(self.composites) > 0:
                     mi = db.get_metadata(item[0], index_is_id=True)
                     for k,c in self.composites:
-                        item[c] = mi.format_field(k)[1]
+                        item[c] = mi.get(k)
 
         self._map = [i[0] for i in self._data if i is not None]
         if field is not None:
@@ -779,7 +800,7 @@ class SortKeyGenerator(object):
                     sidx = record[sidx_fm['rec_index']]
                     val = (val, sidx)
 
-            elif dt in ('text', 'comments'):
+            elif dt in ('text', 'comments', 'composite'):
                 if val is None:
                     val = ''
                 val = val.lower()

src/calibre/library/custom_columns.py

@@ -214,6 +214,7 @@ class CustomColumns(object):
             'SELECT id FROM %s WHERE value=?'%table, (new_name,), all=False)
         if new_id is None or old_id == new_id:
             self.conn.execute('UPDATE %s SET value=? WHERE id=?'%table, (new_name, old_id))
+            new_id = old_id
         else:
             # New id exists. If the column is_multiple, then process like
             # tags, otherwise process like publishers (see database2)
@@ -226,6 +227,7 @@ class CustomColumns(object):
             self.conn.execute('''UPDATE %s SET value=?
                                  WHERE value=?'''%lt, (new_id, old_id,))
             self.conn.execute('DELETE FROM %s WHERE id=?'%table, (old_id,))
+        self.dirty_books_referencing('#'+data['label'], new_id, commit=False)
         self.conn.commit()
 
     def delete_custom_item_using_id(self, id, label=None, num=None):

src/calibre/library/database2.py

@@ -47,13 +47,21 @@ def delete_file(path):
 
 def delete_tree(path, permanent=False):
     if permanent:
-        shutil.rmtree(path)
+        try:
+            # For completely mysterious reasons, sometimes a file is left open
+            # leading to access errors. If we get an exception, wait and hope
+            # that whatever has the file (the O/S?) lets go of it.
+            shutil.rmtree(path)
+        except:
+            traceback.print_exc()
+            time.sleep(1)
+            shutil.rmtree(path)
     else:
         try:
             if not permanent:
                 winshell.delete_file(path, silent=True, no_confirm=True)
         except:
-            shutil.rmtree(path)
+            delete_tree(path, permanent=True)
 
 copyfile = os.link if hasattr(os, 'link') else shutil.copyfile
 
@@ -627,6 +635,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         if commit:
             self.conn.commit()
 
+    def dirty_queue_length(self):
+        return len(self.dirtied_cache)
+
     def commit_dirty_cache(self):
         '''
         Set the dirty indication for every book in the cache. The vast majority
@@ -1286,7 +1297,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                      val=mi.get(key),
                      extra=mi.get_extra(key),
                      label=user_mi[key]['label'], commit=False)
-        self.commit()
+        self.conn.commit()
         self.notify('metadata', [id])
 
     def authors_sort_strings(self, id, index_is_id=False):
@@ -1444,6 +1455,19 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
     # Convenience methods for tags_list_editor
     # Note: we generally do not need to refresh_ids because library_view will
     # refresh everything.
+
+    def dirty_books_referencing(self, field, id, commit=True):
+        # Get the list of books to dirty -- all books that reference the item
+        table = self.field_metadata[field]['table']
+        link = self.field_metadata[field]['link_column']
+        bks = self.conn.get(
+            'SELECT book from books_{0}_link WHERE {1}=?'.format(table, link),
+            (id,))
+        books = []
+        for (book_id,) in bks:
+            books.append(book_id)
+        self.dirtied(books, commit=commit)
+
     def get_tags_with_ids(self):
         result = self.conn.get('SELECT id,name FROM tags')
         if not result:
@@ -1460,6 +1484,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             # there is a change of case
             self.conn.execute('''UPDATE tags SET name=?
                                  WHERE id=?''', (new_name, old_id))
+            self.dirty_books_referencing('tags', new_id, commit=False)
+            new_id = old_id
         else:
             # It is possible that by renaming a tag, the tag will appear
             # twice on a book. This will throw an integrity error, aborting
@@ -1477,9 +1503,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                                  WHERE tag=?''',(new_id, old_id,))
             # Get rid of the no-longer used publisher
             self.conn.execute('DELETE FROM tags WHERE id=?', (old_id,))
+        self.dirty_books_referencing('tags', new_id, commit=False)
         self.conn.commit()
 
     def delete_tag_using_id(self, id):
+        self.dirty_books_referencing('tags', id, commit=False)
         self.conn.execute('DELETE FROM books_tags_link WHERE tag=?', (id,))
         self.conn.execute('DELETE FROM tags WHERE id=?', (id,))
         self.conn.commit()
@@ -1496,6 +1524,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                     '''SELECT id from series
                        WHERE name=?''', (new_name,), all=False)
         if new_id is None or old_id == new_id:
+            new_id = old_id
             self.conn.execute('UPDATE series SET name=? WHERE id=?',
                               (new_name, old_id))
         else:
@@ -1519,15 +1548,17 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                                      SET series_index=?
                                      WHERE id=?''',(index, book_id,))
                 index = index + 1
+        self.dirty_books_referencing('series', new_id, commit=False)
         self.conn.commit()
 
     def delete_series_using_id(self, id):
+        self.dirty_books_referencing('series', id, commit=False)
         books = self.conn.get('SELECT book from books_series_link WHERE series=?', (id,))
         self.conn.execute('DELETE FROM books_series_link WHERE series=?', (id,))
         self.conn.execute('DELETE FROM series WHERE id=?', (id,))
-        self.conn.commit()
         for (book_id,) in books:
             self.conn.execute('UPDATE books SET series_index=1.0 WHERE id=?', (book_id,))
+        self.conn.commit()
 
     def get_publishers_with_ids(self):
         result = self.conn.get('SELECT id,name FROM publishers')
@@ -1541,6 +1572,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                     '''SELECT id from publishers
                        WHERE name=?''', (new_name,), all=False)
         if new_id is None or old_id == new_id:
+            new_id = old_id
             # New name doesn't exist. Simply change the old name
             self.conn.execute('UPDATE publishers SET name=? WHERE id=?', \
                               (new_name, old_id))
@@ -1551,9 +1583,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                                  WHERE publisher=?''',(new_id, old_id,))
             # Get rid of the no-longer used publisher
             self.conn.execute('DELETE FROM publishers WHERE id=?', (old_id,))
+        self.dirty_books_referencing('publisher', new_id, commit=False)
         self.conn.commit()
 
     def delete_publisher_using_id(self, old_id):
+        self.dirty_books_referencing('publisher', id, commit=False)
         self.conn.execute('''DELETE FROM books_publishers_link
                              WHERE publisher=?''', (old_id,))
         self.conn.execute('DELETE FROM publishers WHERE id=?', (old_id,))
@@ -1634,6 +1668,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             # Now delete the old author from the DB
             bks = self.conn.get('SELECT book FROM books_authors_link WHERE author=?', (old_id,))
             self.conn.execute('DELETE FROM authors WHERE id=?', (old_id,))
+        self.dirtied(books, commit=False)
         self.conn.commit()
         # the authors are now changed, either by changing the author's name
         # or replacing the author in the list. Now must fix up the books.

src/calibre/library/save_to_disk.py

@@ -19,6 +19,11 @@ from calibre.ebooks.metadata import fmt_sidx
 from calibre.ebooks.metadata import title_sort
 from calibre import strftime
 
+plugboard_any_device_value = 'any device'
+plugboard_any_format_value = 'any format'
+plugboard_save_to_disk_value = 'save_to_disk'
+
+
 DEFAULT_TEMPLATE = '{author_sort}/{title}/{title} - {authors}'
 DEFAULT_SEND_TEMPLATE = '{author_sort}/{title} - {authors}'
 
@@ -107,8 +112,6 @@ class SafeFormat(TemplateFormatter):
     Provides a format function that substitutes '' for any missing value
     '''
 
-    composite_values = {}
-
     def get_value(self, key, args, kwargs):
         try:
             b = self.book.get_user_metadata(key, False)
@@ -126,11 +129,6 @@ class SafeFormat(TemplateFormatter):
         except:
             return ''
 
-    def safe_format(self, fmt, kwargs, error_value, book, sanitize=None):
-        self.composite_values = {}
-        return TemplateFormatter.safe_format(self, fmt, kwargs, error_value,
-                                             book, sanitize)
-
 safe_formatter = SafeFormat()
 
 def get_components(template, mi, id, timefmt='%b %Y', length=250,
@@ -245,6 +243,23 @@ def save_book_to_disk(id, db, root, opts, length):
 
     written = False
     for fmt in formats:
+        global plugboard_save_to_disk_value, plugboard_any_format_value
+        dev_name = plugboard_save_to_disk_value
+        plugboards = db.prefs.get('plugboards', {})
+        cpb = None
+        if fmt in plugboards:
+            cpb = plugboards[fmt]
+            if dev_name in cpb:
+                cpb = cpb[dev_name]
+            else:
+                cpb = None
+        if cpb is None and plugboard_any_format_value in plugboards:
+            cpb = plugboards[plugboard_any_format_value]
+            if dev_name in cpb:
+                cpb = cpb[dev_name]
+            else:
+                cpb = None
+        #prints('Using plugboard:', fmt, cpb)
         data = db.format(id, fmt, index_is_id=True)
         if data is None:
             continue
@@ -255,7 +270,12 @@ def save_book_to_disk(id, db, root, opts, length):
             stream.write(data)
             stream.seek(0)
             try:
-                set_metadata(stream, mi, fmt)
+                if cpb:
+                    newmi = mi.deepcopy()
+                    newmi.template_to_attribute(mi, cpb)
+                else:
+                    newmi = mi
+                set_metadata(stream, newmi, fmt)
             except:
                 traceback.print_exc()
             stream.seek(0)

src/calibre/utils/formatter.py

@@ -11,6 +11,10 @@ class TemplateFormatter(string.Formatter):
     Provides a format function that substitutes '' for any missing value
     '''
 
+    # Dict to do recursion detection. It is up the the individual get_value
+    # method to use it. It is cleared when starting to format a template
+    composite_values = {}
+
     def __init__(self):
         string.Formatter.__init__(self)
         self.book = None
@@ -114,6 +118,7 @@ class TemplateFormatter(string.Formatter):
         self.kwargs = kwargs
         self.book = book
         self.sanitize = sanitize
+        self.composite_values = {}
         try:
             ans = self.vformat(fmt, [], kwargs).strip()
         except: