1) add dirtied when renaming items

2) make bulk edit use the GUI thread
3) add a 'books remaiing' menu item

src/calibre/gui2/actions/choose_library.py

@@ -14,7 +14,7 @@ from calibre import isbytestring
 from calibre.constants import filesystem_encoding
 from calibre.utils.config import prefs
 from calibre.gui2 import gprefs, warning_dialog, Dispatcher, error_dialog, \
-    question_dialog
+    question_dialog, info_dialog
 from calibre.gui2.actions import InterfaceAction
 
 class LibraryUsageStats(object):
@@ -115,6 +115,14 @@ class ChooseLibraryAction(InterfaceAction):
                     type=Qt.QueuedConnection)
             self.choose_menu.addAction(ac)
 
+        self.rename_separator = self.choose_menu.addSeparator()
+
+        self.create_action(spec=(_('Library backup status...'), 'lt.png', None,
+            None), attr='action_backup_status')
+        self.action_backup_status.triggered.connect(self.backup_status,
+                type=Qt.QueuedConnection)
+        self.choose_menu.addAction(self.action_backup_status)
+
     def library_name(self):
         db = self.gui.library_view.model().db
         path = db.library_path
@@ -206,6 +214,17 @@ class ChooseLibraryAction(InterfaceAction):
         self.stats.remove(location)
         self.build_menus()
 
+    def backup_status(self, location):
+        dirty_text = 'no'
+        try:
+            print 'here'
+            dirty_text = \
+                  unicode(self.gui.library_view.model().db.dirty_queue_length())
+        except:
+            dirty_text = _('none')
+        info_dialog(self.gui, _('Backup status'), '<p>'+
+                _('Book metadata files remaining to be written: %s') % dirty_text,
+                show=True)
 
     def switch_requested(self, location):
         if not self.change_library_allowed():

src/calibre/gui2/dialogs/metadata_bulk.py

@@ -3,42 +3,109 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 
 '''Dialog to edit metadata in bulk'''
 
-from threading import Thread
+import re
-import re, string
 
-from PyQt4.Qt import Qt, QDialog, QGridLayout
+from PyQt4.Qt import Qt, QDialog, QGridLayout, QVBoxLayout, QFont, QLabel, \
+                     pyqtSignal
 from PyQt4 import QtGui
 
 from calibre.gui2.dialogs.metadata_bulk_ui import Ui_MetadataBulkDialog
 from calibre.gui2.dialogs.tag_editor import TagEditor
 from calibre.ebooks.metadata import string_to_authors, authors_to_string
 from calibre.gui2.custom_column_widgets import populate_metadata_page
-from calibre.gui2.dialogs.progress import BlockingBusy
+from calibre.gui2 import error_dialog
-from calibre.gui2 import error_dialog, Dispatcher
+from calibre.gui2.progress_indicator import ProgressIndicator
 from calibre.utils.config import dynamic
 
-class Worker(Thread):
+class MyBlockingBusy(QDialog):
+
+    do_one_signal = pyqtSignal()
+
+    phases = ['',
+              _('Title/Author'),
+              _('Standard metadata'),
+              _('Custom metadata'),
+              _('Search/Replace'),
+    ]
+
+    def __init__(self, msg, args, db, ids, cc_widgets, s_r_func,
+                 parent=None, window_title=_('Working')):
+        QDialog.__init__(self, parent)
+
+        self._layout = QVBoxLayout()
+        self.setLayout(self._layout)
+        self.msg_text = msg
+        self.msg = QLabel(msg+'        ') # Ensure dialog is wide enough
+        #self.msg.setWordWrap(True)
+        self.font = QFont()
+        self.font.setPointSize(self.font.pointSize() + 8)
+        self.msg.setFont(self.font)
+        self.pi = ProgressIndicator(self)
+        self.pi.setDisplaySize(100)
+        self._layout.addWidget(self.pi, 0, Qt.AlignHCenter)
+        self._layout.addSpacing(15)
+        self._layout.addWidget(self.msg, 0, Qt.AlignHCenter)
+        self.setWindowTitle(window_title)
+        self.resize(self.sizeHint())
+        self.start()
 
-    def __init__(self, args, db, ids, cc_widgets, callback):
-        Thread.__init__(self)
         self.args = args
         self.db = db
         self.ids = ids
         self.error = None
-        self.callback = callback
         self.cc_widgets = cc_widgets
+        self.s_r_func = s_r_func
+        self.do_one_signal.connect(self.do_one_safe, Qt.QueuedConnection)
 
-    def doit(self):
+    def start(self):
+        self.pi.startAnimation()
+
+    def stop(self):
+        self.pi.stopAnimation()
+
+    def accept(self):
+        self.stop()
+        return QDialog.accept(self)
+
+    def exec_(self):
+        self.current_index = 0
+        self.current_phase = 1
+        self.do_one_signal.emit()
+        return QDialog.exec_(self)
+
+    def do_one_safe(self):
+        try:
+            if self.current_index >= len(self.ids):
+                self.current_phase += 1
+                self.current_index = 0
+                if self.current_phase > 4:
+                    self.db.commit()
+                    return self.accept()
+            id = self.ids[self.current_index]
+            self.msg.setText(self.msg_text.format(self.phases[self.current_phase],
+                                        (self.current_index*100)/len(self.ids)))
+            self.do_one(id)
+        except Exception, err:
+            import traceback
+            try:
+                err = unicode(err)
+            except:
+                err = repr(err)
+            self.error = (err, traceback.format_exc())
+            return self.accept()
+
+    def do_one(self, id):
         remove, add, au, aus, do_aus, rating, pub, do_series, \
             do_autonumber, do_remove_format, remove_format, do_swap_ta, \
             do_remove_conv, do_auto_author, series, do_series_restart, \
             series_start_value, do_title_case, clear_series = self.args
 
+
         # first loop: do author and title. These will commit at the end of each
         # operation, because each operation modifies the file system. We want to
         # try hard to keep the DB and the file system in sync, even in the face
         # of exceptions or forced exits.
-        for id in self.ids:
+        if self.current_phase == 1:
             title_set = False
             if do_swap_ta:
                 title = self.db.title(id, index_is_id=True)
@@ -58,9 +125,8 @@ class Worker(Thread):
                 self.db.set_title(id, title.title(), notify=False)
             if au:
                 self.db.set_authors(id, string_to_authors(au), notify=False)
-
+        elif self.current_phase == 2:
-        # All of these just affect the DB, so we can tolerate a total rollback
+            # All of these just affect the DB, so we can tolerate a total rollback
-        for id in self.ids:
             if do_auto_author:
                 x = self.db.author_sort_from_book(id, index_is_id=True)
                 if x:
@@ -93,37 +159,19 @@ class Worker(Thread):
 
             if do_remove_conv:
                 self.db.delete_conversion_options(id, 'PIPE', commit=False)
-        self.db.commit()
+        elif self.current_phase == 3:
+            # both of these are fast enough to just do them all
+            for w in self.cc_widgets:
+                w.commit(self.ids)
+            self.db.bulk_modify_tags(self.ids, add=add, remove=remove,
+                                         notify=False)
+            self.current_index = len(self.ids)
+        elif self.current_phase == 4:
+            self.s_r_func(id)
+        # do the next one
+        self.current_index += 1
+        self.do_one_signal.emit()
 
-        for w in self.cc_widgets:
-            w.commit(self.ids)
-        self.db.bulk_modify_tags(self.ids, add=add, remove=remove,
-                notify=False)
-
-    def run(self):
-        try:
-            self.doit()
-        except Exception, err:
-            import traceback
-            try:
-                err = unicode(err)
-            except:
-                err = repr(err)
-            self.error = (err, traceback.format_exc())
-
-        self.callback()
-
-class SafeFormat(string.Formatter):
-    '''
-    Provides a format function that substitutes '' for any missing value
-    '''
-    def get_value(self, key, args, vals):
-        v = vals.get(key, None)
-        if v is None:
-            return ''
-        if isinstance(v, (tuple, list)):
-            v = ','.join(v)
-        return v
 
 class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
 
@@ -452,7 +500,7 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
                 self.s_r_set_colors()
                 break
 
-    def do_search_replace(self):
+    def do_search_replace(self, id):
         source = unicode(self.search_field.currentText())
         if not source or not self.s_r_obj:
             return
@@ -461,48 +509,45 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
             dest = source
         dfm = self.db.field_metadata[dest]
 
-        for id in self.ids:
+        mi = self.db.get_metadata(id, index_is_id=True,)
-            mi = self.db.get_metadata(id, index_is_id=True,)
+        val = mi.get(source)
-            val = mi.get(source)
+        if val is None:
-            if val is None:
+            return
-                continue
+        val = self.s_r_do_regexp(mi)
-            val = self.s_r_do_regexp(mi)
+        val = self.s_r_do_destination(mi, val)
-            val = self.s_r_do_destination(mi, val)
+        if dfm['is_multiple']:
-            if dfm['is_multiple']:
-                if dfm['is_custom']:
-                    # The standard tags and authors values want to be lists.
-                    # All custom columns are to be strings
-                    val = dfm['is_multiple'].join(val)
-                if dest == 'authors' and len(val) == 0:
-                    error_dialog(self, _('Search/replace invalid'),
-                                 _('Authors cannot be set to the empty string. '
-                                   'Book title %s not processed')%mi.title,
-                                 show=True)
-                    continue
-            else:
-                val = self.s_r_replace_mode_separator().join(val)
-                if dest == 'title' and len(val) == 0:
-                    error_dialog(self, _('Search/replace invalid'),
-                                 _('Title cannot be set to the empty string. '
-                                   'Book title %s not processed')%mi.title,
-                                 show=True)
-                    continue
-
             if dfm['is_custom']:
-                extra = self.db.get_custom_extra(id, label=dfm['label'], index_is_id=True)
+                # The standard tags and authors values want to be lists.
-                self.db.set_custom(id, val, label=dfm['label'], extra=extra,
+                # All custom columns are to be strings
-                                   commit=False)
+                val = dfm['is_multiple'].join(val)
+            if dest == 'authors' and len(val) == 0:
+                error_dialog(self, _('Search/replace invalid'),
+                             _('Authors cannot be set to the empty string. '
+                               'Book title %s not processed')%mi.title,
+                             show=True)
+                return
+        else:
+            val = self.s_r_replace_mode_separator().join(val)
+            if dest == 'title' and len(val) == 0:
+                error_dialog(self, _('Search/replace invalid'),
+                             _('Title cannot be set to the empty string. '
+                               'Book title %s not processed')%mi.title,
+                             show=True)
+                return
+
+        if dfm['is_custom']:
+            extra = self.db.get_custom_extra(id, label=dfm['label'], index_is_id=True)
+            self.db.set_custom(id, val, label=dfm['label'], extra=extra,
+                               commit=False)
+        else:
+            if dest == 'comments':
+                setter = self.db.set_comment
             else:
-                if dest == 'comments':
+                setter = getattr(self.db, 'set_'+dest)
-                    setter = self.db.set_comment
+            if dest in ['title', 'authors']:
-                else:
+                setter(id, val, notify=False)
-                    setter = getattr(self.db, 'set_'+dest)
+            else:
-                if dest in ['title', 'authors']:
+                setter(id, val, notify=False, commit=False)
-                    setter(id, val, notify=False)
-                else:
-                    setter(id, val, notify=False, commit=False)
-        self.db.commit()
-        dynamic['s_r_search_mode'] = self.search_mode.currentIndex()
 
     def create_custom_column_editors(self):
         w = self.central_widget.widget(1)
@@ -525,11 +570,11 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
 
     def initalize_authors(self):
         all_authors = self.db.all_authors()
-        all_authors.sort(cmp=lambda x, y : cmp(x[1], y[1]))
+        all_authors.sort(cmp=lambda x, y : cmp(x[1].lower(), y[1].lower()))
 
         for i in all_authors:
             id, name = i
-            name = authors_to_string([name.strip().replace('|', ',') for n in name.split(',')])
+            name = name.strip().replace('|', ',')
             self.authors.addItem(name)
         self.authors.setEditText('')
 
@@ -613,28 +658,32 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
                 do_remove_conv, do_auto_author, series, do_series_restart,
                 series_start_value, do_title_case, clear_series)
 
-        bb = BlockingBusy(_('Applying changes to %d books. This may take a while.')
+#        bb = BlockingBusy(_('Applying changes to %d books. This may take a while.')
-                %len(self.ids), parent=self)
+#                %len(self.ids), parent=self)
-        self.worker = Worker(args, self.db, self.ids,
+#        self.worker = Worker(args, self.db, self.ids,
+#                getattr(self, 'custom_column_widgets', []),
+#                Dispatcher(bb.accept, parent=bb))
+
+        bb = MyBlockingBusy(_('Applying changes to %d books.\nPhase {0} {1}%%.')
+                %len(self.ids), args, self.db, self.ids,
                 getattr(self, 'custom_column_widgets', []),
-                Dispatcher(bb.accept, parent=bb))
+                self.do_search_replace, parent=self)
 
         # The metadata backup thread causes database commits
         # which can slow down bulk editing of large numbers of books
         self.model.stop_metadata_backup()
         try:
-            self.worker.start()
+#            self.worker.start()
             bb.exec_()
         finally:
             self.model.start_metadata_backup()
 
-        if self.worker.error is not None:
+        if bb.error is not None:
             return error_dialog(self, _('Failed'),
-                    self.worker.error[0], det_msg=self.worker.error[1],
+                    bb.error[0], det_msg=bb.error[1],
                     show=True)
 
-        self.do_search_replace()
+        dynamic['s_r_search_mode'] = self.search_mode.currentIndex()
-
         self.db.clean()
         return QDialog.accept(self)
 

src/calibre/library/caches.py

@@ -138,25 +138,37 @@ class CoverCache(Thread): # {{{
     def run(self):
         while self.keep_running:
             try:
-                time.sleep(0.050) # Limit 20/second to not overwhelm the GUI
+                # The GUI puts the same ID into the queue many times. The code
+                # below emptys the queue, building a set of unique values. When
+                # the queue is empty, do the work
+                ids = set()
                 id_ = self.load_queue.get(True, 2)
+                ids.add(id_)
+                try:
+                    while True:
+                        # Give the gui some time to put values into the queue
+                        id_ = self.load_queue.get(True, 0.5)
+                        ids.add(id_)
+                except Empty:
+                    pass
             except Empty:
                 continue
             except:
                 #Happens during interpreter shutdown
                 break
-            try:
+            for id_ in ids:
-                img = self._image_for_id(id_)
+                time.sleep(0.050) # Limit 20/second to not overwhelm the GUI
-            except:
+                try:
-                import traceback
+                    img = self._image_for_id(id_)
-                traceback.print_exc()
+                except:
-                continue
+                    traceback.print_exc()
-            try:
+                    continue
-                with self.lock:
+                try:
-                    self.cache[id_] = img
+                    with self.lock:
-            except:
+                        self.cache[id_] = img
-                # Happens during interpreter shutdown
+                except:
-                break
+                    # Happens during interpreter shutdown
+                    break
 
     def set_cache(self, ids):
         with self.lock:

src/calibre/library/custom_columns.py

@@ -214,6 +214,7 @@ class CustomColumns(object):
             'SELECT id FROM %s WHERE value=?'%table, (new_name,), all=False)
         if new_id is None or old_id == new_id:
             self.conn.execute('UPDATE %s SET value=? WHERE id=?'%table, (new_name, old_id))
+            new_id = old_id
         else:
             # New id exists. If the column is_multiple, then process like
             # tags, otherwise process like publishers (see database2)
@@ -226,6 +227,7 @@ class CustomColumns(object):
             self.conn.execute('''UPDATE %s SET value=?
                                  WHERE value=?'''%lt, (new_id, old_id,))
             self.conn.execute('DELETE FROM %s WHERE id=?'%table, (old_id,))
+        self.dirty_books_referencing('#'+data['label'], new_id, commit=False)
         self.conn.commit()
 
     def delete_custom_item_using_id(self, id, label=None, num=None):

src/calibre/library/database2.py

@@ -47,13 +47,21 @@ def delete_file(path):
 
 def delete_tree(path, permanent=False):
     if permanent:
-        shutil.rmtree(path)
+        try:
+            # For completely mysterious reasons, sometimes a file is left open
+            # leading to access errors. If we get an exception, wait and hope
+            # that whatever has the file (the O/S?) lets go of it.
+            shutil.rmtree(path)
+        except:
+            traceback.print_exc()
+            time.sleep(1)
+            shutil.rmtree(path)
     else:
         try:
             if not permanent:
                 winshell.delete_file(path, silent=True, no_confirm=True)
         except:
-            shutil.rmtree(path)
+            delete_tree(path, permanent=True)
 
 copyfile = os.link if hasattr(os, 'link') else shutil.copyfile
 
@@ -520,6 +528,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             try:
                 f = open(path, 'rb')
             except (IOError, OSError):
+                try:
+                    f.close()
+                    print 'cover exception left file open!', path
+                except:
+                    pass
                 time.sleep(0.2)
                 f = open(path, 'rb')
             if as_image:
@@ -627,6 +640,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         if commit:
             self.conn.commit()
 
+    def dirty_queue_length(self):
+        return len(self.dirtied_cache)
+
     def commit_dirty_cache(self):
         '''
         Set the dirty indication for every book in the cache. The vast majority
@@ -1286,7 +1302,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                      val=mi.get(key),
                      extra=mi.get_extra(key),
                      label=user_mi[key]['label'], commit=False)
-        self.commit()
+        self.conn.commit()
         self.notify('metadata', [id])
 
     def authors_sort_strings(self, id, index_is_id=False):
@@ -1444,6 +1460,19 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
     # Convenience methods for tags_list_editor
     # Note: we generally do not need to refresh_ids because library_view will
     # refresh everything.
+
+    def dirty_books_referencing(self, field, id, commit=True):
+        # Get the list of books to dirty -- all books that reference the item
+        table = self.field_metadata[field]['table']
+        link = self.field_metadata[field]['link_column']
+        bks = self.conn.get(
+            'SELECT book from books_{0}_link WHERE {1}=?'.format(table, link),
+            (id,))
+        books = []
+        for (book_id,) in bks:
+            books.append(book_id)
+        self.dirtied(books, commit=commit)
+
     def get_tags_with_ids(self):
         result = self.conn.get('SELECT id,name FROM tags')
         if not result:
@@ -1460,6 +1489,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             # there is a change of case
             self.conn.execute('''UPDATE tags SET name=?
                                  WHERE id=?''', (new_name, old_id))
+            self.dirty_books_referencing('tags', new_id, commit=False)
+            new_id = old_id
         else:
             # It is possible that by renaming a tag, the tag will appear
             # twice on a book. This will throw an integrity error, aborting
@@ -1477,9 +1508,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                                  WHERE tag=?''',(new_id, old_id,))
             # Get rid of the no-longer used publisher
             self.conn.execute('DELETE FROM tags WHERE id=?', (old_id,))
+        self.dirty_books_referencing('tags', new_id, commit=False)
         self.conn.commit()
 
     def delete_tag_using_id(self, id):
+        self.dirty_books_referencing('tags', id, commit=False)
         self.conn.execute('DELETE FROM books_tags_link WHERE tag=?', (id,))
         self.conn.execute('DELETE FROM tags WHERE id=?', (id,))
         self.conn.commit()
@@ -1496,6 +1529,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                     '''SELECT id from series
                        WHERE name=?''', (new_name,), all=False)
         if new_id is None or old_id == new_id:
+            new_id = old_id
             self.conn.execute('UPDATE series SET name=? WHERE id=?',
                               (new_name, old_id))
         else:
@@ -1519,15 +1553,17 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                                      SET series_index=?
                                      WHERE id=?''',(index, book_id,))
                 index = index + 1
+        self.dirty_books_referencing('series', new_id, commit=False)
         self.conn.commit()
 
     def delete_series_using_id(self, id):
+        self.dirty_books_referencing('series', id, commit=False)
         books = self.conn.get('SELECT book from books_series_link WHERE series=?', (id,))
         self.conn.execute('DELETE FROM books_series_link WHERE series=?', (id,))
         self.conn.execute('DELETE FROM series WHERE id=?', (id,))
-        self.conn.commit()
         for (book_id,) in books:
             self.conn.execute('UPDATE books SET series_index=1.0 WHERE id=?', (book_id,))
+        self.conn.commit()
 
     def get_publishers_with_ids(self):
         result = self.conn.get('SELECT id,name FROM publishers')
@@ -1541,6 +1577,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                     '''SELECT id from publishers
                        WHERE name=?''', (new_name,), all=False)
         if new_id is None or old_id == new_id:
+            new_id = old_id
             # New name doesn't exist. Simply change the old name
             self.conn.execute('UPDATE publishers SET name=? WHERE id=?', \
                               (new_name, old_id))
@@ -1551,9 +1588,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                                  WHERE publisher=?''',(new_id, old_id,))
             # Get rid of the no-longer used publisher
             self.conn.execute('DELETE FROM publishers WHERE id=?', (old_id,))
+        self.dirty_books_referencing('publisher', new_id, commit=False)
         self.conn.commit()
 
     def delete_publisher_using_id(self, old_id):
+        self.dirty_books_referencing('publisher', id, commit=False)
         self.conn.execute('''DELETE FROM books_publishers_link
                              WHERE publisher=?''', (old_id,))
         self.conn.execute('DELETE FROM publishers WHERE id=?', (old_id,))
@@ -1634,6 +1673,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             # Now delete the old author from the DB
             bks = self.conn.get('SELECT book FROM books_authors_link WHERE author=?', (old_id,))
             self.conn.execute('DELETE FROM authors WHERE id=?', (old_id,))
+        self.dirtied(books, commit=False)
         self.conn.commit()
         # the authors are now changed, either by changing the author's name
         # or replacing the author in the list. Now must fix up the books.