Speedup bulk metadata editing of remaining standard fields

src/calibre/gui2/dialogs/metadata_bulk.py

@@ -31,6 +31,10 @@ class Worker(Thread):
             do_autonumber, do_remove_format, remove_format, do_swap_ta, \
             do_remove_conv, do_auto_author, series = self.args
 
+        # first loop: do author and title. These will commit at the end of each
+        # operation, because each operation modifies the file system. We want to
+        # try hard to keep the DB and the file system in sync, even in the face
+        # of exceptions or forced exits.
         for id in self.ids:
             if do_swap_ta:
                 title = self.db.title(id, index_is_id=True)
@@ -46,31 +50,34 @@ class Worker(Thread):
             if au:
                 self.db.set_authors(id, string_to_authors(au), notify=False)
 
+        # All of these just affect the DB, so we can tolerate a total rollback
+        for id in self.ids:
             if do_auto_author:
                 x = self.db.author_sort_from_book(id, index_is_id=True)
                 if x:
-                    self.db.set_author_sort(id, x, notify=False)
+                    self.db.set_author_sort(id, x, notify=False, commit=False)
 
             if aus and do_aus:
-                self.db.set_author_sort(id, aus, notify=False)
+                self.db.set_author_sort(id, aus, notify=False, commit=False)
 
             if rating != -1:
-                self.db.set_rating(id, 2*rating, notify=False)
+                self.db.set_rating(id, 2*rating, notify=False, commit=False)
 
             if pub:
-                self.db.set_publisher(id, pub, notify=False)
+                self.db.set_publisher(id, pub, notify=False, commit=False)
 
             if do_series:
                 next = self.db.get_next_series_num_for(series)
-                self.db.set_series(id, series, notify=False)
+                self.db.set_series(id, series, notify=False, commit=False)
                 num = next if do_autonumber and series else 1.0
-                self.db.set_series_index(id, num, notify=False)
+                self.db.set_series_index(id, num, notify=False, commit=False)
 
             if do_remove_format:
-                self.db.remove_format(id, remove_format, index_is_id=True, notify=False)
+                self.db.remove_format(id, remove_format, index_is_id=True, notify=False, commit=False)
 
             if do_remove_conv:
-                self.db.delete_conversion_options(id, 'PIPE')
+                self.db.delete_conversion_options(id, 'PIPE', commit=False)
+        self.db.commit()
 
         for w in self.cc_widgets:
             w.commit(self.ids)
@@ -224,6 +231,11 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
             return error_dialog(self, _('Failed'),
                     self.worker.error[0], det_msg=self.worker.error[1],
                     show=True)
+
+        for w in getattr(self, 'custom_column_widgets', []):
+            w.commit(self.ids)
+        self.db.clean()
+
         return QDialog.accept(self)
 
 

src/calibre/library/database.py

@@ -1080,9 +1080,11 @@ ALTER TABLE books ADD COLUMN isbn TEXT DEFAULT "" COLLATE NOCASE;
             return cPickle.loads(str(data))
         return None
 
-    def delete_conversion_options(self, id, format):
+    def delete_conversion_options(self, id, format, commit=True):
         self.conn.execute('DELETE FROM conversion_options WHERE book=? AND format=?',
                 (id, format.upper()))
+        if commit:
+            self.conn.commit()
 
 
     def add_format(self, index, ext, stream, index_is_id=False):

src/calibre/library/database2.py

@@ -389,7 +389,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             path = path.lower()
         return path
 
-    def set_path(self, index, index_is_id=False):
+    def set_path(self, index, index_is_id=False, commit=True):
         '''
         Set the path to the directory containing this books files based on its
         current title and author. If there was a previous directory, its contents
@@ -428,7 +428,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                 stream = cStringIO.StringIO(f)
                 self.add_format(id, format, stream, index_is_id=True, path=tpath)
         self.conn.execute('UPDATE books SET path=? WHERE id=?', (path, id))
-        self.conn.commit()
+        if commit:
+            self.conn.commit()
         self.data.set(id, self.FIELD_MAP['path'], path, row_is_id=True)
         # Delete not needed directories
         if current_path and os.path.exists(spath):
@@ -729,7 +730,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         if notify:
             self.notify('delete', [id])
 
-    def remove_format(self, index, format, index_is_id=False, notify=True):
+    def remove_format(self, index, format, index_is_id=False, notify=True, commit=True):
         id = index if index_is_id else self.id(index)
         name = self.conn.get('SELECT name FROM data WHERE book=? AND format=?', (id, format), all=False)
         if name:
@@ -739,7 +740,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             except:
                 traceback.print_exc()
             self.conn.execute('DELETE FROM data WHERE book=? AND format=?', (id, format.upper()))
-            self.conn.commit()
+            if commit:
+                self.conn.commit()
             self.refresh_ids([id])
             if notify:
                 self.notify('metadata', [id])
@@ -1016,7 +1018,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         if mi.series:
             doit(self.set_series, id, mi.series, notify=False)
         if mi.cover_data[1] is not None:
-            doit(self.set_cover, id, mi.cover_data[1])
+            doit(self.set_cover, id, mi.cover_data[1]) # doesn't use commit
         elif mi.cover is not None and os.access(mi.cover, os.R_OK):
             doit(self.set_cover, id, open(mi.cover, 'rb'))
         if mi.tags:
@@ -1092,7 +1094,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                                    (id, aid))
             except IntegrityError: # Sometimes books specify the same author twice in their metadata
                 pass
-        self.conn.commit()
         ss = self.author_sort_from_book(id, index_is_id=True)
         self.conn.execute('UPDATE books SET author_sort=? WHERE id=?',
                           (ss, id))
@@ -1121,24 +1122,26 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         if notify:
             self.notify('metadata', [id])
 
-    def set_timestamp(self, id, dt, notify=True):
+    def set_timestamp(self, id, dt, notify=True, commit=True):
         if dt:
             self.conn.execute('UPDATE books SET timestamp=? WHERE id=?', (dt, id))
             self.data.set(id, self.FIELD_MAP['timestamp'], dt, row_is_id=True)
-            self.conn.commit()
+            if commit:
+                self.conn.commit()
             if notify:
                 self.notify('metadata', [id])
 
-    def set_pubdate(self, id, dt, notify=True):
+    def set_pubdate(self, id, dt, notify=True, commit=True):
         if dt:
             self.conn.execute('UPDATE books SET pubdate=? WHERE id=?', (dt, id))
             self.data.set(id, self.FIELD_MAP['pubdate'], dt, row_is_id=True)
-            self.conn.commit()
+            if commit:
+                self.conn.commit()
             if notify:
                 self.notify('metadata', [id])
 
 
-    def set_publisher(self, id, publisher, notify=True):
+    def set_publisher(self, id, publisher, notify=True, commit=True):
         self.conn.execute('DELETE FROM books_publishers_link WHERE book=?',(id,))
         self.conn.execute('DELETE FROM publishers WHERE (SELECT COUNT(id) FROM books_publishers_link WHERE publisher=publishers.id) < 1')
         if publisher:
@@ -1150,7 +1153,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             else:
                 aid = self.conn.execute('INSERT INTO publishers(name) VALUES (?)', (publisher,)).lastrowid
             self.conn.execute('INSERT INTO books_publishers_link(book, publisher) VALUES (?,?)', (id, aid))
-            self.conn.commit()
+            if commit:
+                self.conn.commit()
             self.data.set(id, self.FIELD_MAP['publisher'], publisher, row_is_id=True)
             if notify:
                 self.notify('metadata', [id])
@@ -1447,7 +1451,10 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         if notify:
             self.notify('metadata', ids)
 
-    def set_tags(self, id, tags, append=False, notify=True):
+    def commit(self):
+        self.conn.commit()
+
+    def set_tags(self, id, tags, append=False, notify=True, commit=True):
         '''
         @param tags: list of strings
         @param append: If True existing tags are not removed
@@ -1481,7 +1488,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                                         (id, tid), all=False):
                 self.conn.execute('INSERT INTO books_tags_link(book, tag) VALUES (?,?)',
                               (id, tid))
-        self.conn.commit()
+        if commit:
+            self.conn.commit()
         tags = u','.join(self.get_tags(id))
         self.data.set(id, self.FIELD_MAP['tags'], tags, row_is_id=True)
         if notify:
@@ -1520,7 +1528,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                 self.conn.execute('DELETE FROM tags WHERE id=?', (id,))
                 self.conn.commit()
 
-    def set_series(self, id, series, notify=True):
+    def set_series(self, id, series, notify=True, commit=True):
         self.conn.execute('DELETE FROM books_series_link WHERE book=?',(id,))
         self.conn.execute('DELETE FROM series WHERE (SELECT COUNT(id) FROM books_series_link WHERE series=series.id) < 1')
         if series:
@@ -1534,12 +1542,13 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             else:
                 aid = self.conn.execute('INSERT INTO series(name) VALUES (?)', (series,)).lastrowid
             self.conn.execute('INSERT INTO books_series_link(book, series) VALUES (?,?)', (id, aid))
-        self.conn.commit()
+        if commit:
+            self.conn.commit()
         self.data.set(id, self.FIELD_MAP['series'], series, row_is_id=True)
         if notify:
             self.notify('metadata', [id])
 
-    def set_series_index(self, id, idx, notify=True):
+    def set_series_index(self, id, idx, notify=True, commit=True):
         if idx is None:
             idx = 1.0
         try:
@@ -1547,40 +1556,45 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         except:
             idx = 1.0
         self.conn.execute('UPDATE books SET series_index=? WHERE id=?', (idx, id))
-        self.conn.commit()
+        if commit:
+            self.conn.commit()
         self.data.set(id, self.FIELD_MAP['series_index'], idx, row_is_id=True)
         if notify:
             self.notify('metadata', [id])
 
-    def set_rating(self, id, rating, notify=True):
+    def set_rating(self, id, rating, notify=True, commit=True):
         rating = int(rating)
         self.conn.execute('DELETE FROM books_ratings_link WHERE book=?',(id,))
         rat = self.conn.get('SELECT id FROM ratings WHERE rating=?', (rating,), all=False)
         rat = rat if rat else self.conn.execute('INSERT INTO ratings(rating) VALUES (?)', (rating,)).lastrowid
         self.conn.execute('INSERT INTO books_ratings_link(book, rating) VALUES (?,?)', (id, rat))
-        self.conn.commit()
+        if commit:
+            self.conn.commit()
         self.data.set(id, self.FIELD_MAP['rating'], rating, row_is_id=True)
         if notify:
             self.notify('metadata', [id])
 
-    def set_comment(self, id, text, notify=True):
+    def set_comment(self, id, text, notify=True, commit=True):
         self.conn.execute('DELETE FROM comments WHERE book=?', (id,))
         self.conn.execute('INSERT INTO comments(book,text) VALUES (?,?)', (id, text))
-        self.conn.commit()
+        if commit:
+            self.conn.commit()
         self.data.set(id, self.FIELD_MAP['comments'], text, row_is_id=True)
         if notify:
             self.notify('metadata', [id])
 
-    def set_author_sort(self, id, sort, notify=True):
+    def set_author_sort(self, id, sort, notify=True, commit=True):
         self.conn.execute('UPDATE books SET author_sort=? WHERE id=?', (sort, id))
-        self.conn.commit()
+        if commit:
+            self.conn.commit()
         self.data.set(id, self.FIELD_MAP['author_sort'], sort, row_is_id=True)
         if notify:
             self.notify('metadata', [id])
 
-    def set_isbn(self, id, isbn, notify=True):
+    def set_isbn(self, id, isbn, notify=True, commit=True):
         self.conn.execute('UPDATE books SET isbn=? WHERE id=?', (isbn, id))
-        self.conn.commit()
+        if commit:
+            self.conn.commit()
         self.data.set(id, self.FIELD_MAP['isbn'], isbn, row_is_id=True)
         if notify:
             self.notify('metadata', [id])