diff --git a/src/calibre/gui2/actions/choose_library.py b/src/calibre/gui2/actions/choose_library.py index 79406da40c..d3045fecf4 100644 --- a/src/calibre/gui2/actions/choose_library.py +++ b/src/calibre/gui2/actions/choose_library.py @@ -14,7 +14,7 @@ from calibre import isbytestring from calibre.constants import filesystem_encoding from calibre.utils.config import prefs from calibre.gui2 import gprefs, warning_dialog, Dispatcher, error_dialog, \ - question_dialog + question_dialog, info_dialog from calibre.gui2.actions import InterfaceAction class LibraryUsageStats(object): @@ -115,6 +115,14 @@ class ChooseLibraryAction(InterfaceAction): type=Qt.QueuedConnection) self.choose_menu.addAction(ac) + self.rename_separator = self.choose_menu.addSeparator() + + self.create_action(spec=(_('Library backup status...'), 'lt.png', None, + None), attr='action_backup_status') + self.action_backup_status.triggered.connect(self.backup_status, + type=Qt.QueuedConnection) + self.choose_menu.addAction(self.action_backup_status) + def library_name(self): db = self.gui.library_view.model().db path = db.library_path @@ -206,6 +214,17 @@ class ChooseLibraryAction(InterfaceAction): self.stats.remove(location) self.build_menus() + def backup_status(self, location): + dirty_text = 'no' + try: + print 'here' + dirty_text = \ + unicode(self.gui.library_view.model().db.dirty_queue_length()) + except: + dirty_text = _('none') + info_dialog(self.gui, _('Backup status'), '

'+ + _('Book metadata files remaining to be written: %s') % dirty_text, + show=True) def switch_requested(self, location): if not self.change_library_allowed(): diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index b0ce0a1e6d..4fc85f2b30 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -3,42 +3,109 @@ __copyright__ = '2008, Kovid Goyal ' '''Dialog to edit metadata in bulk''' -from threading import Thread -import re, string +import re -from PyQt4.Qt import Qt, QDialog, QGridLayout +from PyQt4.Qt import Qt, QDialog, QGridLayout, QVBoxLayout, QFont, QLabel, \ + pyqtSignal from PyQt4 import QtGui from calibre.gui2.dialogs.metadata_bulk_ui import Ui_MetadataBulkDialog from calibre.gui2.dialogs.tag_editor import TagEditor from calibre.ebooks.metadata import string_to_authors, authors_to_string from calibre.gui2.custom_column_widgets import populate_metadata_page -from calibre.gui2.dialogs.progress import BlockingBusy -from calibre.gui2 import error_dialog, Dispatcher +from calibre.gui2 import error_dialog +from calibre.gui2.progress_indicator import ProgressIndicator from calibre.utils.config import dynamic -class Worker(Thread): +class MyBlockingBusy(QDialog): + + do_one_signal = pyqtSignal() + + phases = ['', + _('Title/Author'), + _('Standard metadata'), + _('Custom metadata'), + _('Search/Replace'), + ] + + def __init__(self, msg, args, db, ids, cc_widgets, s_r_func, + parent=None, window_title=_('Working')): + QDialog.__init__(self, parent) + + self._layout = QVBoxLayout() + self.setLayout(self._layout) + self.msg_text = msg + self.msg = QLabel(msg+' ') # Ensure dialog is wide enough + #self.msg.setWordWrap(True) + self.font = QFont() + self.font.setPointSize(self.font.pointSize() + 8) + self.msg.setFont(self.font) + self.pi = ProgressIndicator(self) + self.pi.setDisplaySize(100) + self._layout.addWidget(self.pi, 0, Qt.AlignHCenter) + self._layout.addSpacing(15) + self._layout.addWidget(self.msg, 0, Qt.AlignHCenter) + self.setWindowTitle(window_title) + self.resize(self.sizeHint()) + self.start() - def __init__(self, args, db, ids, cc_widgets, callback): - Thread.__init__(self) self.args = args self.db = db self.ids = ids self.error = None - self.callback = callback self.cc_widgets = cc_widgets + self.s_r_func = s_r_func + self.do_one_signal.connect(self.do_one_safe, Qt.QueuedConnection) - def doit(self): + def start(self): + self.pi.startAnimation() + + def stop(self): + self.pi.stopAnimation() + + def accept(self): + self.stop() + return QDialog.accept(self) + + def exec_(self): + self.current_index = 0 + self.current_phase = 1 + self.do_one_signal.emit() + return QDialog.exec_(self) + + def do_one_safe(self): + try: + if self.current_index >= len(self.ids): + self.current_phase += 1 + self.current_index = 0 + if self.current_phase > 4: + self.db.commit() + return self.accept() + id = self.ids[self.current_index] + self.msg.setText(self.msg_text.format(self.phases[self.current_phase], + (self.current_index*100)/len(self.ids))) + self.do_one(id) + except Exception, err: + import traceback + try: + err = unicode(err) + except: + err = repr(err) + self.error = (err, traceback.format_exc()) + return self.accept() + + def do_one(self, id): remove, add, au, aus, do_aus, rating, pub, do_series, \ do_autonumber, do_remove_format, remove_format, do_swap_ta, \ do_remove_conv, do_auto_author, series, do_series_restart, \ series_start_value, do_title_case, clear_series = self.args + # first loop: do author and title. These will commit at the end of each # operation, because each operation modifies the file system. We want to # try hard to keep the DB and the file system in sync, even in the face # of exceptions or forced exits. - for id in self.ids: + if self.current_phase == 1: title_set = False if do_swap_ta: title = self.db.title(id, index_is_id=True) @@ -58,9 +125,8 @@ class Worker(Thread): self.db.set_title(id, title.title(), notify=False) if au: self.db.set_authors(id, string_to_authors(au), notify=False) - - # All of these just affect the DB, so we can tolerate a total rollback - for id in self.ids: + elif self.current_phase == 2: + # All of these just affect the DB, so we can tolerate a total rollback if do_auto_author: x = self.db.author_sort_from_book(id, index_is_id=True) if x: @@ -93,37 +159,19 @@ class Worker(Thread): if do_remove_conv: self.db.delete_conversion_options(id, 'PIPE', commit=False) - self.db.commit() + elif self.current_phase == 3: + # both of these are fast enough to just do them all + for w in self.cc_widgets: + w.commit(self.ids) + self.db.bulk_modify_tags(self.ids, add=add, remove=remove, + notify=False) + self.current_index = len(self.ids) + elif self.current_phase == 4: + self.s_r_func(id) + # do the next one + self.current_index += 1 + self.do_one_signal.emit() - for w in self.cc_widgets: - w.commit(self.ids) - self.db.bulk_modify_tags(self.ids, add=add, remove=remove, - notify=False) - - def run(self): - try: - self.doit() - except Exception, err: - import traceback - try: - err = unicode(err) - except: - err = repr(err) - self.error = (err, traceback.format_exc()) - - self.callback() - -class SafeFormat(string.Formatter): - ''' - Provides a format function that substitutes '' for any missing value - ''' - def get_value(self, key, args, vals): - v = vals.get(key, None) - if v is None: - return '' - if isinstance(v, (tuple, list)): - v = ','.join(v) - return v class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog): @@ -452,7 +500,7 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog): self.s_r_set_colors() break - def do_search_replace(self): + def do_search_replace(self, id): source = unicode(self.search_field.currentText()) if not source or not self.s_r_obj: return @@ -461,48 +509,45 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog): dest = source dfm = self.db.field_metadata[dest] - for id in self.ids: - mi = self.db.get_metadata(id, index_is_id=True,) - val = mi.get(source) - if val is None: - continue - val = self.s_r_do_regexp(mi) - val = self.s_r_do_destination(mi, val) - if dfm['is_multiple']: - if dfm['is_custom']: - # The standard tags and authors values want to be lists. - # All custom columns are to be strings - val = dfm['is_multiple'].join(val) - if dest == 'authors' and len(val) == 0: - error_dialog(self, _('Search/replace invalid'), - _('Authors cannot be set to the empty string. ' - 'Book title %s not processed')%mi.title, - show=True) - continue - else: - val = self.s_r_replace_mode_separator().join(val) - if dest == 'title' and len(val) == 0: - error_dialog(self, _('Search/replace invalid'), - _('Title cannot be set to the empty string. ' - 'Book title %s not processed')%mi.title, - show=True) - continue - + mi = self.db.get_metadata(id, index_is_id=True,) + val = mi.get(source) + if val is None: + return + val = self.s_r_do_regexp(mi) + val = self.s_r_do_destination(mi, val) + if dfm['is_multiple']: if dfm['is_custom']: - extra = self.db.get_custom_extra(id, label=dfm['label'], index_is_id=True) - self.db.set_custom(id, val, label=dfm['label'], extra=extra, - commit=False) + # The standard tags and authors values want to be lists. + # All custom columns are to be strings + val = dfm['is_multiple'].join(val) + if dest == 'authors' and len(val) == 0: + error_dialog(self, _('Search/replace invalid'), + _('Authors cannot be set to the empty string. ' + 'Book title %s not processed')%mi.title, + show=True) + return + else: + val = self.s_r_replace_mode_separator().join(val) + if dest == 'title' and len(val) == 0: + error_dialog(self, _('Search/replace invalid'), + _('Title cannot be set to the empty string. ' + 'Book title %s not processed')%mi.title, + show=True) + return + + if dfm['is_custom']: + extra = self.db.get_custom_extra(id, label=dfm['label'], index_is_id=True) + self.db.set_custom(id, val, label=dfm['label'], extra=extra, + commit=False) + else: + if dest == 'comments': + setter = self.db.set_comment else: - if dest == 'comments': - setter = self.db.set_comment - else: - setter = getattr(self.db, 'set_'+dest) - if dest in ['title', 'authors']: - setter(id, val, notify=False) - else: - setter(id, val, notify=False, commit=False) - self.db.commit() - dynamic['s_r_search_mode'] = self.search_mode.currentIndex() + setter = getattr(self.db, 'set_'+dest) + if dest in ['title', 'authors']: + setter(id, val, notify=False) + else: + setter(id, val, notify=False, commit=False) def create_custom_column_editors(self): w = self.central_widget.widget(1) @@ -525,11 +570,11 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog): def initalize_authors(self): all_authors = self.db.all_authors() - all_authors.sort(cmp=lambda x, y : cmp(x[1], y[1])) + all_authors.sort(cmp=lambda x, y : cmp(x[1].lower(), y[1].lower())) for i in all_authors: id, name = i - name = authors_to_string([name.strip().replace('|', ',') for n in name.split(',')]) + name = name.strip().replace('|', ',') self.authors.addItem(name) self.authors.setEditText('') @@ -613,28 +658,32 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog): do_remove_conv, do_auto_author, series, do_series_restart, series_start_value, do_title_case, clear_series) - bb = BlockingBusy(_('Applying changes to %d books. This may take a while.') - %len(self.ids), parent=self) - self.worker = Worker(args, self.db, self.ids, +# bb = BlockingBusy(_('Applying changes to %d books. This may take a while.') +# %len(self.ids), parent=self) +# self.worker = Worker(args, self.db, self.ids, +# getattr(self, 'custom_column_widgets', []), +# Dispatcher(bb.accept, parent=bb)) + + bb = MyBlockingBusy(_('Applying changes to %d books.\nPhase {0} {1}%%.') + %len(self.ids), args, self.db, self.ids, getattr(self, 'custom_column_widgets', []), - Dispatcher(bb.accept, parent=bb)) + self.do_search_replace, parent=self) # The metadata backup thread causes database commits # which can slow down bulk editing of large numbers of books self.model.stop_metadata_backup() try: - self.worker.start() +# self.worker.start() bb.exec_() finally: self.model.start_metadata_backup() - if self.worker.error is not None: + if bb.error is not None: return error_dialog(self, _('Failed'), - self.worker.error[0], det_msg=self.worker.error[1], + bb.error[0], det_msg=bb.error[1], show=True) - self.do_search_replace() - + dynamic['s_r_search_mode'] = self.search_mode.currentIndex() self.db.clean() return QDialog.accept(self) diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 281d1485b7..a36dbe57a9 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -138,25 +138,37 @@ class CoverCache(Thread): # {{{ def run(self): while self.keep_running: try: - time.sleep(0.050) # Limit 20/second to not overwhelm the GUI + # The GUI puts the same ID into the queue many times. The code + # below emptys the queue, building a set of unique values. When + # the queue is empty, do the work + ids = set() id_ = self.load_queue.get(True, 2) + ids.add(id_) + try: + while True: + # Give the gui some time to put values into the queue + id_ = self.load_queue.get(True, 0.5) + ids.add(id_) + except Empty: + pass except Empty: continue except: #Happens during interpreter shutdown break - try: - img = self._image_for_id(id_) - except: - import traceback - traceback.print_exc() - continue - try: - with self.lock: - self.cache[id_] = img - except: - # Happens during interpreter shutdown - break + for id_ in ids: + time.sleep(0.050) # Limit 20/second to not overwhelm the GUI + try: + img = self._image_for_id(id_) + except: + traceback.print_exc() + continue + try: + with self.lock: + self.cache[id_] = img + except: + # Happens during interpreter shutdown + break def set_cache(self, ids): with self.lock: diff --git a/src/calibre/library/custom_columns.py b/src/calibre/library/custom_columns.py index 97c8565177..fdd78e89f8 100644 --- a/src/calibre/library/custom_columns.py +++ b/src/calibre/library/custom_columns.py @@ -214,6 +214,7 @@ class CustomColumns(object): 'SELECT id FROM %s WHERE value=?'%table, (new_name,), all=False) if new_id is None or old_id == new_id: self.conn.execute('UPDATE %s SET value=? WHERE id=?'%table, (new_name, old_id)) + new_id = old_id else: # New id exists. If the column is_multiple, then process like # tags, otherwise process like publishers (see database2) @@ -226,6 +227,7 @@ class CustomColumns(object): self.conn.execute('''UPDATE %s SET value=? WHERE value=?'''%lt, (new_id, old_id,)) self.conn.execute('DELETE FROM %s WHERE id=?'%table, (old_id,)) + self.dirty_books_referencing('#'+data['label'], new_id, commit=False) self.conn.commit() def delete_custom_item_using_id(self, id, label=None, num=None): diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 192de21df3..85fb955448 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -47,13 +47,21 @@ def delete_file(path): def delete_tree(path, permanent=False): if permanent: - shutil.rmtree(path) + try: + # For completely mysterious reasons, sometimes a file is left open + # leading to access errors. If we get an exception, wait and hope + # that whatever has the file (the O/S?) lets go of it. + shutil.rmtree(path) + except: + traceback.print_exc() + time.sleep(1) + shutil.rmtree(path) else: try: if not permanent: winshell.delete_file(path, silent=True, no_confirm=True) except: - shutil.rmtree(path) + delete_tree(path, permanent=True) copyfile = os.link if hasattr(os, 'link') else shutil.copyfile @@ -520,6 +528,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): try: f = open(path, 'rb') except (IOError, OSError): + try: + f.close() + print 'cover exception left file open!', path + except: + pass time.sleep(0.2) f = open(path, 'rb') if as_image: @@ -627,6 +640,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if commit: self.conn.commit() + def dirty_queue_length(self): + return len(self.dirtied_cache) + def commit_dirty_cache(self): ''' Set the dirty indication for every book in the cache. The vast majority @@ -1286,7 +1302,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): val=mi.get(key), extra=mi.get_extra(key), label=user_mi[key]['label'], commit=False) - self.commit() + self.conn.commit() self.notify('metadata', [id]) def authors_sort_strings(self, id, index_is_id=False): @@ -1444,6 +1460,19 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): # Convenience methods for tags_list_editor # Note: we generally do not need to refresh_ids because library_view will # refresh everything. + + def dirty_books_referencing(self, field, id, commit=True): + # Get the list of books to dirty -- all books that reference the item + table = self.field_metadata[field]['table'] + link = self.field_metadata[field]['link_column'] + bks = self.conn.get( + 'SELECT book from books_{0}_link WHERE {1}=?'.format(table, link), + (id,)) + books = [] + for (book_id,) in bks: + books.append(book_id) + self.dirtied(books, commit=commit) + def get_tags_with_ids(self): result = self.conn.get('SELECT id,name FROM tags') if not result: @@ -1460,6 +1489,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): # there is a change of case self.conn.execute('''UPDATE tags SET name=? WHERE id=?''', (new_name, old_id)) + self.dirty_books_referencing('tags', new_id, commit=False) + new_id = old_id else: # It is possible that by renaming a tag, the tag will appear # twice on a book. This will throw an integrity error, aborting @@ -1477,9 +1508,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): WHERE tag=?''',(new_id, old_id,)) # Get rid of the no-longer used publisher self.conn.execute('DELETE FROM tags WHERE id=?', (old_id,)) + self.dirty_books_referencing('tags', new_id, commit=False) self.conn.commit() def delete_tag_using_id(self, id): + self.dirty_books_referencing('tags', id, commit=False) self.conn.execute('DELETE FROM books_tags_link WHERE tag=?', (id,)) self.conn.execute('DELETE FROM tags WHERE id=?', (id,)) self.conn.commit() @@ -1496,6 +1529,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): '''SELECT id from series WHERE name=?''', (new_name,), all=False) if new_id is None or old_id == new_id: + new_id = old_id self.conn.execute('UPDATE series SET name=? WHERE id=?', (new_name, old_id)) else: @@ -1519,15 +1553,17 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): SET series_index=? WHERE id=?''',(index, book_id,)) index = index + 1 + self.dirty_books_referencing('series', new_id, commit=False) self.conn.commit() def delete_series_using_id(self, id): + self.dirty_books_referencing('series', id, commit=False) books = self.conn.get('SELECT book from books_series_link WHERE series=?', (id,)) self.conn.execute('DELETE FROM books_series_link WHERE series=?', (id,)) self.conn.execute('DELETE FROM series WHERE id=?', (id,)) - self.conn.commit() for (book_id,) in books: self.conn.execute('UPDATE books SET series_index=1.0 WHERE id=?', (book_id,)) + self.conn.commit() def get_publishers_with_ids(self): result = self.conn.get('SELECT id,name FROM publishers') @@ -1541,6 +1577,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): '''SELECT id from publishers WHERE name=?''', (new_name,), all=False) if new_id is None or old_id == new_id: + new_id = old_id # New name doesn't exist. Simply change the old name self.conn.execute('UPDATE publishers SET name=? WHERE id=?', \ (new_name, old_id)) @@ -1551,9 +1588,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): WHERE publisher=?''',(new_id, old_id,)) # Get rid of the no-longer used publisher self.conn.execute('DELETE FROM publishers WHERE id=?', (old_id,)) + self.dirty_books_referencing('publisher', new_id, commit=False) self.conn.commit() def delete_publisher_using_id(self, old_id): + self.dirty_books_referencing('publisher', id, commit=False) self.conn.execute('''DELETE FROM books_publishers_link WHERE publisher=?''', (old_id,)) self.conn.execute('DELETE FROM publishers WHERE id=?', (old_id,)) @@ -1634,6 +1673,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): # Now delete the old author from the DB bks = self.conn.get('SELECT book FROM books_authors_link WHERE author=?', (old_id,)) self.conn.execute('DELETE FROM authors WHERE id=?', (old_id,)) + self.dirtied(books, commit=False) self.conn.commit() # the authors are now changed, either by changing the author's name # or replacing the author in the list. Now must fix up the books.