From 67ebd98cf1e56d1f20b01c70a90823f44f8f933f Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Fri, 8 Oct 2010 08:47:57 +0100 Subject: [PATCH 1/9] Correct order of assignment in set_books_in_library. --- src/calibre/gui2/device.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index 01f9347f67..0344a8f21d 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -1468,21 +1468,21 @@ class DeviceMixin(object): # {{{ # will match if any of the db_id, author, or author_sort # also match. if getattr(book, 'application_id', None) in d['db_ids']: - book.in_library = True # app_id already matches a db_id. No need to set it. if update_metadata: book.smart_update(d['db_ids'][book.application_id], replace_metadata=True) + book.in_library = True continue # Sonys know their db_id independent of the application_id # in the metadata cache. Check that as well. if getattr(book, 'db_id', None) in d['db_ids']: - book.in_library = True - book.application_id = \ - d['db_ids'][book.db_id].application_id if update_metadata: book.smart_update(d['db_ids'][book.db_id], replace_metadata=True) + book.in_library = True + book.application_id = \ + d['db_ids'][book.db_id].application_id continue # We now know that the application_id is not right. Set it # to None to prevent book_on_device from accidentally @@ -1494,19 +1494,19 @@ class DeviceMixin(object): # {{{ # either can appear as the author book_authors = clean_string(authors_to_string(book.authors)) if book_authors in d['authors']: - book.in_library = True - book.application_id = \ - d['authors'][book_authors].application_id if update_metadata: book.smart_update(d['authors'][book_authors], replace_metadata=True) - elif book_authors in d['author_sort']: book.in_library = True book.application_id = \ - d['author_sort'][book_authors].application_id + d['authors'][book_authors].application_id + elif book_authors in d['author_sort']: if update_metadata: book.smart_update(d['author_sort'][book_authors], replace_metadata=True) + book.in_library = True + book.application_id = \ + d['author_sort'][book_authors].application_id else: # Book definitely not matched. Clear its application ID book.application_id = None From 61ba29ab615f069f998ca1bd9bd465a4f063a922 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Fri, 8 Oct 2010 09:48:32 +0100 Subject: [PATCH 2/9] Fix regression that caused title_sort to stop working --- src/calibre/ebooks/metadata/book/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py index 0d08218790..3b96c98a7b 100644 --- a/src/calibre/ebooks/metadata/book/base.py +++ b/src/calibre/ebooks/metadata/book/base.py @@ -38,7 +38,8 @@ class SafeFormat(TemplateFormatter): def get_value(self, key, args, kwargs): try: - key = field_metadata.search_term_to_field_key(key.lower()) + if key != 'title_sort': + key = field_metadata.search_term_to_field_key(key.lower()) b = self.book.get_user_metadata(key, False) if b and b['datatype'] == 'int' and self.book.get(key, 0) == 0: v = '' From 56f079a08e96cfa7b5cc3f06d5bb39fe0e3b45ac Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Fri, 8 Oct 2010 09:52:34 +0100 Subject: [PATCH 3/9] Change add/delete to delete/add, just in case the add ends up creating another of what is being deleted. --- src/calibre/gui2/tag_view.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py index a054bb0645..88a9220024 100644 --- a/src/calibre/gui2/tag_view.py +++ b/src/calibre/gui2/tag_view.py @@ -836,11 +836,11 @@ class TagBrowserMixin(object): # {{{ rename_func = partial(db.rename_custom_item, label=cc_label) delete_func = partial(db.delete_custom_item_using_id, label=cc_label) if rename_func: + for item in to_delete: + delete_func(item) for text in to_rename: for old_id in to_rename[text]: rename_func(old_id, new_name=unicode(text)) - for item in to_delete: - delete_func(item) # Clean up everything, as information could have changed for many books. self.library_view.model().refresh() From 2793c31ac7673227f6691ddb6b0766d605ceae45 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Fri, 8 Oct 2010 09:52:57 +0100 Subject: [PATCH 4/9] Make rename_tags handle splitting tags (1 -> many) --- src/calibre/library/database2.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 6666af8a8c..a748a8c45b 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -1485,7 +1485,18 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): return result def rename_tag(self, old_id, new_name): - new_name = new_name.strip() + # It is possible that new_name is in fact a set of names. Split it on + # comma to find out. If it is, then rename the first one and append the + # rest + new_names = [t.strip() for t in new_name.strip().split(',') if t.strip()] + new_name = new_names[0] + new_names = new_names[1:] + + # get the list of books that reference the tag being changed + books = self.conn.get('''SELECT book from books_tags_link + WHERE tag=?''', (old_id,)) + books = [b[0] for b in books] + new_id = self.conn.get( '''SELECT id from tags WHERE name=?''', (new_name,), all=False) @@ -1501,9 +1512,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): # all the changes. To get around this, we first delete any links # to the new_id from books referencing the old_id, so that # renaming old_id to new_id will be unique on the book - books = self.conn.get('''SELECT book from books_tags_link - WHERE tag=?''', (old_id,)) - for (book_id,) in books: + for book_id in books: self.conn.execute('''DELETE FROM books_tags_link WHERE book=? and tag=?''', (book_id, new_id)) @@ -1512,7 +1521,13 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): WHERE tag=?''',(new_id, old_id,)) # Get rid of the no-longer used publisher self.conn.execute('DELETE FROM tags WHERE id=?', (old_id,)) - self.dirty_books_referencing('tags', new_id, commit=False) + + if new_names: + # have some left-over names to process. Add them to the book. + for book_id in books: + self.set_tags(book_id, new_names, append=True, notify=False, + commit=False) + self.dirtied(books, commit=False) self.conn.commit() def delete_tag_using_id(self, id): From 8203f5335427a76a90b9afdb3fae352d222f1117 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Fri, 8 Oct 2010 09:58:17 +0100 Subject: [PATCH 5/9] Finish list_categories (forgot to implement the -r option) --- src/calibre/library/cli.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/calibre/library/cli.py b/src/calibre/library/cli.py index a11d81cc8c..63361d87f5 100644 --- a/src/calibre/library/cli.py +++ b/src/calibre/library/cli.py @@ -1025,7 +1025,7 @@ information is the equivalent of what is shown in the tags pane. parser.add_option('-q', '--quote', default='"', help=_('The character to put around the category value in CSV mode. ' 'Default is quotes (").')) - parser.add_option('-r', '--categories', default=None, dest='report', + parser.add_option('-r', '--categories', default='', dest='report', help=_("Comma-separated list of category lookup names.\n" "Default: all")) parser.add_option('-w', '--line-width', default=-1, type=int, @@ -1052,8 +1052,10 @@ def command_list_categories(args, dbpath): db = LibraryDatabase2(dbpath) category_data = db.get_categories() data = [] + report_on = [c.strip() for c in opts.report.split(',') if c.strip()] categories = [k for k in category_data.keys() - if db.metadata_for_field(k)['kind'] not in ['user', 'search']] + if db.metadata_for_field(k)['kind'] not in ['user', 'search'] and + (not report_on or k in report_on)] categories.sort(cmp=lambda x,y: cmp(x if x[0] != '#' else x[1:], y if y[0] != '#' else y[1:])) From 3ad46e0018f5c9b96e26d73b3d01c4a7cc4c70e0 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Fri, 8 Oct 2010 11:01:54 +0100 Subject: [PATCH 6/9] fix #7097 and #7079 - covers not being copied between libraries under some condition --- src/calibre/library/database2.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index a748a8c45b..f1087b3898 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -681,7 +681,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): mi = self.data.get(idx, self.FIELD_MAP['all_metadata'], row_is_id = index_is_id) if mi is not None: - if get_cover and mi.cover is None: + if get_cover: + # Always get the cover, because the value can be wrong if the + # original mi was from the OPF mi.cover = self.cover(idx, index_is_id=index_is_id, as_path=True) return mi @@ -1281,8 +1283,10 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): doit(self.set_series, id, mi.series, notify=False, commit=False) if mi.cover_data[1] is not None: doit(self.set_cover, id, mi.cover_data[1]) # doesn't use commit - elif mi.cover is not None and os.access(mi.cover, os.R_OK): - doit(self.set_cover, id, lopen(mi.cover, 'rb')) + elif mi.cover is not None: + if os.access(mi.cover, os.R_OK): + with lopen(mi.cover, 'rb') as f: + doit(self.set_cover, id, f) if mi.tags: doit(self.set_tags, id, mi.tags, notify=False, commit=False) if mi.comments: @@ -2157,6 +2161,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): else: with lopen(path, 'rb') as f: self.add_format(id, ext, f, index_is_id=True) + # Mark the book dirty, It probably already has been done by + # set_metadata, but probably isn't good enough + self.dirtied([id], commit=False) self.conn.commit() self.data.refresh_ids(self, [id]) # Needed to update format list and size if notify: From 76cd6956692bf859c82881b0c75ee85e8c4d8fd4 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Fri, 8 Oct 2010 13:04:59 +0100 Subject: [PATCH 7/9] Add 'Library maintenance' to the library dropdown menu 1) move check integrity to here 2) move check library to here 3) move mark dirty to here 4) add a stub recover_database command 5) improve help message for cli calibredb restore_database, and add the --really-do-it option --- src/calibre/gui2/actions/choose_library.py | 127 ++++++++++++++++++++- src/calibre/gui2/dialogs/check_library.py | 2 +- src/calibre/gui2/preferences/misc.py | 93 +-------------- src/calibre/gui2/preferences/misc.ui | 21 ---- src/calibre/library/cli.py | 24 +++- 5 files changed, 141 insertions(+), 126 deletions(-) diff --git a/src/calibre/gui2/actions/choose_library.py b/src/calibre/gui2/actions/choose_library.py index 044cbcdf85..95b3f9e24d 100644 --- a/src/calibre/gui2/actions/choose_library.py +++ b/src/calibre/gui2/actions/choose_library.py @@ -8,7 +8,7 @@ __docformat__ = 'restructuredtext en' import os, shutil from functools import partial -from PyQt4.Qt import QMenu, Qt, QInputDialog +from PyQt4.Qt import QMenu, Qt, QInputDialog, QThread, pyqtSignal, QProgressDialog from calibre import isbytestring from calibre.constants import filesystem_encoding @@ -16,6 +16,7 @@ from calibre.utils.config import prefs from calibre.gui2 import gprefs, warning_dialog, Dispatcher, error_dialog, \ question_dialog, info_dialog from calibre.gui2.actions import InterfaceAction +from calibre.gui2.dialogs.check_library import CheckLibraryDialog class LibraryUsageStats(object): # {{{ @@ -75,6 +76,72 @@ class LibraryUsageStats(object): # {{{ self.write_stats() # }}} +# Check Integrity {{{ + +class VacThread(QThread): + + check_done = pyqtSignal(object, object) + callback = pyqtSignal(object, object) + + def __init__(self, parent, db): + QThread.__init__(self, parent) + self.db = db + self._parent = parent + + def run(self): + err = bad = None + try: + bad = self.db.check_integrity(self.callbackf) + except: + import traceback + err = traceback.format_exc() + self.check_done.emit(bad, err) + + def callbackf(self, progress, msg): + self.callback.emit(progress, msg) + + +class CheckIntegrity(QProgressDialog): + + def __init__(self, db, parent=None): + QProgressDialog.__init__(self, parent) + self.db = db + self.setCancelButton(None) + self.setMinimum(0) + self.setMaximum(100) + self.setWindowTitle(_('Checking database integrity')) + self.setAutoReset(False) + self.setValue(0) + + self.vthread = VacThread(self, db) + self.vthread.check_done.connect(self.check_done, + type=Qt.QueuedConnection) + self.vthread.callback.connect(self.callback, type=Qt.QueuedConnection) + self.vthread.start() + + def callback(self, progress, msg): + self.setLabelText(msg) + self.setValue(int(100*progress)) + + def check_done(self, bad, err): + if err: + error_dialog(self, _('Error'), + _('Failed to check database integrity'), + det_msg=err, show=True) + elif bad: + titles = [self.db.title(x, index_is_id=True) for x in bad] + det_msg = '\n'.join(titles) + warning_dialog(self, _('Some inconsistencies found'), + _('The following books had formats listed in the ' + 'database that are not actually available. ' + 'The entries for the formats have been removed. ' + 'You should check them manually. This can ' + 'happen if you manipulate the files in the ' + 'library folder directly.'), det_msg=det_msg, show=True) + self.reset() + +# }}} + class ChooseLibraryAction(InterfaceAction): name = 'Choose Library' @@ -117,11 +184,28 @@ class ChooseLibraryAction(InterfaceAction): self.rename_separator = self.choose_menu.addSeparator() - self.create_action(spec=(_('Library backup status...'), 'lt.png', None, - None), attr='action_backup_status') - self.action_backup_status.triggered.connect(self.backup_status, - type=Qt.QueuedConnection) - self.choose_menu.addAction(self.action_backup_status) + self.maintenance_menu = QMenu(_('Library Maintenance')) + ac = self.create_action(spec=(_('Library metadata backup status'), + 'lt.png', None, None), attr='action_backup_status') + ac.triggered.connect(self.backup_status, type=Qt.QueuedConnection) + self.maintenance_menu.addAction(ac) + ac = self.create_action(spec=(_('Start backing up metadata of all books'), + 'lt.png', None, None), attr='action_backup_metadata') + ac.triggered.connect(self.mark_dirty, type=Qt.QueuedConnection) + self.maintenance_menu.addAction(ac) + ac = self.create_action(spec=(_('Check library'), 'lt.png', + None, None), attr='action_check_library') + ac.triggered.connect(self.check_library, type=Qt.QueuedConnection) + self.maintenance_menu.addAction(ac) + ac = self.create_action(spec=(_('Check database integrity'), 'lt.png', + None, None), attr='action_check_database') + ac.triggered.connect(self.check_database, type=Qt.QueuedConnection) + self.maintenance_menu.addAction(ac) + ac = self.create_action(spec=(_('Recover database'), 'lt.png', + None, None), attr='action_restore_database') + ac.triggered.connect(self.restore_database, type=Qt.QueuedConnection) + self.maintenance_menu.addAction(ac) + self.choose_menu.addMenu(self.maintenance_menu) def library_name(self): db = self.gui.library_view.model().db @@ -234,6 +318,37 @@ class ChooseLibraryAction(InterfaceAction): _('Book metadata files remaining to be written: %s') % dirty_text, show=True) + def mark_dirty(self): + db = self.gui.library_view.model().db + db.dirtied(list(db.data.iterallids())) + info_dialog(self.gui, _('Backup metadata'), + _('Metadata will be backed up while calibre is running, at the ' + 'rate of approximately 1 book per second.'), show=True) + + def check_library(self): + db = self.gui.library_view.model().db + d = CheckLibraryDialog(self.gui.parent(), db) + d.exec_() + + def check_database(self, *args): + m = self.gui.library_view.model() + m.stop_metadata_backup() + try: + d = CheckIntegrity(m.db, self.gui) + d.exec_() + finally: + m.start_metadata_backup() + + def restore_database(self): + info_dialog(self.gui, _('Recover database'), + _( + 'This command rebuilds your calibre database from the information ' + 'stored by calibre in the OPF files.' + '

' + + 'This function is not currently available in the GUI. You can ' + 'recover your database using the \'calibredb restore_database\' ' + 'command line function.' + ), show=True) + def switch_requested(self, location): if not self.change_library_allowed(): return diff --git a/src/calibre/gui2/dialogs/check_library.py b/src/calibre/gui2/dialogs/check_library.py index 29e5a2097c..46071d3c06 100644 --- a/src/calibre/gui2/dialogs/check_library.py +++ b/src/calibre/gui2/dialogs/check_library.py @@ -32,7 +32,7 @@ class CheckLibraryDialog(QDialog): self.copy = QPushButton(_('Copy to clipboard')) self.copy.setDefault(False) self.copy.clicked.connect(self.copy_to_clipboard) - self.ok = QPushButton('&OK') + self.ok = QPushButton('&Done') self.ok.setDefault(True) self.ok.clicked.connect(self.accept) self.cancel = QPushButton('&Cancel') diff --git a/src/calibre/gui2/preferences/misc.py b/src/calibre/gui2/preferences/misc.py index c9dc25caff..330332a716 100644 --- a/src/calibre/gui2/preferences/misc.py +++ b/src/calibre/gui2/preferences/misc.py @@ -5,81 +5,14 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -from PyQt4.Qt import QProgressDialog, QThread, Qt, pyqtSignal -from calibre.gui2.dialogs.check_library import CheckLibraryDialog from calibre.gui2.preferences import ConfigWidgetBase, test_widget from calibre.gui2.preferences.misc_ui import Ui_Form -from calibre.gui2 import error_dialog, config, warning_dialog, \ - open_local_file, info_dialog +from calibre.gui2 import error_dialog, config, open_local_file, info_dialog from calibre.constants import isosx # Check Integrity {{{ -class VacThread(QThread): - - check_done = pyqtSignal(object, object) - callback = pyqtSignal(object, object) - - def __init__(self, parent, db): - QThread.__init__(self, parent) - self.db = db - self._parent = parent - - def run(self): - err = bad = None - try: - bad = self.db.check_integrity(self.callbackf) - except: - import traceback - err = traceback.format_exc() - self.check_done.emit(bad, err) - - def callbackf(self, progress, msg): - self.callback.emit(progress, msg) - - -class CheckIntegrity(QProgressDialog): - - def __init__(self, db, parent=None): - QProgressDialog.__init__(self, parent) - self.db = db - self.setCancelButton(None) - self.setMinimum(0) - self.setMaximum(100) - self.setWindowTitle(_('Checking database integrity')) - self.setAutoReset(False) - self.setValue(0) - - self.vthread = VacThread(self, db) - self.vthread.check_done.connect(self.check_done, - type=Qt.QueuedConnection) - self.vthread.callback.connect(self.callback, type=Qt.QueuedConnection) - self.vthread.start() - - def callback(self, progress, msg): - self.setLabelText(msg) - self.setValue(int(100*progress)) - - def check_done(self, bad, err): - if err: - error_dialog(self, _('Error'), - _('Failed to check database integrity'), - det_msg=err, show=True) - elif bad: - titles = [self.db.title(x, index_is_id=True) for x in bad] - det_msg = '\n'.join(titles) - warning_dialog(self, _('Some inconsistencies found'), - _('The following books had formats listed in the ' - 'database that are not actually available. ' - 'The entries for the formats have been removed. ' - 'You should check them manually. This can ' - 'happen if you manipulate the files in the ' - 'library folder directly.'), det_msg=det_msg, show=True) - self.reset() - -# }}} - class ConfigWidget(ConfigWidgetBase, Ui_Form): def genesis(self, gui): @@ -88,39 +21,15 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): r('worker_limit', config, restart_required=True) r('enforce_cpu_limit', config, restart_required=True) self.device_detection_button.clicked.connect(self.debug_device_detection) - self.compact_button.clicked.connect(self.compact) - self.button_all_books_dirty.clicked.connect(self.mark_dirty) - self.button_check_library.clicked.connect(self.check_library) self.button_open_config_dir.clicked.connect(self.open_config_dir) self.button_osx_symlinks.clicked.connect(self.create_symlinks) self.button_osx_symlinks.setVisible(isosx) - def mark_dirty(self): - db = self.gui.library_view.model().db - db.dirtied(list(db.data.iterallids())) - info_dialog(self, _('Backup metadata'), - _('Metadata will be backed up while calibre is running, at the ' - 'rate of 30 books per minute.'), show=True) - - def check_library(self): - db = self.gui.library_view.model().db - d = CheckLibraryDialog(self.gui.parent(), db) - d.exec_() - def debug_device_detection(self, *args): from calibre.gui2.preferences.device_debug import DebugDevice d = DebugDevice(self) d.exec_() - def compact(self, *args): - m = self.gui.library_view.model() - m.stop_metadata_backup() - try: - d = CheckIntegrity(m.db, self) - d.exec_() - finally: - m.start_metadata_backup() - def open_config_dir(self, *args): from calibre.utils.config import config_dir open_local_file(config_dir) diff --git a/src/calibre/gui2/preferences/misc.ui b/src/calibre/gui2/preferences/misc.ui index dd0ca15840..c036cb971b 100644 --- a/src/calibre/gui2/preferences/misc.ui +++ b/src/calibre/gui2/preferences/misc.ui @@ -77,13 +77,6 @@ - - - - &Check database integrity - - - @@ -124,20 +117,6 @@ - - - - Back up metadata of all books - - - - - - - Check the library folders for potential problems - - - diff --git a/src/calibre/library/cli.py b/src/calibre/library/cli.py index 63361d87f5..44a7368ce1 100644 --- a/src/calibre/library/cli.py +++ b/src/calibre/library/cli.py @@ -961,13 +961,19 @@ def restore_database_option_parser(): ''' %prog restore_database [options] - Restore this database from the metadata stored in OPF - files in each directory of the calibre library. This is - useful if your metadata.db file has been corrupted. + Restore this database from the metadata stored in OPF files in each + directory of the calibre library. This is useful if your metadata.db file + has been corrupted. - WARNING: This completely regenerates your database. You will - lose stored per-book conversion settings and custom recipes. + WARNING: This command completely regenerates your database. You will lose + all saved searches, user categories, plugboards, stored per-book conversion + settings, and custom recipes. Restored metadata will only be as accurate as + what is found in the OPF files. ''')) + + parser.add_option('-r', '--really-do-it', default=False, action='store_true', + help=_('Really do the recovery. The command will not run ' + 'unless this option is specified.')) return parser def command_restore_database(args, dbpath): @@ -978,6 +984,12 @@ def command_restore_database(args, dbpath): parser.print_help() return 1 + if not opts.really_do_it: + print _('You must provide the --really-do-it option to do a recovery\n') + parser.print_help() + return 1 + + return if opts.library_path is not None: dbpath = opts.library_path @@ -1028,7 +1040,7 @@ information is the equivalent of what is shown in the tags pane. parser.add_option('-r', '--categories', default='', dest='report', help=_("Comma-separated list of category lookup names.\n" "Default: all")) - parser.add_option('-w', '--line-width', default=-1, type=int, + parser.add_option('-w', '--idth', default=-1, type=int, help=_('The maximum width of a single line in the output. ' 'Defaults to detecting screen size.')) parser.add_option('-s', '--separator', default=',', From 7fb8f74c1629dded7cfa8c19c669900af58b47a5 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Fri, 8 Oct 2010 17:34:57 +0100 Subject: [PATCH 8/9] UUID fixes, plus some debugging information --- src/calibre/gui2/actions/copy_to_library.py | 9 ++++++--- src/calibre/gui2/device.py | 17 +++++++++-------- src/calibre/gui2/library/models.py | 12 ++++++++++-- src/calibre/library/database2.py | 14 +++++++++++++- 4 files changed, 38 insertions(+), 14 deletions(-) diff --git a/src/calibre/gui2/actions/copy_to_library.py b/src/calibre/gui2/actions/copy_to_library.py index 513026f757..47f7904841 100644 --- a/src/calibre/gui2/actions/copy_to_library.py +++ b/src/calibre/gui2/actions/copy_to_library.py @@ -18,7 +18,7 @@ from calibre.utils.config import prefs, tweaks class Worker(Thread): - def __init__(self, ids, db, loc, progress, done): + def __init__(self, ids, db, loc, progress, done, delete_after): Thread.__init__(self) self.ids = ids self.processed = set([]) @@ -27,6 +27,7 @@ class Worker(Thread): self.error = None self.progress = progress self.done = done + self.delete_after = delete_after def run(self): try: @@ -68,7 +69,8 @@ class Worker(Thread): self.add_formats(identical_book, paths, newdb, replace=False) if not added: newdb.import_book(mi, paths, notify=False, import_hooks=False, - apply_import_tags=tweaks['add_new_book_tags_when_importing_books']) + apply_import_tags=tweaks['add_new_book_tags_when_importing_books'], + preserve_uuid=self.delete_after) co = self.db.conversion_options(x, 'PIPE') if co is not None: newdb.set_conversion_options(x, 'PIPE', co) @@ -134,7 +136,8 @@ class CopyToLibraryAction(InterfaceAction): self.pd.set_msg(_('Copying') + ' ' + title) self.pd.set_value(idx) - self.worker = Worker(ids, db, loc, Dispatcher(progress), Dispatcher(self.pd.accept)) + self.worker = Worker(ids, db, loc, Dispatcher(progress), + Dispatcher(self.pd.accept), delete_after) self.worker.start() self.pd.exec_() diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index 0344a8f21d..bc9f5cf671 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -1413,15 +1413,16 @@ class DeviceMixin(object): # {{{ # Force a reset if the caches are not initialized if reset or not hasattr(self, 'db_book_title_cache'): + # Build a cache (map) of the library, so the search isn't On**2 + self.db_book_title_cache = {} + self.db_book_uuid_cache = {} # It might be possible to get here without having initialized the # library view. In this case, simply give up try: db = self.library_view.model().db except: return False - # Build a cache (map) of the library, so the search isn't On**2 - self.db_book_title_cache = {} - self.db_book_uuid_cache = {} + for id in db.data.iterallids(): mi = db.get_metadata(id, index_is_id=True) title = clean_string(mi.title) @@ -1455,7 +1456,7 @@ class DeviceMixin(object): # {{{ if update_metadata: book.smart_update(self.db_book_uuid_cache[book.uuid], replace_metadata=True) - book.in_library = True + book.in_library = 'UUID' # ensure that the correct application_id is set book.application_id = \ self.db_book_uuid_cache[book.uuid].application_id @@ -1472,7 +1473,7 @@ class DeviceMixin(object): # {{{ if update_metadata: book.smart_update(d['db_ids'][book.application_id], replace_metadata=True) - book.in_library = True + book.in_library = 'APP_ID' continue # Sonys know their db_id independent of the application_id # in the metadata cache. Check that as well. @@ -1480,7 +1481,7 @@ class DeviceMixin(object): # {{{ if update_metadata: book.smart_update(d['db_ids'][book.db_id], replace_metadata=True) - book.in_library = True + book.in_library = 'DB_ID' book.application_id = \ d['db_ids'][book.db_id].application_id continue @@ -1497,14 +1498,14 @@ class DeviceMixin(object): # {{{ if update_metadata: book.smart_update(d['authors'][book_authors], replace_metadata=True) - book.in_library = True + book.in_library = 'AUTHOR' book.application_id = \ d['authors'][book_authors].application_id elif book_authors in d['author_sort']: if update_metadata: book.smart_update(d['author_sort'][book_authors], replace_metadata=True) - book.in_library = True + book.in_library = 'AUTH_SORT' book.application_id = \ d['author_sort'][book_authors].application_id else: diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index 53e0982211..2946985342 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -24,7 +24,7 @@ from calibre.library.caches import _match, CONTAINS_MATCH, EQUALS_MATCH, \ REGEXP_MATCH, CoverCache, MetadataBackup from calibre.library.cli import parse_series_string from calibre import strftime, isbytestring, prepare_string_for_xml -from calibre.constants import filesystem_encoding +from calibre.constants import filesystem_encoding, DEBUG from calibre.gui2.library import DEFAULT_SORT def human_readable(size, precision=1): @@ -699,6 +699,10 @@ class BooksModel(QAbstractTableModel): # {{{ if role == Qt.DisplayRole: return QVariant(self.headers[self.column_map[section]]) return NONE + if DEBUG and role == Qt.ToolTipRole and orientation == Qt.Vertical: + col = self.db.field_metadata['uuid']['rec_index'] + return QVariant(_('This book\'s UUID is "{0}"').format(self.db.data[section][col])) + if role == Qt.DisplayRole: # orientation is vertical return QVariant(section+1) return NONE @@ -1206,6 +1210,8 @@ class DeviceBooksModel(BooksModel): # {{{ if tags: tags.sort(cmp=lambda x,y: cmp(x.lower(), y.lower())) return QVariant(', '.join(tags)) + elif DEBUG and cname == 'inlibrary': + return QVariant(self.db[self.map[row]].in_library) elif role == Qt.ToolTipRole and index.isValid(): if self.map[row] in self.indices_to_be_deleted(): return QVariant(_('Marked for deletion')) @@ -1227,8 +1233,10 @@ class DeviceBooksModel(BooksModel): # {{{ return NONE def headerData(self, section, orientation, role): - if role == Qt.ToolTipRole: + if role == Qt.ToolTipRole and orientation == Qt.Horizontal: return QVariant(_('The lookup/search name is "{0}"').format(self.column_map[section])) + if DEBUG and role == Qt.ToolTipRole and orientation == Qt.Vertical: + return QVariant(_('This book\'s UUID is "{0}"').format(self.db[self.map[section]].uuid)) if role != Qt.DisplayRole: return NONE if orientation == Qt.Horizontal: diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index f1087b3898..f123dbac79 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -1466,6 +1466,16 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if notify: self.notify('metadata', [id]) + def set_uuid(self, id, uuid, notify=True, commit=True): + if uuid: + self.conn.execute('UPDATE books SET uuid=? WHERE id=?', (uuid, id)) + self.data.set(id, self.FIELD_MAP['uuid'], uuid, row_is_id=True) + self.dirtied([id], commit=False) + if commit: + self.conn.commit() + if notify: + self.notify('metadata', [id]) + # Convenience methods for tags_list_editor # Note: we generally do not need to refresh_ids because library_view will # refresh everything. @@ -2129,7 +2139,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): return None, len(ids) def import_book(self, mi, formats, notify=True, import_hooks=True, - apply_import_tags=True): + apply_import_tags=True, preserve_uuid=False): series_index = 1.0 if mi.series_index is None else mi.series_index if apply_import_tags: self._add_newbook_tag(mi) @@ -2152,6 +2162,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if mi.pubdate is None: mi.pubdate = utcnow() self.set_metadata(id, mi, ignore_errors=True) + if preserve_uuid and mi.uuid: + self.set_uuid(id, mi.uuid, commit=False) for path in formats: ext = os.path.splitext(path)[1][1:].lower() if ext == 'opf': From bdfb1cf6432adf3efc601377e4d6d028958a26fa Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Fri, 8 Oct 2010 17:55:02 +0100 Subject: [PATCH 9/9] Take out debugging return from cli restore_db Add restore UUID to recover.py --- src/calibre/library/cli.py | 1 - src/calibre/library/restore.py | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/library/cli.py b/src/calibre/library/cli.py index 44a7368ce1..8ba73c0bd1 100644 --- a/src/calibre/library/cli.py +++ b/src/calibre/library/cli.py @@ -989,7 +989,6 @@ def command_restore_database(args, dbpath): parser.print_help() return 1 - return if opts.library_path is not None: dbpath = opts.library_path diff --git a/src/calibre/library/restore.py b/src/calibre/library/restore.py index ab4e59a4c3..16aba3aebd 100644 --- a/src/calibre/library/restore.py +++ b/src/calibre/library/restore.py @@ -200,6 +200,8 @@ class Restore(Thread): def restore_book(self, book, db): db.create_book_entry(book['mi'], add_duplicates=True, force_id=book['id']) + if book['mi'].uuid: + db.set_uuid(book['id'], book['mi'].uuid, commit=False, notify=False) db.conn.execute('UPDATE books SET path=? WHERE id=?', (book['path'], book['id']))