From c02df37c780d5a4bb47e4951427b3fc6e85ae612 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 18 Aug 2010 09:50:58 -0600 Subject: [PATCH 1/4] Run bulk metadata updates in a separate thread --- src/calibre/gui2/custom_column_widgets.py | 62 +++++++++------ src/calibre/gui2/dialogs/metadata_bulk.py | 94 +++++++++++++---------- 2 files changed, 90 insertions(+), 66 deletions(-) diff --git a/src/calibre/gui2/custom_column_widgets.py b/src/calibre/gui2/custom_column_widgets.py index d624d5320d..6bd2f3c77a 100644 --- a/src/calibre/gui2/custom_column_widgets.py +++ b/src/calibre/gui2/custom_column_widgets.py @@ -11,7 +11,7 @@ from functools import partial from PyQt4.Qt import QComboBox, QLabel, QSpinBox, QDoubleSpinBox, QDateEdit, \ QDate, QGroupBox, QVBoxLayout, QPlainTextEdit, QSizePolicy, \ QSpacerItem, QIcon, QCheckBox, QWidget, QHBoxLayout, SIGNAL, \ - QPushButton, QCoreApplication + QPushButton from calibre.utils.date import qt_to_dt, now from calibre.gui2.widgets import TagsLineEdit, EnComboBox @@ -32,8 +32,13 @@ class Base(object): val = self.normalize_db_val(val) self.setter(val) + @property + def gui_val(self): + return self.getter() + + def commit(self, book_id, notify=False): - val = self.getter() + val = self.gui_val val = self.normalize_ui_val(val) if val != self.initial_val: self.db.set_custom(book_id, val, num=self.col_id, notify=notify) @@ -284,10 +289,14 @@ class Series(Base): if idx is not None: self.widgets[1].setCurrentIndex(idx) + def getter(self): + n = unicode(self.name_widget.currentText()).strip() + i = self.idx_widget.value() + return n, i + def commit(self, book_id, notify=False): - val = unicode(self.name_widget.currentText()).strip() + val, s_index = self.gui_val val = self.normalize_ui_val(val) - s_index = self.idx_widget.value() if val != self.initial_val or s_index != self.initial_index: if s_index == 0.0: if tweaks['series_index_auto_increment'] == 'next': @@ -378,6 +387,13 @@ def populate_metadata_page(layout, db, book_id, bulk=False, two_column=False, pa class BulkBase(Base): + @property + def gui_val(self): + if not hasattr(self, '_cached_gui_val_'): + self._cached_gui_val_ = self.getter() + return self._cached_gui_val_ + + def get_initial_value(self, book_ids): values = set([]) for book_id in book_ids: @@ -400,11 +416,10 @@ class BulkBase(Base): self.setter(val) def commit(self, book_ids, notify=False): - val = self.getter() + val = self.gui_val val = self.normalize_ui_val(val) if val != self.initial_val: for book_id in book_ids: - QCoreApplication.processEvents() self.db.set_custom(book_id, val, num=self.col_id, notify=notify) class BulkBool(BulkBase, Bool): @@ -443,13 +458,16 @@ class BulkSeries(BulkBase): self.name_widget.addItem(c) self.name_widget.setEditText('') + def getter(self): + n = unicode(self.name_widget.currentText()).strip() + i = self.idx_widget.checkState() + return n, i + def commit(self, book_ids, notify=False): - val = unicode(self.name_widget.currentText()).strip() + val, update_indices = self.gui_val val = self.normalize_ui_val(val) - update_indices = self.idx_widget.checkState() if val != '': for book_id in book_ids: - QCoreApplication.processEvents() if update_indices: if tweaks['series_index_auto_increment'] == 'next': s_index = self.db.get_next_cc_series_num_for\ @@ -526,41 +544,35 @@ class BulkText(BulkBase): def commit(self, book_ids, notify=False): if self.col_metadata['is_multiple']: + remove_all, adding, rtext = self.gui_val remove = set() - if self.removing_widget.checkbox.isChecked(): + if remove_all: for book_id in book_ids: remove |= set(self.db.get_custom(book_id, num=self.col_id, index_is_id=True)) else: - txt = unicode(self.removing_widget.tags_box.text()) + txt = rtext if txt: remove = set([v.strip() for v in txt.split(',')]) - txt = unicode(self.adding_widget.text()) + txt = adding if txt: add = set([v.strip() for v in txt.split(',')]) else: add = set() self.db.set_custom_bulk(book_ids, add=add, remove=remove, num=self.col_id) else: - val = self.getter() + val = self.gui_val val = self.normalize_ui_val(val) if val != self.initial_val: for book_id in book_ids: - QCoreApplication.processEvents() self.db.set_custom(book_id, val, num=self.col_id, notify=notify) - def getter(self, original_value = None): + def getter(self): if self.col_metadata['is_multiple']: - if self.removing_widget.checkbox.isChecked(): - ans = set() - else: - ans = set(original_value) - ans -= set([v.strip() for v in - unicode(self.removing_widget.tags_box.text()).split(',')]) - txt = unicode(self.adding_widget.text()) - if txt: - ans |= set([v.strip() for v in txt.split(',')]) - return ans # returning a set instead of a list works, for now at least. + return self.removing_widget.checkbox.isChecked(), \ + unicode(self.adding_widget.text()), \ + unicode(self.removing_widget.tags_box.text()) + val = unicode(self.widgets[1].currentText()).strip() if not val: val = None diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index 0139d0aee2..c216463941 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -4,14 +4,33 @@ __copyright__ = '2008, Kovid Goyal ' '''Dialog to edit metadata in bulk''' from PyQt4.Qt import SIGNAL, QObject, QDialog, QGridLayout, \ - QCoreApplication + QThread, Qt from calibre.gui2.dialogs.metadata_bulk_ui import Ui_MetadataBulkDialog from calibre.gui2.dialogs.tag_editor import TagEditor from calibre.ebooks.metadata import string_to_authors, \ authors_to_string from calibre.gui2.custom_column_widgets import populate_metadata_page -from calibre.gui2.dialogs.progress import ProgressDialog +from calibre.gui2.dialogs.progress import BlockingBusy +from calibre.gui2 import error_dialog + +class Worker(QThread): + + def __init__(self, func, parent=None): + QThread.__init__(self, parent) + self.func = func + self.error = None + + def run(self): + try: + self.func() + except Exception, err: + import traceback + try: + err = unicode(err) + except: + err = repr(err) + self.error = (err, traceback.format_exc()) class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog): @@ -107,35 +126,29 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog): if len(self.ids) < 1: return QDialog.accept(self) - pd = ProgressDialog(_('Working'), - _('Applying changes to %d books. This may take a while.')%len(self.ids), - 0, 0, self, cancelable=False) - pd.setModal(True) - pd.show() - def upd(): - QCoreApplication.processEvents() + remove = unicode(self.remove_tags.text()).strip().split(',') + add = unicode(self.tags.text()).strip().split(',') + au = unicode(self.authors.text()) + aus = unicode(self.author_sort.text()) + do_aus = self.author_sort.isEnabled() + rating = self.rating.value() + pub = unicode(self.publisher.text()) + do_series = self.write_series + series = unicode(self.series.currentText()).strip() + do_autonumber = self.autonumber_series.isChecked() + do_remove_format = self.remove_format.currentIndex() > -1 + remove_format = unicode(self.remove_format.currentText()) + do_swap_ta = self.swap_title_and_author.isChecked() + do_remove_conv = self.remove_conversion_settings.isChecked() + do_auto_author = self.auto_author_sort.isChecked() + self.changed = bool(self.ids) + # Cache values from GUI so that Qt widgets are not used in + # non GUI thread + for w in getattr(self, 'custom_column_widgets', []): + w.gui_val - try: - remove = unicode(self.remove_tags.text()).strip().split(',') - add = unicode(self.tags.text()).strip().split(',') - au = unicode(self.authors.text()) - aus = unicode(self.author_sort.text()) - do_aus = self.author_sort.isEnabled() - rating = self.rating.value() - pub = unicode(self.publisher.text()) - do_series = self.write_series - series = unicode(self.series.currentText()).strip() - do_autonumber = self.autonumber_series.isChecked() - do_remove_format = self.remove_format.currentIndex() > -1 - remove_format = unicode(self.remove_format.currentText()) - do_swap_ta = self.swap_title_and_author.isChecked() - do_remove_conv = self.remove_conversion_settings.isChecked() - do_auto_author = self.auto_author_sort.isChecked() - - upd() - self.changed = bool(self.ids) + def doit(): for id in self.ids: - upd() if do_swap_ta: title = self.db.title(id, index_is_id=True) aum = self.db.authors(id, index_is_id=True) @@ -146,55 +159,54 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog): if title: new_authors = string_to_authors(title) self.db.set_authors(id, new_authors, notify=False) - upd() if au: self.db.set_authors(id, string_to_authors(au), notify=False) - upd() if do_auto_author: x = self.db.author_sort_from_book(id, index_is_id=True) if x: self.db.set_author_sort(id, x, notify=False) - upd() if aus and do_aus: self.db.set_author_sort(id, aus, notify=False) - upd() if rating != -1: self.db.set_rating(id, 2*rating, notify=False) if pub: self.db.set_publisher(id, pub, notify=False) - upd() if do_series: next = self.db.get_next_series_num_for(series) self.db.set_series(id, series, notify=False) num = next if do_autonumber and series else 1.0 self.db.set_series_index(id, num, notify=False) - upd() if do_remove_format: self.db.remove_format(id, remove_format, index_is_id=True, notify=False) - upd() if do_remove_conv: self.db.delete_conversion_options(id, 'PIPE') - upd() for w in getattr(self, 'custom_column_widgets', []): w.commit(self.ids) self.db.bulk_modify_tags(self.ids, add=add, remove=remove, notify=False) - upd() + self.db.clean() + self.worker = Worker(doit, self) + self.worker.start() - finally: - pd.hide() + bb = BlockingBusy(_('Applying changes to %d books. This may take a while.') + %len(self.ids), parent=self) + self.worker.finished.connect(bb.accept, type=Qt.QueuedConnection) + bb.exec_() - self.db.clean() + if self.worker.error is not None: + return error_dialog(self, _('Failed'), + self.worker.error[0], det_msg=self.worker.error[1], + show=True) return QDialog.accept(self) From 9c64ce80683c3622bde8ab215e732e9b3e9a5c9c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 18 Aug 2010 09:57:25 -0600 Subject: [PATCH 2/4] Content Server: Do not add empty categories to the top level OPDS feed --- src/calibre/library/server/opds.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/calibre/library/server/opds.py b/src/calibre/library/server/opds.py index f32e5ad47a..e1cbb79599 100644 --- a/src/calibre/library/server/opds.py +++ b/src/calibre/library/server/opds.py @@ -550,6 +550,8 @@ class OPDSServer(object): (_('Title'), _('Title'), 'Otitle'), ] for category in categories: + if len(categories[category]) == 0: + continue if category == 'formats': continue meta = category_meta.get(category, None) From 39c820043db89dc5fc688a83b038837cd5df1b53 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 18 Aug 2010 10:06:00 -0600 Subject: [PATCH 3/4] Speed up bulk editing on non is_multiple custom columns as well --- src/calibre/gui2/custom_column_widgets.py | 18 +++++++------- src/calibre/library/custom_columns.py | 29 +++++++++++++++++++---- 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/src/calibre/gui2/custom_column_widgets.py b/src/calibre/gui2/custom_column_widgets.py index 6bd2f3c77a..4a1a85f159 100644 --- a/src/calibre/gui2/custom_column_widgets.py +++ b/src/calibre/gui2/custom_column_widgets.py @@ -419,8 +419,7 @@ class BulkBase(Base): val = self.gui_val val = self.normalize_ui_val(val) if val != self.initial_val: - for book_id in book_ids: - self.db.set_custom(book_id, val, num=self.col_id, notify=notify) + self.db.set_custom_bulk(book_ids, val, num=self.col_id, notify=notify) class BulkBool(BulkBase, Bool): pass @@ -467,17 +466,20 @@ class BulkSeries(BulkBase): val, update_indices = self.gui_val val = self.normalize_ui_val(val) if val != '': + extras = [] + next_index = self.db.get_next_cc_series_num_for(val, num=self.col_id) for book_id in book_ids: if update_indices: if tweaks['series_index_auto_increment'] == 'next': - s_index = self.db.get_next_cc_series_num_for\ - (val, num=self.col_id) + s_index = next_index + next_index += 1 else: s_index = 1.0 else: s_index = self.db.get_custom_extra(book_id, num=self.col_id, index_is_id=True) - self.db.set_custom(book_id, val, extra=s_index, + extras.append(s_index) + self.db.set_custom_bulk(book_ids, val, extras=extras, num=self.col_id, notify=notify) class RemoveTags(QWidget): @@ -559,13 +561,13 @@ class BulkText(BulkBase): add = set([v.strip() for v in txt.split(',')]) else: add = set() - self.db.set_custom_bulk(book_ids, add=add, remove=remove, num=self.col_id) + self.db.set_custom_bulk_multiple(book_ids, add=add, remove=remove, + num=self.col_id) else: val = self.gui_val val = self.normalize_ui_val(val) if val != self.initial_val: - for book_id in book_ids: - self.db.set_custom(book_id, val, num=self.col_id, notify=notify) + self.db.set_custom_bulk(book_ids, val, num=self.col_id, notify=notify) def getter(self): if self.col_metadata['is_multiple']: diff --git a/src/calibre/library/custom_columns.py b/src/calibre/library/custom_columns.py index 7c613295b9..f02294a102 100644 --- a/src/calibre/library/custom_columns.py +++ b/src/calibre/library/custom_columns.py @@ -313,7 +313,7 @@ class CustomColumns(object): self.conn.commit() return changed - def set_custom_bulk(self, ids, add=[], remove=[], + def set_custom_bulk_multiple(self, ids, add=[], remove=[], label=None, num=None, notify=False): ''' Fast algorithm for updating custom column is_multiple datatypes. @@ -394,7 +394,30 @@ class CustomColumns(object): if notify: self.notify('metadata', ids) - def set_custom(self, id_, val, label=None, num=None, + def set_custom_bulk(self, ids, val, label=None, num=None, + append=False, notify=True, extras=None): + ''' + Change the value of a column for a set of books. The ids parameter is a + list of book ids to change. The extra field must be None or a list the + same length as ids. + ''' + if extras is not None and len(extras) != len(ids): + raise ValueError('Lentgh of ids and extras is not the same') + ev = None + for idx,id in enumerate(ids): + if extras is not None: + ev = extras[idx] + self._set_custom(id, val, label=label, num=num, append=append, + notify=notify, extra=ev) + self.conn.commit() + + def set_custom(self, id, val, label=None, num=None, + append=False, notify=True, extra=None): + self._set_custom(id, val, label=label, num=num, append=append, + notify=notify, extra=extra) + self.conn.commit() + + def _set_custom(self, id_, val, label=None, num=None, append=False, notify=True, extra=None): if label is not None: data = self.custom_column_label_map[label] @@ -450,7 +473,6 @@ class CustomColumns(object): self.conn.execute( '''INSERT INTO %s(book, value) VALUES (?,?)'''%lt, (id_, xid)) - self.conn.commit() nval = self.conn.get( 'SELECT custom_%s FROM meta2 WHERE id=?'%data['num'], (id_,), all=False) @@ -462,7 +484,6 @@ class CustomColumns(object): self.conn.execute( 'INSERT INTO %s(book,value) VALUES (?,?)'%table, (id_, val)) - self.conn.commit() nval = self.conn.get( 'SELECT custom_%s FROM meta2 WHERE id=?'%data['num'], (id_,), all=False) From 1c8d6e7c73bfdb41d06a63de870e0d5f5de00a16 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 18 Aug 2010 10:11:52 -0600 Subject: [PATCH 4/4] SONY driver: Fix series order being lost when metadata management is set to manual --- src/calibre/devices/usbms/books.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/calibre/devices/usbms/books.py b/src/calibre/devices/usbms/books.py index cdba980642..959f26199c 100644 --- a/src/calibre/devices/usbms/books.py +++ b/src/calibre/devices/usbms/books.py @@ -181,7 +181,9 @@ class CollectionsBookList(BookList): if lpath not in collections_lpaths[category]: collections_lpaths[category].add(lpath) collections[category].append(book) - if attr == 'series': + if attr == 'series' or \ + ('series' in collection_attributes and + getattr(book, 'series', None) == category): series_categories.add(category) # Sort collections for category, books in collections.items():