diff --git a/src/calibre/devices/usbms/books.py b/src/calibre/devices/usbms/books.py index 13fcb90b49..915d937379 100644 --- a/src/calibre/devices/usbms/books.py +++ b/src/calibre/devices/usbms/books.py @@ -111,6 +111,12 @@ class CollectionsBookList(BookList): from calibre.devices.usbms.driver import debug_print debug_print('Starting get_collections:', prefs['manage_device_metadata']) debug_print('Renaming rules:', tweaks['sony_collection_renaming_rules']) + + # Complexity: we can use renaming rules only when using automatic + # management. Otherwise we don't always have the metadata to make the + # right decisions + use_renaming_rules = prefs['manage_device_metadata'] == 'on_connect' + collections = {} # This map of sets is used to avoid linear searches when testing for # book equality @@ -139,7 +145,16 @@ class CollectionsBookList(BookList): attrs = collection_attributes for attr in attrs: attr = attr.strip() - ign, val, orig_val, fm = book.format_field_extended(attr) + # If attr is device_collections, then we cannot use + # format_field, because we don't know the fields where the + # values came from. + if attr == 'device_collections': + doing_dc = True + val = book.device_collections # is a list + else: + doing_dc = False + ign, val, orig_val, fm = book.format_field_extended(attr) + if not val: continue if isbytestring(val): val = val.decode(preferred_encoding, 'replace') @@ -151,9 +166,15 @@ class CollectionsBookList(BookList): val = orig_val else: val = [val] + for category in val: is_series = False - if fm['is_custom']: # is a custom field + if doing_dc: + # Attempt to determine if this value is a series by + # comparing it to the series name. + if category == book.series: + is_series = True + elif fm['is_custom']: # is a custom field if fm['datatype'] == 'text' and len(category) > 1 and \ category[0] == '[' and category[-1] == ']': continue @@ -167,7 +188,11 @@ class CollectionsBookList(BookList): ('series' in collection_attributes and book.get('series', None) == category): is_series = True - cat_name = self.compute_category_name(attr, category, fm) + if use_renaming_rules: + cat_name = self.compute_category_name(attr, category, fm) + else: + cat_name = category + if cat_name not in collections: collections[cat_name] = [] collections_lpaths[cat_name] = set() diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py index fd7ce8a6c3..0526de96a0 100644 --- a/src/calibre/ebooks/metadata/book/base.py +++ b/src/calibre/ebooks/metadata/book/base.py @@ -455,6 +455,8 @@ class Metadata(object): res = format_date(res, cmeta['display'].get('date_format','dd MMM yyyy')) elif datatype == 'bool': res = _('Yes') if res else _('No') + elif datatype == 'float' and key.endswith('_index'): + res = self.format_series_index(res) return (name, unicode(res), orig_res, cmeta) if key in field_metadata and field_metadata[key]['kind'] == 'field': @@ -468,6 +470,8 @@ class Metadata(object): datatype = fmeta['datatype'] if key == 'authors': res = authors_to_string(res) + elif key == 'series_index': + res = self.format_series_index(res) elif datatype == 'text' and fmeta['is_multiple']: res = u', '.join(res) elif datatype == 'series' and series_with_index: diff --git a/src/calibre/gui2/custom_column_widgets.py b/src/calibre/gui2/custom_column_widgets.py index 90abfc2474..1758116e7a 100644 --- a/src/calibre/gui2/custom_column_widgets.py +++ b/src/calibre/gui2/custom_column_widgets.py @@ -452,9 +452,25 @@ class BulkSeries(BulkBase): self.name_widget = w self.widgets = [QLabel('&'+self.col_metadata['name']+':', parent), w] - self.widgets.append(QLabel(_('Automatically number books in this series'), parent)) - self.idx_widget=QCheckBox(parent) - self.widgets.append(self.idx_widget) + self.widgets.append(QLabel('', parent)) + w = QWidget(parent) + layout = QHBoxLayout(w) + layout.setContentsMargins(0, 0, 0, 0) + self.remove_series = QCheckBox(parent) + self.remove_series.setText(_('Remove series')) + layout.addWidget(self.remove_series) + self.idx_widget = QCheckBox(parent) + self.idx_widget.setText(_('Automatically number books')) + layout.addWidget(self.idx_widget) + self.force_number = QCheckBox(parent) + self.force_number.setText(_('Force numbers to start with ')) + layout.addWidget(self.force_number) + self.series_start_number = QSpinBox(parent) + self.series_start_number.setMinimum(1) + self.series_start_number.setProperty("value", 1) + layout.addWidget(self.series_start_number) + layout.addItem(QSpacerItem(20, 10, QSizePolicy.Expanding, QSizePolicy.Minimum)) + self.widgets.append(w) def initialize(self, book_id): self.idx_widget.setChecked(False) @@ -465,17 +481,26 @@ class BulkSeries(BulkBase): def getter(self): n = unicode(self.name_widget.currentText()).strip() i = self.idx_widget.checkState() - return n, i + f = self.force_number.checkState() + s = self.series_start_number.value() + r = self.remove_series.checkState() + return n, i, f, s, r def commit(self, book_ids, notify=False): - val, update_indices = self.gui_val - val = self.normalize_ui_val(val) - if val != '': + val, update_indices, force_start, at_value, clear = self.gui_val + val = '' if clear else self.normalize_ui_val(val) + if clear or val != '': extras = [] next_index = self.db.get_next_cc_series_num_for(val, num=self.col_id) for book_id in book_ids: + if clear: + extras.append(None) + continue if update_indices: - if tweaks['series_index_auto_increment'] == 'next': + if force_start: + s_index = at_value + at_value += 1 + elif tweaks['series_index_auto_increment'] == 'next': s_index = next_index next_index += 1 else: @@ -483,6 +508,8 @@ class BulkSeries(BulkBase): else: s_index = self.db.get_custom_extra(book_id, num=self.col_id, index_is_id=True) + if s_index is None: + s_index = 1.0 extras.append(s_index) self.db.set_custom_bulk(book_ids, val, extras=extras, num=self.col_id, notify=notify) diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index b14390e001..9c83b3aee5 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -32,7 +32,7 @@ class Worker(Thread): remove, add, au, aus, do_aus, rating, pub, do_series, \ do_autonumber, do_remove_format, remove_format, do_swap_ta, \ do_remove_conv, do_auto_author, series, do_series_restart, \ - series_start_value, do_title_case = self.args + series_start_value, do_title_case, clear_series = self.args # first loop: do author and title. These will commit at the end of each # operation, because each operation modifies the file system. We want to @@ -75,6 +75,9 @@ class Worker(Thread): if pub: self.db.set_publisher(id, pub, notify=False, commit=False) + if clear_series: + self.db.set_series(id, '', notify=False, commit=False) + if do_series: if do_series_restart: next = series_start_value @@ -592,6 +595,7 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog): rating = self.rating.value() pub = unicode(self.publisher.text()) do_series = self.write_series + clear_series = self.clear_series.isChecked() series = unicode(self.series.currentText()).strip() do_autonumber = self.autonumber_series.isChecked() do_series_restart = self.series_numbering_restarts.isChecked() @@ -606,7 +610,7 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog): args = (remove, add, au, aus, do_aus, rating, pub, do_series, do_autonumber, do_remove_format, remove_format, do_swap_ta, do_remove_conv, do_auto_author, series, do_series_restart, - series_start_value, do_title_case) + series_start_value, do_title_case, clear_series) bb = BlockingBusy(_('Applying changes to %d books. This may take a while.') %len(self.ids), parent=self) diff --git a/src/calibre/gui2/dialogs/metadata_bulk.ui b/src/calibre/gui2/dialogs/metadata_bulk.ui index e03a59b7ea..60e24dbceb 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.ui +++ b/src/calibre/gui2/dialogs/metadata_bulk.ui @@ -225,61 +225,50 @@ - - - List of known series. You can add new series. - - - List of known series. You can add new series. - - - true - - - QComboBox::InsertAlphabetically - - - QComboBox::AdjustToContents - - - - - - - Remove &format: - - - remove_format - - - - - - - - - - true - - - - - - - &Swap title and author - - - - - - - Change title to title case - - - Force the title to be in title case. If both this and swap authors are checked, -title and author are swapped before the title case is set - - + + + + + List of known series. You can add new series. + + + List of known series. You can add new series. + + + true + + + QComboBox::InsertAlphabetically + + + QComboBox::AdjustToContents + + + + + + + If checked, the series will be cleared + + + Clear series + + + + + + + Qt::Horizontal + + + + 20 + 00 + + + + + @@ -339,6 +328,44 @@ from the value in the box + + + + Remove &format: + + + remove_format + + + + + + + + + + true + + + + + + + &Swap title and author + + + + + + + Change title to title case + + + Force the title to be in title case. If both this and swap authors are checked, +title and author are swapped before the title case is set + + + diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index a2555cfc56..ff6d8b70f0 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -89,6 +89,7 @@ class BooksModel(QAbstractTableModel): # {{{ self.alignment_map = {} self.buffer_size = buffer self.cover_cache = None + self.metadata_backup = None self.bool_yes_icon = QIcon(I('ok.png')) self.bool_no_icon = QIcon(I('list_remove.png')) self.bool_blank_icon = QIcon(I('blank.png')) @@ -154,8 +155,14 @@ class BooksModel(QAbstractTableModel): # {{{ self.database_changed.emit(db) if self.cover_cache is not None: self.cover_cache.stop() + # Would like to to a join here, but the thread might be waiting to + # do something on the GUI thread. Deadlock. self.cover_cache = CoverCache(db, FunctionDispatcher(self.db.cover)) self.cover_cache.start() + if self.metadata_backup is not None: + self.metadata_backup.stop() + # Would like to to a join here, but the thread might be waiting to + # do something on the GUI thread. Deadlock. self.metadata_backup = MetadataBackup(db) self.metadata_backup.start() def refresh_cover(event, ids): diff --git a/src/calibre/gui2/preferences/create_custom_column.py b/src/calibre/gui2/preferences/create_custom_column.py index bec21270df..ebf33784d4 100644 --- a/src/calibre/gui2/preferences/create_custom_column.py +++ b/src/calibre/gui2/preferences/create_custom_column.py @@ -38,7 +38,7 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn): 'is_multiple':False}, 8:{'datatype':'bool', 'text':_('Yes/No'), 'is_multiple':False}, - 8:{'datatype':'composite', + 9:{'datatype':'composite', 'text':_('Column built from other columns'), 'is_multiple':False}, } diff --git a/src/calibre/gui2/preferences/misc.py b/src/calibre/gui2/preferences/misc.py index eae79fdfc0..37887d56dc 100644 --- a/src/calibre/gui2/preferences/misc.py +++ b/src/calibre/gui2/preferences/misc.py @@ -88,10 +88,15 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): r('enforce_cpu_limit', config, restart_required=True) self.device_detection_button.clicked.connect(self.debug_device_detection) self.compact_button.clicked.connect(self.compact) + self.button_all_books_dirty.clicked.connect(self.mark_dirty) self.button_open_config_dir.clicked.connect(self.open_config_dir) self.button_osx_symlinks.clicked.connect(self.create_symlinks) self.button_osx_symlinks.setVisible(isosx) + def mark_dirty(self): + db = self.gui.library_view.model().db + db.dirtied(list(db.data.iterallids())) + def debug_device_detection(self, *args): from calibre.gui2.preferences.device_debug import DebugDevice d = DebugDevice(self) diff --git a/src/calibre/gui2/preferences/misc.ui b/src/calibre/gui2/preferences/misc.ui index f8582a3675..492540901d 100644 --- a/src/calibre/gui2/preferences/misc.ui +++ b/src/calibre/gui2/preferences/misc.ui @@ -124,7 +124,14 @@ - + + + + Back up metadata of all books (while you are working) + + + + Qt::Vertical @@ -132,7 +139,7 @@ 20 - 18 + 1000 diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index bc16681f81..7a15cb3ce1 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -19,6 +19,7 @@ from calibre.utils.date import parse_date, now, UNDEFINED_DATE from calibre.utils.search_query_parser import SearchQueryParser from calibre.utils.pyparsing import ParseException from calibre.ebooks.metadata import title_sort +from calibre.ebooks.metadata.opf2 import metadata_to_opf from calibre import fit_image, prints class MetadataBackup(Thread): # {{{ @@ -36,39 +37,53 @@ class MetadataBackup(Thread): # {{{ self.keep_running = True from calibre.gui2 import FunctionDispatcher self.do_write = FunctionDispatcher(self.write) + self.get_metadata_for_dump = FunctionDispatcher(db.get_metadata_for_dump) + self.clear_dirtied = FunctionDispatcher(db.clear_dirtied) def stop(self): self.keep_running = False def run(self): + import traceback while self.keep_running: try: - id_ = self.db.dirtied_queue.get() + time.sleep(0.5) # Limit to two per second + id_ = self.db.dirtied_queue.get(True, 1.45) except Empty: continue except: # Happens during interpreter shutdown break - dump = [] try: - self.db.dump_metadata([id_], dump_to=dump) + path, mi = self.get_metadata_for_dump(id_) except: prints('Failed to get backup metadata for id:', id_, 'once') - import traceback traceback.print_exc() time.sleep(2) - dump = [] try: - self.db.dump_metadata([id_], dump_to=dump) + path, mi = self.get_metadata_for_dump(id_) except: prints('Failed to get backup metadata for id:', id_, 'again, giving up') traceback.print_exc() continue + + if mi is None: + self.clear_dirtied([id_]) + continue + + # Give the GUI thread a chance to do something. Python threads don't + # have priorities, so this thread would naturally keep the processor + # until some scheduling event happens. The sleep makes such an event + time.sleep(0.1) try: - path, raw = dump[0] + raw = metadata_to_opf(mi) except: - break + prints('Failed to convert to opf for id:', id_) + traceback.print_exc() + continue + + time.sleep(0.1) # Give the GUI thread a chance to do something try: self.do_write(path, raw) except: @@ -79,8 +94,13 @@ class MetadataBackup(Thread): # {{{ except: prints('Failed to write backup metadata for id:', id_, 'again, giving up') + continue - time.sleep(0.5) # Limit to two per second + time.sleep(0.1) # Give the GUI thread a chance to do something + try: + self.clear_dirtied([id_]) + except: + prints('Failed to clear dirtied for id:', id_) def write(self, path, raw): with open(path, 'wb') as f: @@ -106,7 +126,6 @@ class CoverCache(Thread): # {{{ self.keep_running = False def _image_for_id(self, id_): - time.sleep(0.050) # Limit 20/second to not overwhelm the GUI img = self.cover_func(id_, index_is_id=True, as_image=True) if img is None: img = QImage() @@ -122,7 +141,8 @@ class CoverCache(Thread): # {{{ def run(self): while self.keep_running: try: - id_ = self.load_queue.get() + time.sleep(0.050) # Limit 20/second to not overwhelm the GUI + id_ = self.load_queue.get(True, 2) except Empty: continue except: diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index a6f3f286bc..ce72b473e1 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -566,8 +566,26 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): def metadata_for_field(self, key): return self.field_metadata[key] + def clear_dirtied(self, book_ids): + ''' + Clear the dirtied indicator for the books. This is used when fetching + metadata, creating an OPF, and writing a file are separated into steps. + The last step is clearing the indicator + ''' + for book_id in book_ids: + if not self.data.has_id(book_id): + continue + self.conn.execute('DELETE FROM metadata_dirtied WHERE book=?', + (book_id,)) + # if a later exception prevents the commit, then the dirtied + # table will still have the book. No big deal, because the OPF + # is there and correct. We will simply do it again on next + # start + self.dirtied_cache.discard(book_id) + self.conn.commit() + def dump_metadata(self, book_ids=None, remove_from_dirtied=True, - commit=True, dump_to=None): + commit=True): ''' Write metadata for each record to an individual OPF file @@ -580,19 +598,12 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): for book_id in book_ids: if not self.data.has_id(book_id): continue - mi = self.get_metadata(book_id, index_is_id=True, get_cover=False) - # Always set cover to cover.jpg. Even if cover doesn't exist, - # no harm done. This way no need to call dirtied when - # cover is set/removed - mi.cover = 'cover.jpg' + path, mi = self.get_metadata_for_dump(book_id) + if path is None: + continue raw = metadata_to_opf(mi) - path = os.path.join(self.abspath(book_id, index_is_id=True), - 'metadata.opf') - if dump_to is None: - with open(path, 'wb') as f: - f.write(raw) - else: - dump_to.append((path, raw)) + with open(path, 'wb') as f: + f.write(raw) if remove_from_dirtied: self.conn.execute('DELETE FROM metadata_dirtied WHERE book=?', (book_id,)) @@ -638,6 +649,18 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.dirtied_cache = set() self.dirtied(book_ids) + def get_metadata_for_dump(self, idx): + try: + path = os.path.join(self.abspath(idx, index_is_id=True), 'metadata.opf') + mi = self.get_metadata(idx, index_is_id=True) + # Always set cover to cover.jpg. Even if cover doesn't exist, + # no harm done. This way no need to call dirtied when + # cover is set/removed + mi.cover = 'cover.jpg' + except: + return (None, None) + return (path, mi) + def get_metadata(self, idx, index_is_id=False, get_cover=False): ''' Convenience method to return metadata as a :class:`Metadata` object. @@ -647,6 +670,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): mi = self.data.get(idx, self.FIELD_MAP['all_metadata'], row_is_id = index_is_id) if mi is not None: + if get_cover and mi.cover is None: + mi.cover = self.cover(idx, index_is_id=index_is_id, as_path=True) return mi self.gm_missed += 1 diff --git a/src/calibre/library/restore.py b/src/calibre/library/restore.py index 89c7fe8395..7225802236 100644 --- a/src/calibre/library/restore.py +++ b/src/calibre/library/restore.py @@ -48,12 +48,13 @@ class Restore(Thread): self.books = [] self.conflicting_custom_cols = {} self.failed_restores = [] + self.mismatched_dirs = [] self.successes = 0 self.tb = None @property def errors_occurred(self): - return self.failed_dirs or \ + return self.failed_dirs or self.mismatched_dirs or \ self.conflicting_custom_cols or self.failed_restores @property @@ -74,6 +75,13 @@ class Restore(Thread): for x in self.conflicting_custom_cols: ans += '\t#'+x+'\n' + if self.mismatched_dirs: + ans += '\n\n' + ans += 'The following folders were ignored:\n' + for x in self.mismatched_dirs: + ans += '\t'+x+'\n' + + return ans @@ -140,7 +148,7 @@ class Restore(Thread): 'path': path, }) else: - self.ignored_dirs.append(dirpath) + self.mismatched_dirs.append(dirpath) def create_cc_metadata(self): self.books.sort(key=itemgetter('timestamp'))