From 829447f01fd34dd780a7ee6083cdd53b751e5c40 Mon Sep 17 00:00:00 2001 From: Charles Haley Date: Tue, 9 Sep 2014 13:04:57 +0200 Subject: [PATCH 1/2] Performance improvements: - Add an option to booklists to not check for duplicates if the device knows there aren't any. Eliminates N squared lookup times. - Make the wireless device driver use the above option. - Change a comprehension to use a generator instead of a returned list of keys. FWIW: found these because a CC user has 18,000 books on his device! --- src/calibre/devices/smart_device_app/driver.py | 12 +++++++++--- src/calibre/devices/usbms/books.py | 4 ++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/calibre/devices/smart_device_app/driver.py b/src/calibre/devices/smart_device_app/driver.py index 2ca7df6926..b285bd814e 100644 --- a/src/calibre/devices/smart_device_app/driver.py +++ b/src/calibre/devices/smart_device_app/driver.py @@ -1246,6 +1246,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): opcode, result = self._receive_from_client(print_debug_info=False) books_on_device.append(result) + self._debug('received all books. count=', count) + books_to_send = [] lpaths_on_device = set() for r in books_on_device: @@ -1258,7 +1260,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): if book: if self.client_cache_uses_lpaths: lpaths_on_device.add(r.get('lpath')) - bl.add_book(book, replace_metadata=True) + bl.add_book(book, replace_metadata=True, + check_for_duplicates=not self.client_cache_uses_lpaths) book.set('_is_read_', r.get('_is_read_', None)) book.set('_sync_type_', r.get('_sync_type_', None)) book.set('_last_read_date_', r.get('_last_read_date_', None)) @@ -1266,9 +1269,10 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): else: books_to_send.append(r['priKey']) + self._debug('processed cache. count=', len(books_on_device)) count_of_cache_items_deleted = 0 if self.client_cache_uses_lpaths: - for lpath in self.known_metadata.keys(): + for lpath in self.known_metadata.iterkeys(): if lpath not in lpaths_on_device: try: uuid = self.known_metadata[lpath].get('uuid', None) @@ -1299,10 +1303,12 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): if '_series_sort_' in result: del result['_series_sort_'] book = self.json_codec.raw_to_book(result, SDBook, self.PREFIX) + self._debug('title', book.title) book.set('_is_read_', result.get('_is_read_', None)) book.set('_sync_type_', result.get('_sync_type_', None)) book.set('_last_read_date_', result.get('_last_read_date_', None)) - bl.add_book(book, replace_metadata=True) + bl.add_book(book, replace_metadata=True, + check_for_duplicates=not self.client_cache_uses_lpaths) if '_new_book_' in result: book.set('_new_book_', True) else: diff --git a/src/calibre/devices/usbms/books.py b/src/calibre/devices/usbms/books.py index 5254a814be..cf49188278 100644 --- a/src/calibre/devices/usbms/books.py +++ b/src/calibre/devices/usbms/books.py @@ -72,13 +72,13 @@ class BookList(_BookList): def supports_collections(self): return False - def add_book(self, book, replace_metadata): + def add_book(self, book, replace_metadata, check_for_duplicates=True): ''' Add the book to the booklist, if needed. Return None if the book is already there and not updated, otherwise return the book. ''' try: - b = self.index(book) + b = self.index(book) if check_for_duplicates else None except (ValueError, IndexError): b = None if b is None: From c0b4994b6e31a590143933863916dfd54a6c2879 Mon Sep 17 00:00:00 2001 From: Charles Haley Date: Tue, 9 Sep 2014 14:09:51 +0200 Subject: [PATCH 2/2] Performance improvement again: don't change an existing method's signature. --- src/calibre/devices/smart_device_app/driver.py | 4 ++-- src/calibre/devices/usbms/books.py | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/calibre/devices/smart_device_app/driver.py b/src/calibre/devices/smart_device_app/driver.py index b285bd814e..fba450d714 100644 --- a/src/calibre/devices/smart_device_app/driver.py +++ b/src/calibre/devices/smart_device_app/driver.py @@ -1260,7 +1260,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): if book: if self.client_cache_uses_lpaths: lpaths_on_device.add(r.get('lpath')) - bl.add_book(book, replace_metadata=True, + bl.add_book_extended(book, replace_metadata=True, check_for_duplicates=not self.client_cache_uses_lpaths) book.set('_is_read_', r.get('_is_read_', None)) book.set('_sync_type_', r.get('_sync_type_', None)) @@ -1307,7 +1307,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): book.set('_is_read_', result.get('_is_read_', None)) book.set('_sync_type_', result.get('_sync_type_', None)) book.set('_last_read_date_', result.get('_last_read_date_', None)) - bl.add_book(book, replace_metadata=True, + bl.add_book_extended(book, replace_metadata=True, check_for_duplicates=not self.client_cache_uses_lpaths) if '_new_book_' in result: book.set('_new_book_', True) diff --git a/src/calibre/devices/usbms/books.py b/src/calibre/devices/usbms/books.py index cf49188278..bcc910282b 100644 --- a/src/calibre/devices/usbms/books.py +++ b/src/calibre/devices/usbms/books.py @@ -72,7 +72,10 @@ class BookList(_BookList): def supports_collections(self): return False - def add_book(self, book, replace_metadata, check_for_duplicates=True): + def add_book(self, book, replace_metadata): + self.add_book_extended(book, replace_metadata, check_for_duplicates=True) + + def add_book_extended(self, book, replace_metadata, check_for_duplicates): ''' Add the book to the booklist, if needed. Return None if the book is already there and not updated, otherwise return the book.