From 9a0dfff78e93857ed4bf4dad44efff506ba6f913 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sat, 15 May 2010 13:37:39 +0100 Subject: [PATCH] 1) improve performance of OnDevice refresh. Instead of rebuilding the complete book list, iterate through the existing one and set the value of OnDevice correctly. 2) Fix problems with Sony readers and metadata caching. Needed to ensure that when a book is added to the booklist from the JSON cache, it is added to the sony cache if it isn't already there. 3) Build the sony metadata maps (caches) on the fly instead of in reorder_playlists. 4) Refactor method declarations. 5) Move the JSON cache to the root of the card for Sony devices. --- src/calibre/devices/prs505/books.py | 78 ++++++++++++++++------------ src/calibre/devices/prs505/driver.py | 2 - src/calibre/devices/usbms/books.py | 7 ++- src/calibre/devices/usbms/driver.py | 19 +++---- src/calibre/gui2/library.py | 2 +- src/calibre/gui2/ui.py | 2 +- src/calibre/library/caches.py | 6 +++ src/calibre/library/database2.py | 2 + 8 files changed, 71 insertions(+), 47 deletions(-) diff --git a/src/calibre/devices/prs505/books.py b/src/calibre/devices/prs505/books.py index 7f4071a6cf..40a98913be 100644 --- a/src/calibre/devices/prs505/books.py +++ b/src/calibre/devices/prs505/books.py @@ -10,9 +10,8 @@ from base64 import b64encode as encode from calibre.devices.usbms.books import BookList as _BookList from calibre.devices import strftime as _strftime -from calibre.devices.usbms.books import Book as _Book -from calibre.devices.prs505 import MEDIA_XML -from calibre.devices.prs505 import CACHE_XML +from calibre.devices.prs505 import MEDIA_XML, CACHE_XML +from calibre.devices.errors import PathError strftime = functools.partial(_strftime, zone=time.gmtime) @@ -33,10 +32,14 @@ def sortable_title(title): class BookList(_BookList): - def __init__(self, oncard, prefix): - _BookList.__init__(self, oncard, prefix) + def __init__(self, oncard, prefix, settings): + _BookList.__init__(self, oncard, prefix, settings) if prefix is None: return + self.sony_id_cache = {} + self.books_lpath_cache = {} + opts = settings() + self.collections = opts.extra_customization.split(',') if opts.extra_customization else [] db = CACHE_XML if oncard else MEDIA_XML xml_file = open(prefix + db, 'rb') xml_file.seek(0) @@ -50,8 +53,21 @@ class BookList(_BookList): self.root_element = records[0] else: self.prefix = '' + for child in self.root_element.childNodes: + if child.nodeType == child.ELEMENT_NODE and child.hasAttribute("id"): + self.sony_id_cache[child.getAttribute('id')] = child.getAttribute('path') + # set the key to none. Will be filled in later when booklist is built + self.books_lpath_cache[child.getAttribute('path')] = None self.tag_order = {} + paths = self.purge_corrupted_files() + for path in paths: + try: + self.del_file(path, end_session=False) + except PathError: # Incase this is a refetch without a sync in between + continue + + def max_id(self): max = 0 for child in self.root_element.childNodes: @@ -73,22 +89,27 @@ class BookList(_BookList): def supports_tags(self): return True - def book_by_path(self, path): - for child in self.root_element.childNodes: - if child.nodeType == child.ELEMENT_NODE and child.hasAttribute("path"): - if path == child.getAttribute('path'): - return child - return None - - def add_book(self, book, collections=None): + def add_book(self, book, replace_metadata): + # Add a node into the DOM tree, representing a book. Also add to booklist if book in self: - return - """ Add a node into the DOM tree, representing a book """ - node = self.document.createElement(self.prefix + "text") - mime = MIME_MAP.get(book.lpath.rpartition('.')[-1].lower(), MIME_MAP['epub']) + # replacing metadata for book + self.delete_node(book.lpath) + else: + self.append(book) + if not replace_metadata: + if self.books_lpath_cache.has_key(book.lpath): + self.books_lpath_cache[book.lpath] = book + return + # Book not in metadata. Add it. Note that we don't need to worry about + # extra books in the Sony metadata. The reader deletes them for us when + # we disconnect. That said, if it becomes important one day, we can do + # it by scanning the books_lpath_cache for None entries and removing the + # corresponding nodes. + self.books_lpath_cache[book.lpath] = book cid = self.max_id()+1 - book.sony_id = cid - self.append(book) + node = self.document.createElement(self.prefix + "text") + self.sony_id_cache[cid] = book.lpath + mime = MIME_MAP.get(book.lpath.rpartition('.')[-1].lower(), MIME_MAP['epub']) try: sourceid = str(self[0].sourceid) if len(self) else '1' except: @@ -120,7 +141,7 @@ class BookList(_BookList): self.root_element.appendChild(node) tags = [] - for item in collections: + for item in self.collections: item = item.strip() mitem = getattr(book, item, None) titems = [] @@ -141,6 +162,7 @@ class BookList(_BookList): if hasattr(book, 'tag_order'): self.tag_order.update(book.tag_order) self.set_playlists(cid, tags) + return True # metadata cache has changed. Must sync at end def _delete_node(self, node): nid = node.getAttribute('id') @@ -162,7 +184,8 @@ class BookList(_BookList): def remove_book(self, book): ''' Remove DOM node corresponding to book with C{path == path}. - Also remove book from any collections it is part of. + Also remove book from any collections it is part of, and remove + from the booklist ''' self.remove(book) self.delete_node(book.lpath) @@ -264,15 +287,6 @@ class BookList(_BookList): stream.write(src.replace("'", ''')) def reorder_playlists(self): - sony_id_cache = {} - for child in self.root_element.childNodes: - if child.nodeType == child.ELEMENT_NODE and child.hasAttribute("id"): - sony_id_cache[child.getAttribute('id')] = child.getAttribute('path') - - books_lpath_cache = {} - for book in self: - books_lpath_cache[book.lpath] = book - for title in self.tag_order.keys(): pl = self.playlist_by_title(title) if not pl: @@ -281,9 +295,9 @@ class BookList(_BookList): sony_ids = [id.getAttribute('id') \ for id in pl.childNodes if hasattr(id, 'getAttribute')] # convert IDs in playlist to a list of lpaths - sony_paths = [sony_id_cache[id] for id in sony_ids] + sony_paths = [self.sony_id_cache[id] for id in sony_ids] # create list of books containing lpaths - books = [books_lpath_cache.get(p, None) for p in sony_paths] + books = [self.books_lpath_cache.get(p, None) for p in sony_paths] # create dict of db_id -> sony_id imap = {} for book, sony_id in zip(books, sony_ids): diff --git a/src/calibre/devices/prs505/driver.py b/src/calibre/devices/prs505/driver.py index 9ff88da592..d2823ff4a4 100644 --- a/src/calibre/devices/prs505/driver.py +++ b/src/calibre/devices/prs505/driver.py @@ -58,8 +58,6 @@ class PRS505(USBMS): 'series, tags, authors' EXTRA_CUSTOMIZATION_DEFAULT = ', '.join(['series', 'tags']) - METADATA_CACHE = "database/cache/metadata.calibre" - def windows_filter_pnp_id(self, pnp_id): return '_LAUNCHER' in pnp_id diff --git a/src/calibre/devices/usbms/books.py b/src/calibre/devices/usbms/books.py index ce74db6f54..b153300282 100644 --- a/src/calibre/devices/usbms/books.py +++ b/src/calibre/devices/usbms/books.py @@ -108,16 +108,19 @@ class Book(MetaInformation): class BookList(_BookList): + def __init__(self, oncard, prefix, settings): + pass + def supports_tags(self): return True def set_tags(self, book, tags): book.tags = tags - def add_book(self, book, collections=None): + def add_book(self, book, replace_metadata): ''' Add the book to the booklist. Intent is to maintain any device-internal - metadata + metadata. Return True if booklists must be sync'ed ''' if book not in self: self.append(book) diff --git a/src/calibre/devices/usbms/driver.py b/src/calibre/devices/usbms/driver.py index 64b27a993d..95b7441f44 100644 --- a/src/calibre/devices/usbms/driver.py +++ b/src/calibre/devices/usbms/driver.py @@ -58,20 +58,22 @@ class USBMS(CLI, Device): prefix = self._card_a_prefix if oncard == 'carda' else \ self._card_b_prefix if oncard == 'cardb' \ else self._main_prefix - metadata = self.booklist_class(oncard, prefix) ebook_dirs = self.EBOOK_DIR_CARD_A if oncard == 'carda' else \ self.EBOOK_DIR_CARD_B if oncard == 'cardb' else \ self.get_main_ebook_dir() - bl, need_sync = self.parse_metadata_cache(prefix, self.METADATA_CACHE, - self.booklist_class(oncard, prefix)) + # build a temporary list of books from the metadata cache + bl, need_sync = self.parse_metadata_cache(prefix, self.METADATA_CACHE) # make a dict cache of paths so the lookup in the loop below is faster. bl_cache = {} for idx,b in enumerate(bl): bl_cache[b.lpath] = idx self.count_found_in_bl = 0 + # Make the real booklist that will be filled in below + metadata = self.booklist_class(oncard, prefix, self.settings) + def update_booklist(filename, path, prefix): changed = False if path_to_ext(filename) in self.FORMATS: @@ -86,7 +88,8 @@ class USBMS(CLI, Device): else: item = self.book_from_path(prefix, lpath) changed = True - metadata.append(item) + if metadata.add_book(item, replace_metadata=False): + changed = True except: # Probably a filename encoding error import traceback traceback.print_exc() @@ -183,10 +186,7 @@ class USBMS(CLI, Device): if book.size is None: book.size = os.stat(path).st_size - opts = self.settings() - collections = opts.extra_customization.split(',') if opts.extra_customization else [] - booklists[blist].add_book(book, collections, *location[1:-1]) - + booklists[blist].add_book(book, replace_metadata=True) self.report_progress(1.0, _('Adding books to device metadata listing...')) def delete_books(self, paths, end_session=True): @@ -237,7 +237,8 @@ class USBMS(CLI, Device): self.report_progress(1.0, _('Sending metadata to device...')) @classmethod - def parse_metadata_cache(cls, prefix, name, bl): + def parse_metadata_cache(cls, prefix, name): + bl = [] js = [] need_sync = False try: diff --git a/src/calibre/gui2/library.py b/src/calibre/gui2/library.py index e116e39397..eeda687312 100644 --- a/src/calibre/gui2/library.py +++ b/src/calibre/gui2/library.py @@ -371,7 +371,7 @@ class BooksModel(QAbstractTableModel): def set_device_connected(self, is_connected): self.device_connected = is_connected self.read_config() - self.refresh(reset=True) + self.db.refresh_ondevice() self.database_changed.emit(self.db) def set_book_on_device_func(self, func): diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index 48e22f8903..ba6bac76e4 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -947,7 +947,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): self.device_manager.device) self.location_view.model().device_connected(self.device_manager.device) self.eject_action.setEnabled(True) - self.refresh_ondevice_info (device_connected = True) + # don't refresh_ondevice here. It will happen in metadata_downloaded else: self.save_device_view_settings() self.device_connected = False diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 9ed150733a..acc8eaffb6 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -547,6 +547,12 @@ class ResultCache(SearchQueryParser): def count(self): return len(self._map) + def refresh_ondevice(self, db): + ondevice_col = self.FIELD_MAP['ondevice'] + for item in self._data: + if item is not None: + item[ondevice_col] = db.book_on_device_string(item[0]) + def refresh(self, db, field=None, ascending=True): temp = db.conn.get('SELECT * FROM meta2') self._data = list(itertools.repeat(None, temp[-1][0]+2)) if temp else [] diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index b0f2d3cb39..5971333078 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -245,6 +245,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.has_id = self.data.has_id self.count = self.data.count + self.refresh_ondevice = functools.partial(self.data.refresh_ondevice, self) + self.refresh() self.last_update_check = self.last_modified()