1) improve performance of OnDevice refresh. Instead of rebuilding the complete book list, iterate through the existing one and set the value of OnDevice correctly.

2) Fix problems with Sony readers and metadata caching. Needed to ensure that when a book is added to the booklist from the JSON cache, it is added to the sony cache if it isn't already there.
3) Build the sony metadata maps (caches) on the fly instead of in reorder_playlists.
4) Refactor method declarations.
5) Move the JSON cache to the root of the card for Sony devices.

src/calibre/devices/prs505/books.py

@@ -10,9 +10,8 @@ from base64 import b64encode as encode
 
 from calibre.devices.usbms.books import BookList as _BookList
 from calibre.devices import strftime as _strftime
-from calibre.devices.usbms.books import Book as _Book
+from calibre.devices.prs505 import MEDIA_XML, CACHE_XML
-from calibre.devices.prs505 import MEDIA_XML
+from calibre.devices.errors import PathError
-from calibre.devices.prs505 import CACHE_XML
 
 strftime = functools.partial(_strftime, zone=time.gmtime)
 
@@ -33,10 +32,14 @@ def sortable_title(title):
 
 class BookList(_BookList):
 
-    def __init__(self, oncard, prefix):
+    def __init__(self, oncard, prefix, settings):
-        _BookList.__init__(self, oncard, prefix)
+        _BookList.__init__(self, oncard, prefix, settings)
         if prefix is None:
             return
+        self.sony_id_cache = {}
+        self.books_lpath_cache = {}
+        opts = settings()
+        self.collections = opts.extra_customization.split(',') if opts.extra_customization else []
         db = CACHE_XML if oncard else MEDIA_XML
         xml_file = open(prefix + db, 'rb')
         xml_file.seek(0)
@@ -50,8 +53,21 @@ class BookList(_BookList):
             self.root_element = records[0]
         else:
             self.prefix = ''
+        for child in self.root_element.childNodes:
+            if child.nodeType == child.ELEMENT_NODE and child.hasAttribute("id"):
+                self.sony_id_cache[child.getAttribute('id')] = child.getAttribute('path')
+                # set the key to none. Will be filled in later when booklist is built
+                self.books_lpath_cache[child.getAttribute('path')] = None
         self.tag_order = {}
 
+        paths = self.purge_corrupted_files()
+        for path in paths:
+            try:
+                self.del_file(path, end_session=False)
+            except PathError: # Incase this is a refetch without a sync in between
+                continue
+
+
     def max_id(self):
         max = 0
         for child in self.root_element.childNodes:
@@ -73,22 +89,27 @@ class BookList(_BookList):
     def supports_tags(self):
         return True
 
-    def book_by_path(self, path):
+    def add_book(self, book, replace_metadata):
-        for child in self.root_element.childNodes:
+        # Add a node into the DOM tree, representing a book. Also add to booklist
-            if child.nodeType == child.ELEMENT_NODE and child.hasAttribute("path"):
-                if path == child.getAttribute('path'):
-                    return child
-        return None
-
-    def add_book(self, book, collections=None):
         if book in self:
-            return
+            # replacing metadata for book
-        """ Add a node into the DOM tree, representing a book """
+            self.delete_node(book.lpath)
-        node = self.document.createElement(self.prefix + "text")
+        else:
-        mime = MIME_MAP.get(book.lpath.rpartition('.')[-1].lower(), MIME_MAP['epub'])
+            self.append(book)
+            if not replace_metadata:
+                if self.books_lpath_cache.has_key(book.lpath):
+                    self.books_lpath_cache[book.lpath] = book
+                    return
+        # Book not in metadata. Add it. Note that we don't need to worry about
+        # extra books in the Sony metadata. The reader deletes them for us when
+        # we disconnect. That said, if it becomes important one day, we can do
+        # it by scanning the books_lpath_cache for None entries and removing the
+        # corresponding nodes.
+        self.books_lpath_cache[book.lpath] = book
         cid = self.max_id()+1
-        book.sony_id = cid
+        node = self.document.createElement(self.prefix + "text")
-        self.append(book)
+        self.sony_id_cache[cid] = book.lpath
+        mime = MIME_MAP.get(book.lpath.rpartition('.')[-1].lower(), MIME_MAP['epub'])
         try:
             sourceid = str(self[0].sourceid) if len(self) else '1'
         except:
@@ -120,7 +141,7 @@ class BookList(_BookList):
         self.root_element.appendChild(node)
 
         tags = []
-        for item in collections:
+        for item in self.collections:
             item = item.strip()
             mitem = getattr(book, item, None)
             titems = []
@@ -141,6 +162,7 @@ class BookList(_BookList):
             if hasattr(book, 'tag_order'):
                 self.tag_order.update(book.tag_order)
             self.set_playlists(cid, tags)
+        return True  # metadata cache has changed. Must sync at end
 
     def _delete_node(self, node):
         nid = node.getAttribute('id')
@@ -162,7 +184,8 @@ class BookList(_BookList):
     def remove_book(self, book):
         '''
         Remove DOM node corresponding to book with C{path == path}.
-        Also remove book from any collections it is part of.
+        Also remove book from any collections it is part of, and remove
+        from the booklist
         '''
         self.remove(book)
         self.delete_node(book.lpath)
@@ -264,15 +287,6 @@ class BookList(_BookList):
         stream.write(src.replace("'", '''))
 
     def reorder_playlists(self):
-        sony_id_cache = {}
-        for child in self.root_element.childNodes:
-            if child.nodeType == child.ELEMENT_NODE and child.hasAttribute("id"):
-                sony_id_cache[child.getAttribute('id')] = child.getAttribute('path')
-
-        books_lpath_cache = {}
-        for book in self:
-            books_lpath_cache[book.lpath] = book
-
         for title in self.tag_order.keys():
             pl = self.playlist_by_title(title)
             if not pl:
@@ -281,9 +295,9 @@ class BookList(_BookList):
             sony_ids = [id.getAttribute('id') \
                     for id in pl.childNodes if hasattr(id, 'getAttribute')]
             # convert IDs in playlist to a list of lpaths
-            sony_paths = [sony_id_cache[id] for id in sony_ids]
+            sony_paths = [self.sony_id_cache[id] for id in sony_ids]
             # create list of books containing lpaths
-            books = [books_lpath_cache.get(p, None) for p in sony_paths]
+            books = [self.books_lpath_cache.get(p, None) for p in sony_paths]
             # create dict of db_id -> sony_id
             imap = {}
             for book, sony_id in zip(books, sony_ids):

src/calibre/devices/prs505/driver.py

@@ -58,8 +58,6 @@ class PRS505(USBMS):
                     'series, tags, authors'
     EXTRA_CUSTOMIZATION_DEFAULT = ', '.join(['series', 'tags'])
 
-    METADATA_CACHE = "database/cache/metadata.calibre"
-
     def windows_filter_pnp_id(self, pnp_id):
         return '_LAUNCHER' in pnp_id
 

src/calibre/devices/usbms/books.py

@@ -108,16 +108,19 @@ class Book(MetaInformation):
 
 class BookList(_BookList):
 
+    def __init__(self, oncard, prefix, settings):
+        pass
+
     def supports_tags(self):
         return True
 
     def set_tags(self, book, tags):
         book.tags = tags
 
-    def add_book(self, book, collections=None):
+    def add_book(self, book, replace_metadata):
         '''
         Add the book to the booklist. Intent is to maintain any device-internal
-        metadata
+        metadata. Return True if booklists must be sync'ed
         '''
         if book not in self:
             self.append(book)

src/calibre/devices/usbms/driver.py

@@ -58,20 +58,22 @@ class USBMS(CLI, Device):
         prefix = self._card_a_prefix if oncard == 'carda' else \
                                      self._card_b_prefix if oncard == 'cardb' \
                                                          else self._main_prefix
-        metadata = self.booklist_class(oncard, prefix)
 
         ebook_dirs = self.EBOOK_DIR_CARD_A if oncard == 'carda' else \
             self.EBOOK_DIR_CARD_B if oncard == 'cardb' else \
             self.get_main_ebook_dir()
 
-        bl, need_sync = self.parse_metadata_cache(prefix, self.METADATA_CACHE,
+        # build a temporary list of books from the metadata cache
-                                                  self.booklist_class(oncard, prefix))
+        bl, need_sync = self.parse_metadata_cache(prefix, self.METADATA_CACHE)
         # make a dict cache of paths so the lookup in the loop below is faster.
         bl_cache = {}
         for idx,b in enumerate(bl):
             bl_cache[b.lpath] = idx
         self.count_found_in_bl = 0
 
+        # Make the real booklist that will be filled in below
+        metadata = self.booklist_class(oncard, prefix, self.settings)
+
         def update_booklist(filename, path, prefix):
             changed = False
             if path_to_ext(filename) in self.FORMATS:
@@ -86,7 +88,8 @@ class USBMS(CLI, Device):
                     else:
                         item = self.book_from_path(prefix, lpath)
                         changed = True
-                    metadata.append(item)
+                    if metadata.add_book(item, replace_metadata=False):
+                        changed = True
                 except: # Probably a filename encoding error
                     import traceback
                     traceback.print_exc()
@@ -183,10 +186,7 @@ class USBMS(CLI, Device):
             if book.size is None:
                 book.size = os.stat(path).st_size
 
-            opts = self.settings()
+            booklists[blist].add_book(book, replace_metadata=True)
-            collections = opts.extra_customization.split(',') if opts.extra_customization else []
-            booklists[blist].add_book(book, collections, *location[1:-1])
-
         self.report_progress(1.0, _('Adding books to device metadata listing...'))
 
     def delete_books(self, paths, end_session=True):
@@ -237,7 +237,8 @@ class USBMS(CLI, Device):
         self.report_progress(1.0, _('Sending metadata to device...'))
 
     @classmethod
-    def parse_metadata_cache(cls, prefix, name, bl):
+    def parse_metadata_cache(cls, prefix, name):
+        bl = []
         js = []
         need_sync = False
         try:

src/calibre/gui2/library.py

@@ -371,7 +371,7 @@ class BooksModel(QAbstractTableModel):
     def set_device_connected(self, is_connected):
         self.device_connected = is_connected
         self.read_config()
-        self.refresh(reset=True)
+        self.db.refresh_ondevice()
         self.database_changed.emit(self.db)
 
     def set_book_on_device_func(self, func):

src/calibre/gui2/ui.py

@@ -947,7 +947,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
                     self.device_manager.device)
             self.location_view.model().device_connected(self.device_manager.device)
             self.eject_action.setEnabled(True)
-            self.refresh_ondevice_info (device_connected = True)
+            # don't refresh_ondevice here. It will happen in metadata_downloaded
         else:
             self.save_device_view_settings()
             self.device_connected = False

src/calibre/library/caches.py

@@ -547,6 +547,12 @@ class ResultCache(SearchQueryParser):
     def count(self):
         return len(self._map)
 
+    def refresh_ondevice(self, db):
+        ondevice_col = self.FIELD_MAP['ondevice']
+        for item in self._data:
+            if item is not None:
+                item[ondevice_col] = db.book_on_device_string(item[0])
+
     def refresh(self, db, field=None, ascending=True):
         temp = db.conn.get('SELECT * FROM meta2')
         self._data = list(itertools.repeat(None, temp[-1][0]+2)) if temp else []

src/calibre/library/database2.py

@@ -245,6 +245,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         self.has_id  = self.data.has_id
         self.count   = self.data.count
 
+        self.refresh_ondevice = functools.partial(self.data.refresh_ondevice, self)
+
         self.refresh()
         self.last_update_check = self.last_modified()