More testing of:

1) initial condition: cache does not exist on book
2) adding and removing books
3) subsequent conditions: cache exists

In addition:
1) added metadata correction for books matched with something other than UUID.
2) Refactored changes to BookList, to move the additional methods into USMBS from Interface.
3) Made classmethods in USBMS into normal methods.

src/calibre/devices/interface.py

@@ -402,17 +402,3 @@ class BookList(list):
         '''
         raise NotImplementedError()
 
-    def add_book(self, book, collections=None):
-        '''
-        Add the book to the booklist. Intent is to maintain any device-internal
-        metadata
-        '''
-        if book not in self:
-            self.append(book)
-
-    def remove_book(self, book):
-        '''
-        Remove a book from the booklist. Correct any device metadata at the
-        same time
-        '''
-        self.remove(book)

src/calibre/devices/prs505/books.py

@@ -8,7 +8,7 @@ import xml.dom.minidom as dom
 from base64 import b64encode as encode
 
 
-from calibre.devices.interface import BookList as _BookList
+from calibre.devices.usbms.books import BookList as _BookList
 from calibre.devices import strftime as _strftime
 from calibre.devices.usbms.books import Book as _Book
 from calibre.devices.prs505 import MEDIA_XML
@@ -31,36 +31,6 @@ def uuid():
 def sortable_title(title):
     return re.sub('^\s*A\s+|^\s*The\s+|^\s*An\s+', '', title).rstrip()
 
-class book_metadata_field(object):
-    """ Represents metadata stored as an attribute """
-    def __init__(self, attr, formatter=None, setter=None):
-        self.attr = attr
-        self.formatter = formatter
-        self.setter = setter
-
-    def __get__(self, obj, typ=None):
-        """ Return a string. String may be empty if self.attr is absent """
-        return self.formatter(obj.elem.getAttribute(self.attr)) if \
-                           self.formatter else obj.elem.getAttribute(self.attr).strip()
-
-    def __set__(self, obj, val):
-        """ Set the attribute """
-        val = self.setter(val) if self.setter else val
-        if not isinstance(val, unicode):
-            val = unicode(val, 'utf8', 'replace')
-        obj.elem.setAttribute(self.attr, val)
-
-
-class Book(_Book):
-    @dynamic_property
-    def db_id(self):
-        doc = '''The database id in the application database that this file corresponds to'''
-        def fget(self):
-            match = re.search(r'_(\d+)$', self.rpath.rpartition('.')[0])
-            if match:
-                return int(match.group(1))
-        return property(fget=fget, doc=doc)
-
 class BookList(_BookList):
 
     def __init__(self, oncard, prefix):
@@ -318,6 +288,11 @@ class BookList(_BookList):
             imap = {}
             for book, sony_id in zip(books, sony_ids):
                 if book is not None:
+                    db_id = book.application_id
+                    if db_id is None:
+                        db_id = book.db_id
+                        print 'here', db_id
+                    if db_id is not None:
                         imap[book.application_id] = sony_id
             # filter the list, removing books not on device but on playlist
             books = [i for i in books if i is not None]

src/calibre/devices/prs505/driver.py

@@ -13,6 +13,7 @@ import os
 import re
 
 from calibre.devices.usbms.driver import USBMS
+from calibre.devices.usbms.books import Book
 from calibre.devices.prs505.books import BookList, fix_ids
 from calibre.devices.prs505 import MEDIA_XML
 from calibre.devices.prs505 import CACHE_XML
@@ -59,8 +60,9 @@ class PRS505(USBMS):
     METADATA_CACHE = "database/cache/metadata.calibre"
 
     def initialize(self):
-        USBMS.initialize(self)
+        USBMS.initialize(self) # Must be first, so _class vars are set right
         self.booklist_class = BookList
+        self.book_class = Book
 
     def windows_filter_pnp_id(self, pnp_id):
         return '_LAUNCHER' in pnp_id

src/calibre/devices/usbms/books.py

@@ -37,11 +37,7 @@ class Book(MetaInformation):
         else:
             self.lpath = lpath
         self.mime = mime_type_ext(path_to_ext(lpath))
-        self.size = os.stat(self.path).st_size if size == None else size
+        self.size = None # will be set later
-        self.db_id = None
-        try:
-            self.datetime = time.gmtime(os.path.getctime(self.path))
-        except ValueError:
         self.datetime = time.gmtime()
 
         if other:
@@ -70,6 +66,16 @@ class Book(MetaInformation):
 
         return spath == opath
 
+    @dynamic_property
+    def db_id(self):
+        doc = '''The database id in the application database that this file corresponds to'''
+        def fget(self):
+            match = re.search(r'_(\d+)$', self.lpath.rpartition('.')[0])
+            if match:
+                return int(match.group(1))
+            return None
+        return property(fget=fget, doc=doc)
+
     @dynamic_property
     def title_sorter(self):
         doc = '''String to sort the title. If absent, title is returned'''
@@ -81,13 +87,6 @@ class Book(MetaInformation):
     def thumbnail(self):
         return None
 
-#    def __str__(self):
-#        '''
-#        Return a utf-8 encoded string with title author and path information
-#        '''
-#        return self.title.encode('utf-8') + " by " + \
-#               self.authors.encode('utf-8') + " at " + self.path.encode('utf-8')
-
     def smart_update(self, other):
         '''
         Merge the information in C{other} into self. In case of conflicts, the information
@@ -115,3 +114,17 @@ class BookList(_BookList):
     def set_tags(self, book, tags):
         book.tags = tags
 
+    def add_book(self, book, collections=None):
+        '''
+        Add the book to the booklist. Intent is to maintain any device-internal
+        metadata
+        '''
+        if book not in self:
+            self.append(book)
+
+    def remove_book(self, book):
+        '''
+        Remove a book from the booklist. Correct any device metadata at the
+        same time
+        '''
+        self.remove(book)

src/calibre/devices/usbms/driver.py

@@ -34,6 +34,7 @@ class USBMS(CLI, Device):
     def initialize(self):
         Device.initialize(self)
         self.booklist_class = BookList
+        self.book_class = Book
 
     def get_device_information(self, end_session=True):
         self.report_progress(1.0, _('Get device information...'))
@@ -52,7 +53,9 @@ class USBMS(CLI, Device):
             self.report_progress(1.0, _('Getting list of books on device...'))
             return []
 
-        prefix = self._card_a_prefix if oncard == 'carda' else self._card_b_prefix if oncard == 'cardb' else self._main_prefix
+        prefix = self._card_a_prefix if oncard == 'carda' else \
+                                     self._card_b_prefix if oncard == 'cardb' \
+                                                         else self._main_prefix
         metadata = self.booklist_class(oncard, prefix)
 
         ebook_dirs = self.EBOOK_DIR_CARD_A if oncard == 'carda' else \
@@ -61,7 +64,6 @@ class USBMS(CLI, Device):
 
         bl, need_sync = self.parse_metadata_cache(prefix, self.METADATA_CACHE,
                                                   self.booklist_class(oncard, prefix))
-
         # make a dict cache of paths so the lookup in the loop below is faster.
         bl_cache = {}
         for idx,b in enumerate(bl):
@@ -77,10 +79,10 @@ class USBMS(CLI, Device):
                         lpath = lpath[len(os.sep):]
                     idx = bl_cache.get(lpath.replace('\\', '/'), None)
                     if idx is not None:
-                        item, changed = self.__class__.update_metadata_item(bl[idx])
+                        item, changed = self.update_metadata_item(bl[idx])
                         self.count_found_in_bl += 1
                     else:
-                        item = self.__class__.book_from_path(prefix, lpath)
+                        item = self.book_from_path(prefix, lpath)
                         changed = True
                     metadata.append(item)
                 except: # Probably a filename encoding error
@@ -175,7 +177,10 @@ class USBMS(CLI, Device):
             lpath = path.partition(prefix)[2]
             if lpath.startswith(os.sep):
                 lpath = lpath[len(os.sep):]
-            book = Book(prefix, lpath, other=info)
+            book = self.book_class(prefix, lpath, other=info)
+            if book.size is None:
+                book.size = os.stat(path).st_size
+
             opts = self.settings()
             collections = opts.extra_customization.split(',') if opts.extra_customization else []
             booklists[blist].add_book(book, collections, *location[1:-1])
@@ -229,19 +234,14 @@ class USBMS(CLI, Device):
 
         self.report_progress(1.0, _('Sending metadata to device...'))
 
-    @classmethod
+    def parse_metadata_cache(self, prefix, name, bl):
-    def parse_metadata_cache(cls, prefix, name, bl):
         js = []
         need_sync = False
         try:
             with open(os.path.join(prefix, name), 'rb') as f:
                 js = json.load(f, encoding='utf-8')
             for item in js:
-                lpath = item.get('lpath', None)
+                book = self.book_class(prefix, item.get('lpath', None))
-                if not lpath or not os.path.exists(os.path.join(prefix, lpath)):
-                    need_sync = True
-                    continue
-                book = Book(prefix, lpath)
                 for key in item.keys():
                     setattr(book, key, item[key])
                 bl.append(book)
@@ -249,35 +249,33 @@ class USBMS(CLI, Device):
             import traceback
             traceback.print_exc()
             bl = []
+            need_sync = True
         return bl, need_sync
 
-    @classmethod
+    def update_metadata_item(self, item):
-    def update_metadata_item(cls, item):
         changed = False
         size = os.stat(item.path).st_size
         if size != item.size:
             changed = True
-            mi = cls.metadata_from_path(item.path)
+            mi = self.metadata_from_path(item.path)
             item.smart_update(mi)
+            item.size = size
         return item, changed
 
-    @classmethod
+    def metadata_from_path(self, path):
-    def metadata_from_path(cls, path):
+        return self.metadata_from_formats([path])
-        return cls.metadata_from_formats([path])
 
-    @classmethod
+    def metadata_from_formats(self, fmts):
-    def metadata_from_formats(cls, fmts):
         from calibre.ebooks.metadata.meta import metadata_from_formats
         from calibre.customize.ui import quick_metadata
         with quick_metadata:
             return metadata_from_formats(fmts)
 
-    @classmethod
+    def book_from_path(self, prefix, path):
-    def book_from_path(cls, prefix, path):
         from calibre.ebooks.metadata import MetaInformation
 
-        if cls.settings().read_metadata or cls.MUST_READ_METADATA:
+        if self.settings().read_metadata or self.MUST_READ_METADATA:
-            mi = cls.metadata_from_path(os.path.join(prefix, path))
+            mi = self.metadata_from_path(os.path.join(prefix, path))
         else:
             from calibre.ebooks.metadata.meta import metadata_from_filename
             mi = metadata_from_filename(os.path.basename(path),
@@ -286,6 +284,6 @@ class USBMS(CLI, Device):
         if mi is None:
             mi = MetaInformation(os.path.splitext(os.path.basename(path))[0],
                     [_('Unknown')])
-
+        mi.size = os.stat(os.path.join(prefix, path)).st_size
-        book = Book(prefix, path, other=mi)
+        book = self.book_class(prefix, path, other=mi)
         return book

src/calibre/gui2/device.py

@@ -523,6 +523,7 @@ class DeviceGUI(object):
             d = ChooseFormatDialog(self, _('Choose format to send to device'),
                                 self.device_manager.device.settings().format_map)
             d.exec_()
+            if d.format():
                 fmt = d.format().lower()
         dest, sub_dest = dest.split(':')
         if dest in ('main', 'carda', 'cardb'):
@@ -998,7 +999,7 @@ class DeviceGUI(object):
             if changed:
                 self.library_view.model().refresh_ids(list(changed))
 
-    def book_on_device(self, index, format=None, reset=False):
+    def book_on_device(self, id, format=None, reset=False):
         loc = [None, None, None]
 
         if reset:
@@ -1030,7 +1031,7 @@ class DeviceGUI(object):
                     if uuid is not None:
                         self.book_db_uuid_cache[i].add(uuid)
 
-        mi = self.library_view.model().db.get_metadata(index, index_is_id=True)
+        mi = self.library_view.model().db.get_metadata(id, index_is_id=True)
         for i, l in enumerate(self.booklists()):
             if mi.uuid in self.book_db_uuid_cache[i]:
                 loc[i] = True
@@ -1038,7 +1039,7 @@ class DeviceGUI(object):
             db_title = re.sub('(?u)\W|[_]', '', mi.title.lower())
             cache = self.book_db_title_cache[i].get(db_title, None)
             if cache:
-                if index in cache['db_ids']:
+                if id in cache['db_ids']:
                     loc[i] = True
                     break
                 if mi.authors and \
@@ -1057,11 +1058,11 @@ class DeviceGUI(object):
                 mi = self.library_view.model().db.get_metadata(idx, index_is_id=False)
                 title = re.sub('(?u)\W|[_]', '', mi.title.lower())
                 if title not in self.db_book_title_cache:
-                    self.db_book_title_cache[title] = {'authors':set(), 'db_ids':set()}
+                    self.db_book_title_cache[title] = {'authors':{}, 'db_ids':{}}
                 authors = authors_to_string(mi.authors).lower() if mi.authors else ''
                 authors = re.sub('(?u)\W|[_]', '', authors)
-                self.db_book_title_cache[title]['authors'].add(authors)
+                self.db_book_title_cache[title]['authors'][authors] = mi
-                self.db_book_title_cache[title]['db_ids'].add(mi.application_id)
+                self.db_book_title_cache[title]['db_ids'][mi.application_id] = mi
                 self.db_book_uuid_cache.add(mi.uuid)
 
         # Now iterate through all the books on the device, setting the in_library field
@@ -1069,6 +1070,7 @@ class DeviceGUI(object):
         # is really the db key, but as this can accidentally match across libraries we
         # also verify the title. The db_id exists on Sony devices. Fallback is title
         # and author match
+        resend_metadata = False
         for booklist in booklists:
             for book in booklist:
                 if getattr(book, 'uuid', None) in self.db_book_uuid_cache:
@@ -1082,11 +1084,20 @@ class DeviceGUI(object):
                 if d is not None:
                     if getattr(book, 'application_id', None) in d['db_ids']:
                         book.in_library = True
+                        book.smart_update(d['db_ids'][book.application_id])
+                        resend_metadata = True
                         continue
                     if book.db_id in d['db_ids']:
                         book.in_library = True
+                        book.smart_update(d['db_ids'][book.db_id])
+                        resend_metadata = True
                         continue
                     book_authors = authors_to_string(book.authors).lower() if book.authors else ''
                     book_authors = re.sub('(?u)\W|[_]', '', book_authors)
                     if book_authors in d['authors']:
                         book.in_library = True
+                        book.smart_update(d['authors'][book_authors])
+                        resend_metadata = True
+        if resend_metadata:
+            # Correcting metadata cache on device.
+            self.device_manager.sync_booklists(None, booklists)

src/calibre/library/database2.py

@@ -470,14 +470,14 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             im = PILImage.open(f)
             im.convert('RGB').save(path, 'JPEG')
 
-    def book_on_device(self, index):
+    def book_on_device(self, id):
         if callable(self.book_on_device_func):
-            return self.book_on_device_func(index)
+            return self.book_on_device_func(id)
         return None
 
-    def book_on_device_string(self, index):
+    def book_on_device_string(self, id):
         loc = []
-        on = self.book_on_device(index)
+        on = self.book_on_device(id)
         if on is not None:
             m, a, b = on
             if m is not None: