Pull from chaley custcols for metadata caching, connect to folder

src/calibre/customize/builtins.py

@@ -455,7 +455,7 @@ from calibre.devices.edge.driver import EDGE
 from calibre.devices.teclast.driver import TECLAST_K3
 from calibre.devices.sne.driver import SNE
 from calibre.devices.misc import PALMPRE, KOBO, AVANT
-from calibre.devices.htc_td2.driver import HTC_TD2
+from calibre.devices.folder_device.driver import FOLDER_DEVICE_FOR_CONFIG
 
 from calibre.ebooks.metadata.fetch import GoogleBooks, ISBNDB, Amazon
 from calibre.library.catalog import CSV_XML, EPUB_MOBI
@@ -540,7 +540,7 @@ plugins += [
     PALMPRE,
     KOBO,
     AZBOOKA,
-    HTC_TD2,
+    FOLDER_DEVICE_FOR_CONFIG,
     AVANT,
 ]
 plugins += [x for x in list(locals().values()) if isinstance(x, type) and \

src/calibre/devices/folder_device/driver.py

@@ -0,0 +1,96 @@
+'''
+Created on 15 May 2010
+
+@author: charles
+'''
+import os
+
+from calibre.devices.usbms.driver import USBMS, BookList
+
+# This class is added to the standard device plugin chain, so that it can
+# be configured. It has invalid vendor_id etc, so it will never match a
+# device. The 'real' FOLDER_DEVICE will use the config from it.
+class FOLDER_DEVICE_FOR_CONFIG(USBMS):
+    name           = 'Folder Device Interface'
+    gui_name       = 'Folder Device'
+    description    = _('Use an arbitrary folder as a device.')
+    author         = 'John Schember/Charles Haley'
+    supported_platforms = ['windows', 'osx', 'linux']
+    FORMATS     = ['epub', 'fb2', 'mobi', 'lrf', 'tcr', 'pmlz', 'lit', 'rtf', 'rb', 'pdf', 'oeb', 'txt', 'pdb']
+    VENDOR_ID   = 0xffff
+    PRODUCT_ID  = 0xffff
+    BCD         = 0xffff
+
+
+class FOLDER_DEVICE(USBMS):
+    type = _('Device Interface')
+
+    name           = 'Folder Device Interface'
+    gui_name       = 'Folder Device'
+    description    = _('Use an arbitrary folder as a device.')
+    author         = 'John Schember/Charles Haley'
+    supported_platforms = ['windows', 'osx', 'linux']
+    FORMATS     = ['epub', 'fb2', 'mobi', 'lrf', 'tcr', 'pmlz', 'lit', 'rtf', 'rb', 'pdf', 'oeb', 'txt', 'pdb']
+
+    VENDOR_ID   = 0xffff
+    PRODUCT_ID  = 0xffff
+    BCD         = 0xffff
+
+    THUMBNAIL_HEIGHT = 68 # Height for thumbnails on device
+
+    CAN_SET_METADATA = True
+    SUPPORTS_SUB_DIRS = True
+
+    #: Icon for this device
+    icon = I('sd.svg')
+    METADATA_CACHE = '.metadata.calibre'
+
+    _main_prefix = ''
+    _card_a_prefix = None
+    _card_b_prefix = None
+
+    is_connected = False
+
+    def __init__(self, path):
+        if not os.path.isdir(path):
+            raise IOError, 'Path is not a folder'
+        self._main_prefix = path
+        self.booklist_class = BookList
+        self.is_connected = True
+
+    @classmethod
+    def get_gui_name(cls):
+        if hasattr(cls, 'gui_name'):
+            return cls.gui_name
+        if hasattr(cls, '__name__'):
+            return cls.__name__
+        return cls.name
+
+    def disconnect_from_folder(self):
+        self._main_prefix = ''
+        self.is_connected = False
+
+    def is_usb_connected(self, devices_on_system, debug=False,
+            only_presence=False):
+        return self.is_connected, self
+
+    def open(self):
+        if not self._main_prefix:
+            return False
+        return True
+
+    def set_progress_reporter(self, report_progress):
+        self.report_progress = report_progress
+
+    def card_prefix(self, end_session=True):
+        return (None, None)
+
+    def get_main_ebook_dir(self):
+        return ''
+
+    def eject(self):
+        self.is_connected = False
+
+    @classmethod
+    def settings(self):
+        return FOLDER_DEVICE_FOR_CONFIG._config().parse()

src/calibre/devices/htc_td2/driver.py

@@ -1,44 +0,0 @@
-# -*- coding: utf-8 -*-
-
-__license__   = 'GPL v3'
-__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
-__docformat__ = 'restructuredtext en'
-
-from calibre.devices.usbms.driver import USBMS
-
-class HTC_TD2(USBMS):
-
-    name           = 'HTC TD2 Phone driver'
-    gui_name       = 'HTC TD2'
-    description    = _('Communicate with HTC TD2 phones.')
-    author         = 'Charles Haley'
-    supported_platforms = ['osx', 'linux']
-
-    # Ordered list of supported formats
-    FORMATS     = ['epub', 'pdf']
-
-    VENDOR_ID   = {
-            # HTC
-            0x0bb4 : { 0x0c30 : [0x000]},
-            }
-    EBOOK_DIR_MAIN = ['EBooks']
-    EXTRA_CUSTOMIZATION_MESSAGE = _('Comma separated list of directories to '
-            'send e-books to on the device. The first one that exists will '
-            'be used')
-    EXTRA_CUSTOMIZATION_DEFAULT = ', '.join(EBOOK_DIR_MAIN)
-
-    VENDOR_NAME      = ['']
-    WINDOWS_MAIN_MEM = ['']
-
-    MAIN_MEMORY_VOLUME_LABEL  = 'HTC Phone Internal Memory'
-
-    SUPPORTS_SUB_DIRS = True
-
-    def post_open_callback(self):
-        opts = self.settings()
-        dirs = opts.extra_customization
-        if not dirs:
-            dirs = self.EBOOK_DIR_MAIN
-        else:
-            dirs = [x.strip() for x in dirs.split(',')]
-        self.EBOOK_DIR_MAIN = dirs

src/calibre/devices/interface.py

@@ -387,6 +387,9 @@ class BookList(list):
     __getslice__ = None
     __setslice__ = None
 
+    def __init__(self, oncard, prefix):
+        pass
+
     def supports_tags(self):
         ''' Return True if the the device supports tags (collections) for this book list. '''
         raise NotImplementedError()

src/calibre/devices/prs505/__init__.py

@@ -1,2 +1,6 @@
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
+
+MEDIA_XML = 'database/cache/media.xml'
+
+CACHE_XML = 'Sony Reader/database/cache.xml'

src/calibre/devices/prs505/books.py

@@ -5,13 +5,13 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 import re, time, functools
 from uuid import uuid4 as _uuid
 import xml.dom.minidom as dom
-from base64 import b64decode as decode
 from base64 import b64encode as encode
 
 
-from calibre.devices.interface import BookList as _BookList
+from calibre.devices.usbms.books import BookList as _BookList
 from calibre.devices import strftime as _strftime
-from calibre.devices import strptime
+from calibre.devices.prs505 import MEDIA_XML, CACHE_XML
+from calibre.devices.errors import PathError
 
 strftime = functools.partial(_strftime, zone=time.gmtime)
 
@@ -30,127 +30,43 @@ def uuid():
 def sortable_title(title):
     return re.sub('^\s*A\s+|^\s*The\s+|^\s*An\s+', '', title).rstrip()
 
-class book_metadata_field(object):
-    """ Represents metadata stored as an attribute """
-    def __init__(self, attr, formatter=None, setter=None):
-        self.attr = attr
-        self.formatter = formatter
-        self.setter = setter
-
-    def __get__(self, obj, typ=None):
-        """ Return a string. String may be empty if self.attr is absent """
-        return self.formatter(obj.elem.getAttribute(self.attr)) if \
-                           self.formatter else obj.elem.getAttribute(self.attr).strip()
-
-    def __set__(self, obj, val):
-        """ Set the attribute """
-        val = self.setter(val) if self.setter else val
-        if not isinstance(val, unicode):
-            val = unicode(val, 'utf8', 'replace')
-        obj.elem.setAttribute(self.attr, val)
-
-
-class Book(object):
-    """ Provides a view onto the XML element that represents a book """
-
-    title        = book_metadata_field("title")
-    authors      = book_metadata_field("author", \
-                            formatter=lambda x: [x if x and x.strip() else _('Unknown')])
-    mime         = book_metadata_field("mime")
-    rpath        = book_metadata_field("path")
-    id           = book_metadata_field("id", formatter=int)
-    sourceid     = book_metadata_field("sourceid", formatter=int)
-    size         = book_metadata_field("size", formatter=lambda x : int(float(x)))
-    # When setting this attribute you must use an epoch
-    datetime     = book_metadata_field("date", formatter=strptime, setter=strftime)
-
-    @dynamic_property
-    def title_sorter(self):
-        doc = '''String to sort the title. If absent, title is returned'''
-        def fget(self):
-            src = self.elem.getAttribute('titleSorter').strip()
-            if not src:
-                src = self.title
-            return src
-        def fset(self, val):
-            self.elem.setAttribute('titleSorter', sortable_title(unicode(val)))
-        return property(doc=doc, fget=fget, fset=fset)
-
-    @dynamic_property
-    def thumbnail(self):
-        doc = \
-        """
-        The thumbnail. Should be a height 68 image.
-        Setting is not supported.
-        """
-        def fget(self):
-            th = self.elem.getElementsByTagName(self.prefix + "thumbnail")
-            if not len(th):
-                th = self.elem.getElementsByTagName("cache:thumbnail")
-            if len(th):
-                for n in th[0].childNodes:
-                    if n.nodeType == n.ELEMENT_NODE:
-                        th = n
-                        break
-                rc = ""
-                for node in th.childNodes:
-                    if node.nodeType == node.TEXT_NODE:
-                        rc += node.data
-                return decode(rc)
-        return property(fget=fget, doc=doc)
-
-    @dynamic_property
-    def path(self):
-        doc = """ Absolute path to book on device. Setting not supported. """
-        def fget(self):
-            return self.mountpath + self.rpath
-        return property(fget=fget, doc=doc)
-
-    @dynamic_property
-    def db_id(self):
-        doc = '''The database id in the application database that this file corresponds to'''
-        def fget(self):
-            match = re.search(r'_(\d+)$', self.rpath.rpartition('.')[0])
-            if match:
-                return int(match.group(1))
-        return property(fget=fget, doc=doc)
-
-    def __init__(self, node, mountpath, tags, prefix=""):
-        self.elem      = node
-        self.prefix    = prefix
-        self.tags      = tags
-        self.mountpath = mountpath
-
-    def __str__(self):
-        """ Return a utf-8 encoded string with title author and path information """
-        return self.title.encode('utf-8') + " by " + \
-               self.authors.encode('utf-8') + " at " + self.path.encode('utf-8')
-
-
 class BookList(_BookList):
 
-    def __init__(self, xml_file, mountpath, report_progress=None):
+    def __init__(self, oncard, prefix, settings):
-        _BookList.__init__(self)
+        _BookList.__init__(self, oncard, prefix, settings)
-        xml_file.seek(0)
+        if prefix is None:
-        self.document = dom.parse(xml_file)
+            return
+        self.sony_id_cache = {}
+        self.books_lpath_cache = {}
+        opts = settings()
+        self.collections = opts.extra_customization.split(',') if opts.extra_customization else []
+        db = CACHE_XML if oncard else MEDIA_XML
+        with open(prefix + db, 'rb') as xml_file:
+            xml_file.seek(0)
+            self.document = dom.parse(xml_file)
         self.root_element = self.document.documentElement
-        self.mountpath = mountpath
+        self.mountpath = prefix
         records = self.root_element.getElementsByTagName('records')
-        self.tag_order = {}
 
         if records:
             self.prefix = 'xs1:'
             self.root_element = records[0]
         else:
             self.prefix = ''
+        for child in self.root_element.childNodes:
+            if child.nodeType == child.ELEMENT_NODE and child.hasAttribute("id"):
+                self.sony_id_cache[child.getAttribute('id')] = child.getAttribute('path')
+                # set the key to none. Will be filled in later when booklist is built
+                self.books_lpath_cache[child.getAttribute('path')] = None
+        self.tag_order = {}
+
+        paths = self.purge_corrupted_files()
+        for path in paths:
+            try:
+                self.del_file(path, end_session=False)
+            except PathError: # Incase this is a refetch without a sync in between
+                continue
 
-        nodes = self.root_element.childNodes
-        for i, book in enumerate(nodes):
-            if report_progress:
-                report_progress((i+1) / float(len(nodes)), _('Getting list of books on device...'))
-            if hasattr(book, 'tagName') and book.tagName.endswith('text'):
-                tags = [i.getAttribute('title') for i in self.get_playlists(book.getAttribute('id'))]
-                self.append(Book(book, mountpath, tags, prefix=self.prefix))
 
     def max_id(self):
         max = 0
@@ -173,39 +89,44 @@ class BookList(_BookList):
     def supports_tags(self):
         return True
 
-    def book_by_path(self, path):
+    def add_book(self, book, replace_metadata):
-        for child in self.root_element.childNodes:
+        # Add a node into the DOM tree, representing a book. Also add to booklist
-            if child.nodeType == child.ELEMENT_NODE and child.hasAttribute("path"):
+        if book in self:
-                if path == child.getAttribute('path'):
+            # replacing metadata for book
-                    return child
+            self.delete_node(book.lpath)
-        return None
+        else:
-
+            self.append(book)
-    def add_book(self, mi, name, collections, size, ctime):
+            if not replace_metadata:
-        """ Add a node into the DOM tree, representing a book """
+                if self.books_lpath_cache.has_key(book.lpath):
-        book = self.book_by_path(name)
+                    self.books_lpath_cache[book.lpath] = book
-        if book is not None:
+                    return
-            self.remove_book(name)
+        # Book not in metadata. Add it. Note that we don't need to worry about
-
+        # extra books in the Sony metadata. The reader deletes them for us when
-        node = self.document.createElement(self.prefix + "text")
+        # we disconnect. That said, if it becomes important one day, we can do
-        mime = MIME_MAP.get(name.rpartition('.')[-1].lower(), MIME_MAP['epub'])
+        # it by scanning the books_lpath_cache for None entries and removing the
+        # corresponding nodes.
+        self.books_lpath_cache[book.lpath] = book
         cid = self.max_id()+1
+        node = self.document.createElement(self.prefix + "text")
+        self.sony_id_cache[cid] = book.lpath
+        mime = MIME_MAP.get(book.lpath.rpartition('.')[-1].lower(), MIME_MAP['epub'])
         try:
             sourceid = str(self[0].sourceid) if len(self) else '1'
         except:
             sourceid = '1'
         attrs = {
-                 "title"  : mi.title,
+                 "title"  : book.title,
-                 'titleSorter' : sortable_title(mi.title),
+                 'titleSorter' : sortable_title(book.title),
-                 "author" : mi.format_authors() if mi.format_authors() else _('Unknown'),
+                 "author" : book.format_authors() if book.format_authors() else _('Unknown'),
                  "page":"0", "part":"0", "scale":"0", \
                  "sourceid":sourceid,  "id":str(cid), "date":"", \
-                 "mime":mime, "path":name, "size":str(size)
+                 "mime":mime, "path":book.lpath, "size":str(book.size)
                  }
         for attr in attrs.keys():
             node.setAttributeNode(self.document.createAttribute(attr))
             node.setAttribute(attr, attrs[attr])
         try:
-            w, h, data = mi.thumbnail
+            w, h, data = book.thumbnail
         except:
             w, h, data = None, None, None
 
@@ -218,14 +139,11 @@ class BookList(_BookList):
             th.appendChild(jpeg)
             node.appendChild(th)
         self.root_element.appendChild(node)
-        book = Book(node, self.mountpath, [], prefix=self.prefix)
-        book.datetime = ctime
-        self.append(book)
 
         tags = []
-        for item in collections:
+        for item in self.collections:
             item = item.strip()
-            mitem = getattr(mi, item, None)
+            mitem = getattr(book, item, None)
             titems = []
             if mitem:
                 if isinstance(mitem, list):
@@ -241,37 +159,36 @@ class BookList(_BookList):
                 tags.extend(titems)
         if tags:
             tags = list(set(tags))
-            if hasattr(mi, 'tag_order'):
+            if hasattr(book, 'tag_order'):
-                self.tag_order.update(mi.tag_order)
+                self.tag_order.update(book.tag_order)
-            self.set_tags(book, tags)
+            self.set_playlists(cid, tags)
+        return True  # metadata cache has changed. Must sync at end
 
-    def _delete_book(self, node):
+    def _delete_node(self, node):
         nid = node.getAttribute('id')
         self.remove_from_playlists(nid)
         node.parentNode.removeChild(node)
         node.unlink()
 
-    def delete_book(self, cid):
+    def delete_node(self, lpath):
         '''
-        Remove DOM node corresponding to book with C{id == cid}.
+        Remove DOM node corresponding to book with lpath.
         Also remove book from any collections it is part of.
         '''
-        for book in self:
+        for child in self.root_element.childNodes:
-            if str(book.id) == str(cid):
+            if child.nodeType == child.ELEMENT_NODE and child.hasAttribute("id"):
-                self.remove(book)
+                if child.getAttribute('path') == lpath:
-                self._delete_book(book.elem)
+                    self._delete_node(child)
-                break
+                    break
 
-    def remove_book(self, path):
+    def remove_book(self, book):
         '''
         Remove DOM node corresponding to book with C{path == path}.
-        Also remove book from any collections it is part of.
+        Also remove book from any collections it is part of, and remove
+        from the booklist
         '''
-        for book in self:
+        self.remove(book)
-            if path.endswith(book.rpath):
+        self.delete_node(book.lpath)
-                self.remove(book)
-                self._delete_book(book.elem)
-                break
 
     def playlists(self):
         ans = []
@@ -358,15 +275,6 @@ class BookList(_BookList):
             item.setAttribute('id', str(id))
             coll.appendChild(item)
 
-    def get_playlists(self, bookid):
-        ans = []
-        for pl in self.playlists():
-            for item in pl.childNodes:
-                if hasattr(item, 'tagName') and item.tagName.endswith('item'):
-                    if item.getAttribute('id') == str(bookid):
-                        ans.append(pl)
-        return ans
-
     def next_id(self):
         return self.document.documentElement.getAttribute('nextID')
 
@@ -378,27 +286,36 @@ class BookList(_BookList):
         src = self.document.toxml('utf-8') + '\n'
         stream.write(src.replace("'", '&apos;'))
 
-    def book_by_id(self, id):
-        for book in self:
-            if str(book.id) == str(id):
-                return book
-
     def reorder_playlists(self):
         for title in self.tag_order.keys():
             pl = self.playlist_by_title(title)
             if not pl:
                 continue
-            db_ids = [i.getAttribute('id') for i in pl.childNodes if hasattr(i, 'getAttribute')]
+            # make a list of the ids
-            pl_book_ids = [getattr(self.book_by_id(i), 'db_id', None) for i in db_ids]
+            sony_ids = [id.getAttribute('id') \
+                    for id in pl.childNodes if hasattr(id, 'getAttribute')]
+            # convert IDs in playlist to a list of lpaths
+            sony_paths = [self.sony_id_cache[id] for id in sony_ids]
+            # create list of books containing lpaths
+            books = [self.books_lpath_cache.get(p, None) for p in sony_paths]
+            # create dict of db_id -> sony_id
             imap = {}
-            for i, j in zip(pl_book_ids, db_ids):
+            for book, sony_id in zip(books, sony_ids):
-                imap[i] = j
+                if book is not None:
-            pl_book_ids = [i for i in pl_book_ids if i is not None]
+                    db_id = book.application_id
-            ordered_ids = [i for i in self.tag_order[title] if i in pl_book_ids]
+                    if db_id is None:
+                        db_id = book.db_id
+                    if db_id is not None:
+                        imap[book.application_id] = sony_id
+            # filter the list, removing books not on device but on playlist
+            books = [i for i in books if i is not None]
+            # filter the order specification to the books we have
+            ordered_ids = [db_id for db_id in self.tag_order[title] if db_id in imap]
 
+            # rewrite the playlist in the correct order
             if len(ordered_ids) < len(pl.childNodes):
                 continue
-            children = [i for i in pl.childNodes  if hasattr(i, 'getAttribute')]
+            children = [i for i in pl.childNodes if hasattr(i, 'getAttribute')]
             for child in children:
                 pl.removeChild(child)
                 child.unlink()
@@ -439,8 +356,12 @@ def fix_ids(main, carda, cardb):
             except KeyError:
                 item.parentNode.removeChild(item)
                 item.unlink()
-
         db.reorder_playlists()
+        db.sony_id_cache = {}
+        for child in db.root_element.childNodes:
+            if child.nodeType == child.ELEMENT_NODE and child.hasAttribute("id"):
+                db.sony_id_cache[child.getAttribute('id')] = child.getAttribute('path')
+
 
     regen_ids(main)
     regen_ids(carda)

src/calibre/devices/prs505/driver.py

@@ -11,15 +11,14 @@ Device driver for the SONY PRS-505
 
 import os
 import re
-import time
-from itertools import cycle
 
-from calibre.devices.usbms.cli import CLI
+from calibre.devices.usbms.driver import USBMS
-from calibre.devices.usbms.device import Device
+from calibre.devices.prs505.books import BookList as PRS_BookList, fix_ids
-from calibre.devices.prs505.books import BookList, fix_ids
+from calibre.devices.prs505 import MEDIA_XML
+from calibre.devices.prs505 import CACHE_XML
 from calibre import __appname__
 
-class PRS505(CLI, Device):
+class PRS505(USBMS):
 
     name           = 'PRS-300/505 Device Interface'
     gui_name       = 'SONY Reader'
@@ -28,6 +27,8 @@ class PRS505(CLI, Device):
     supported_platforms = ['windows', 'osx', 'linux']
     path_sep = '/'
 
+    booklist_class = PRS_BookList # See USBMS for some explanation of this
+
     FORMATS      = ['epub', 'lrf', 'lrx', 'rtf', 'pdf', 'txt']
 
     VENDOR_ID    = [0x054c]   #: SONY Vendor Id
@@ -46,9 +47,6 @@ class PRS505(CLI, Device):
     MAIN_MEMORY_VOLUME_LABEL  = 'Sony Reader Main Memory'
     STORAGE_CARD_VOLUME_LABEL = 'Sony Reader Storage Card'
 
-    MEDIA_XML    = 'database/cache/media.xml'
-    CACHE_XML    = 'Sony Reader/database/cache.xml'
-
     CARD_PATH_PREFIX          = __appname__
 
     SUPPORTS_SUB_DIRS = True
@@ -63,64 +61,9 @@ class PRS505(CLI, Device):
     def windows_filter_pnp_id(self, pnp_id):
         return '_LAUNCHER' in pnp_id
 
-    def open(self):
-        self.report_progress = lambda x, y: x
-        Device.open(self)
-
-        def write_cache(prefix):
-            try:
-                cachep = os.path.join(prefix, *(self.CACHE_XML.split('/')))
-                if not os.path.exists(cachep):
-                    dname = os.path.dirname(cachep)
-                    if not os.path.exists(dname):
-                        try:
-                            os.makedirs(dname, mode=0777)
-                        except:
-                            time.sleep(5)
-                            os.makedirs(dname, mode=0777)
-                    with open(cachep, 'wb') as f:
-                        f.write(u'''<?xml version="1.0" encoding="UTF-8"?>
-                            <cache xmlns="http://www.kinoma.com/FskCache/1">
-                            </cache>
-                            '''.encode('utf8'))
-                return True
-            except:
-                import traceback
-                traceback.print_exc()
-            return False
-
-        if self._card_a_prefix is not None:
-            if not write_cache(self._card_a_prefix):
-                self._card_a_prefix = None
-        if self._card_b_prefix is not None:
-            if not write_cache(self._card_b_prefix):
-                self._card_b_prefix = None
-
     def get_device_information(self, end_session=True):
         return (self.gui_name, '', '', '')
 
-    def books(self, oncard=None, end_session=True):
-        if oncard == 'carda' and not self._card_a_prefix:
-            self.report_progress(1.0, _('Getting list of books on device...'))
-            return []
-        elif oncard == 'cardb' and not self._card_b_prefix:
-            self.report_progress(1.0, _('Getting list of books on device...'))
-            return []
-        elif oncard and oncard != 'carda' and oncard != 'cardb':
-            self.report_progress(1.0, _('Getting list of books on device...'))
-            return []
-
-        db = self.__class__.CACHE_XML if oncard else self.__class__.MEDIA_XML
-        prefix = self._card_a_prefix if oncard == 'carda' else self._card_b_prefix if oncard == 'cardb' else self._main_prefix
-        bl = BookList(open(prefix + db, 'rb'), prefix, self.report_progress)
-        paths = bl.purge_corrupted_files()
-        for path in paths:
-            path = os.path.join(prefix, path)
-            if os.path.exists(path):
-                os.unlink(path)
-        self.report_progress(1.0, _('Getting list of books on device...'))
-        return bl
-
     def filename_callback(self, fname, mi):
         if getattr(mi, 'application_id', None) is not None:
             base = fname.rpartition('.')[0]
@@ -129,90 +72,16 @@ class PRS505(CLI, Device):
                 fname = base + suffix + '.' + fname.rpartition('.')[-1]
         return fname
 
-    def upload_books(self, files, names, on_card=None, end_session=True,
-                     metadata=None):
-
-        path = self._sanity_check(on_card, files)
-
-        paths, ctimes, sizes = [], [], []
-        names = iter(names)
-        metadata = iter(metadata)
-        for i, infile in enumerate(files):
-            mdata, fname = metadata.next(), names.next()
-            filepath = self.create_upload_path(path, mdata, fname)
-
-            paths.append(filepath)
-            self.put_file(infile, paths[-1], replace_file=True)
-            ctimes.append(os.path.getctime(paths[-1]))
-            sizes.append(os.stat(paths[-1]).st_size)
-
-            self.report_progress((i+1) / float(len(files)), _('Transferring books to device...'))
-
-        self.report_progress(1.0, _('Transferring books to device...'))
-
-        return zip(paths, sizes, ctimes, cycle([on_card]))
-
-    def add_books_to_metadata(self, locations, metadata, booklists):
-        if not locations or not metadata:
-            return
-
-        metadata = iter(metadata)
-        for location in locations:
-            info = metadata.next()
-            path = location[0]
-            oncard = location[3]
-            blist = 2 if oncard == 'cardb' else 1 if oncard == 'carda' else 0
-
-            if self._main_prefix and path.startswith(self._main_prefix):
-                name = path.replace(self._main_prefix, '')
-            elif self._card_a_prefix and path.startswith(self._card_a_prefix):
-                name = path.replace(self._card_a_prefix, '')
-            elif self._card_b_prefix and path.startswith(self._card_b_prefix):
-                name = path.replace(self._card_b_prefix, '')
-
-            name = name.replace('\\', '/')
-            name = name.replace('//', '/')
-            if name.startswith('/'):
-                name = name[1:]
-
-            opts = self.settings()
-            collections = opts.extra_customization.split(',') if opts.extra_customization else []
-            booklist = booklists[blist]
-            if not hasattr(booklist, 'add_book'):
-                raise ValueError(('Incorrect upload location %s. Did you choose the'
-                        ' correct card A or B, to send books to?')%oncard)
-            booklist.add_book(info, name, collections, *location[1:-1])
-        fix_ids(*booklists)
-
-    def delete_books(self, paths, end_session=True):
-        for i, path in enumerate(paths):
-            self.report_progress((i+1) / float(len(paths)), _('Removing books from device...'))
-            if os.path.exists(path):
-                os.unlink(path)
-                try:
-                    os.removedirs(os.path.dirname(path))
-                except:
-                    pass
-        self.report_progress(1.0, _('Removing books from device...'))
-
-    @classmethod
-    def remove_books_from_metadata(cls, paths, booklists):
-        for path in paths:
-            for bl in booklists:
-                if hasattr(bl, 'remove_book'):
-                    bl.remove_book(path)
-        fix_ids(*booklists)
-
     def sync_booklists(self, booklists, end_session=True):
         fix_ids(*booklists)
         if not os.path.exists(self._main_prefix):
             os.makedirs(self._main_prefix)
-        with open(self._main_prefix + self.__class__.MEDIA_XML, 'wb') as f:
+        with open(self._main_prefix + MEDIA_XML, 'wb') as f:
             booklists[0].write(f)
 
         def write_card_prefix(prefix, listid):
             if prefix is not None and hasattr(booklists[listid], 'write'):
-                tgt  = os.path.join(prefix, *(self.CACHE_XML.split('/')))
+                tgt  = os.path.join(prefix, *(CACHE_XML.split('/')))
                 base = os.path.dirname(tgt)
                 if not os.path.exists(base):
                     os.makedirs(base)
@@ -221,8 +90,7 @@ class PRS505(CLI, Device):
         write_card_prefix(self._card_a_prefix, 1)
         write_card_prefix(self._card_b_prefix, 2)
 
-        self.report_progress(1.0, _('Sending metadata to device...'))
+        USBMS.sync_booklists(self, booklists, end_session)
-
 
 class PRS700(PRS505):
 
@@ -241,5 +109,3 @@ class PRS700(PRS505):
     OSX_MAIN_MEM   = re.compile(r'Sony PRS-((700/[^:]+)|((6|9)00)) Media')
     OSX_CARD_A_MEM = re.compile(r'Sony PRS-((700/[^:]+:)|((6|9)00 ))MS Media')
     OSX_CARD_B_MEM = re.compile(r'Sony PRS-((700/[^:]+:)|((6|9)00 ))SD Media')
-
-

src/calibre/devices/usbms/books.py

@@ -31,14 +31,14 @@ class Book(MetaInformation):
         MetaInformation.__init__(self, '')
 
         self.path = os.path.join(prefix, lpath)
-        self.lpath = lpath
+        if os.sep == '\\':
+            self.path = self.path.replace('/', '\\')
+            self.lpath = lpath.replace('\\', '/')
+        else:
+            self.lpath = lpath
         self.mime = mime_type_ext(path_to_ext(lpath))
-        self.size = os.stat(self.path).st_size if size == None else size
+        self.size = None # will be set later
-        self.db_id = None
+        self.datetime = time.gmtime()
-        try:
-            self.datetime = time.gmtime(os.path.getctime(self.path))
-        except ValueError:
-            self.datetime = time.gmtime()
 
         if other:
             self.smart_update(other)
@@ -66,6 +66,16 @@ class Book(MetaInformation):
 
         return spath == opath
 
+    @dynamic_property
+    def db_id(self):
+        doc = '''The database id in the application database that this file corresponds to'''
+        def fget(self):
+            match = re.search(r'_(\d+)$', self.lpath.rpartition('.')[0])
+            if match:
+                return int(match.group(1))
+            return None
+        return property(fget=fget, doc=doc)
+
     @dynamic_property
     def title_sorter(self):
         doc = '''String to sort the title. If absent, title is returned'''
@@ -77,13 +87,6 @@ class Book(MetaInformation):
     def thumbnail(self):
         return None
 
-#    def __str__(self):
-#        '''
-#        Return a utf-8 encoded string with title author and path information
-#        '''
-#        return self.title.encode('utf-8') + " by " + \
-#               self.authors.encode('utf-8') + " at " + self.path.encode('utf-8')
-
     def smart_update(self, other):
         '''
         Merge the information in C{other} into self. In case of conflicts, the information
@@ -105,9 +108,26 @@ class Book(MetaInformation):
 
 class BookList(_BookList):
 
+    def __init__(self, oncard, prefix, settings):
+        pass
+
     def supports_tags(self):
         return True
 
     def set_tags(self, book, tags):
         book.tags = tags
 
+    def add_book(self, book, replace_metadata):
+        '''
+        Add the book to the booklist. Intent is to maintain any device-internal
+        metadata. Return True if booklists must be sync'ed
+        '''
+        if book not in self:
+            self.append(book)
+
+    def remove_book(self, book):
+        '''
+        Remove a book from the booklist. Correct any device metadata at the
+        same time
+        '''
+        self.remove(book)

src/calibre/devices/usbms/device.py

@@ -113,15 +113,17 @@ class Device(DeviceConfig, DevicePlugin):
     def _windows_space(cls, prefix):
         if not prefix:
             return 0, 0
+        if prefix.endswith(os.sep):
+            prefix = prefix[:-1]
         win32file = __import__('win32file', globals(), locals(), [], -1)
         try:
             sectors_per_cluster, bytes_per_sector, free_clusters, total_clusters = \
-                win32file.GetDiskFreeSpace(prefix[:-1])
+                win32file.GetDiskFreeSpace(prefix)
         except Exception, err:
             if getattr(err, 'args', [None])[0] == 21: # Disk not ready
                 time.sleep(3)
                 sectors_per_cluster, bytes_per_sector, free_clusters, total_clusters = \
-                    win32file.GetDiskFreeSpace(prefix[:-1])
+                    win32file.GetDiskFreeSpace(prefix)
             else: raise
         mult = sectors_per_cluster * bytes_per_sector
         return total_clusters * mult, free_clusters * mult

src/calibre/devices/usbms/driver.py

@@ -27,6 +27,13 @@ class USBMS(CLI, Device):
     author         = _('John Schember')
     supported_platforms = ['windows', 'osx', 'linux']
 
+    # Store type instances of BookList and Book. We must do this because
+    # a) we need to override these classes in some device drivers, and
+    # b) the classmethods seem only to see real attributes declared in the
+    #    class, not attributes stored in the class
+    booklist_class = BookList
+    book_class = Book
+
     FORMATS = []
     CAN_SET_METADATA = True
     METADATA_CACHE = 'metadata.calibre'
@@ -37,48 +44,52 @@ class USBMS(CLI, Device):
 
     def books(self, oncard=None, end_session=True):
         from calibre.ebooks.metadata.meta import path_to_ext
-        bl = BookList()
-        metadata = BookList()
-        need_sync = False
 
         if oncard == 'carda' and not self._card_a_prefix:
             self.report_progress(1.0, _('Getting list of books on device...'))
-            return bl
+            return []
         elif oncard == 'cardb' and not self._card_b_prefix:
             self.report_progress(1.0, _('Getting list of books on device...'))
-            return bl
+            return []
         elif oncard and oncard != 'carda' and oncard != 'cardb':
             self.report_progress(1.0, _('Getting list of books on device...'))
-            return bl
+            return []
+
+        prefix = self._card_a_prefix if oncard == 'carda' else \
+                                     self._card_b_prefix if oncard == 'cardb' \
+                                                         else self._main_prefix
 
-        prefix = self._card_a_prefix if oncard == 'carda' else self._card_b_prefix if oncard == 'cardb' else self._main_prefix
         ebook_dirs = self.EBOOK_DIR_CARD_A if oncard == 'carda' else \
             self.EBOOK_DIR_CARD_B if oncard == 'cardb' else \
             self.get_main_ebook_dir()
 
+        # build a temporary list of books from the metadata cache
         bl, need_sync = self.parse_metadata_cache(prefix, self.METADATA_CACHE)
-
         # make a dict cache of paths so the lookup in the loop below is faster.
         bl_cache = {}
         for idx,b in enumerate(bl):
-            bl_cache[b.path] = idx
+            bl_cache[b.lpath] = idx
         self.count_found_in_bl = 0
 
+        # Make the real booklist that will be filled in below
+        metadata = self.booklist_class(oncard, prefix, self.settings)
+
         def update_booklist(filename, path, prefix):
             changed = False
             if path_to_ext(filename) in self.FORMATS:
                 try:
-                    lpath = os.path.join(path, filename).partition(prefix)[2]
+                    lpath = os.path.join(path, filename).partition(self.normalize_path(prefix))[2]
                     if lpath.startswith(os.sep):
                         lpath = lpath[len(os.sep):]
-                    p = os.path.join(prefix, lpath)
+                    idx = bl_cache.get(lpath.replace('\\', '/'), None)
-                    if p in bl_cache:
+                    if idx is not None:
-                        item, changed = self.__class__.update_metadata_item(bl[bl_cache[p]])
+                        item, changed = self.update_metadata_item(bl[idx])
                         self.count_found_in_bl += 1
                     else:
-                        item = self.__class__.book_from_path(prefix, lpath)
+                        item = self.book_from_path(prefix, lpath)
+                        changed = True
+                    if metadata.add_book(item, replace_metadata=False):
                         changed = True
-                    metadata.append(item)
                 except: # Probably a filename encoding error
                     import traceback
                     traceback.print_exc()
@@ -87,7 +98,9 @@ class USBMS(CLI, Device):
         if isinstance(ebook_dirs, basestring):
             ebook_dirs = [ebook_dirs]
         for ebook_dir in ebook_dirs:
-            ebook_dir = os.path.join(prefix, *(ebook_dir.split('/'))) if ebook_dir else prefix
+            ebook_dir = self.normalize_path( \
+                            os.path.join(prefix, *(ebook_dir.split('/'))) \
+                                    if ebook_dir else prefix)
             if not os.path.exists(ebook_dir): continue
             # Get all books in the ebook_dir directory
             if self.SUPPORTS_SUB_DIRS:
@@ -108,6 +121,7 @@ class USBMS(CLI, Device):
         # if count != len(bl) then there were items in it that we did not
         # find on the device. If need_sync is True then there were either items
         # on the device that were not in bl or some of the items were changed.
+        print "count found in cache: %d, count of files in cache: %d, must_sync_cache: %s" % (self.count_found_in_bl, len(bl), need_sync)
         if self.count_found_in_bl != len(bl) or need_sync:
             if oncard == 'cardb':
                 self.sync_booklists((None, None, metadata))
@@ -117,12 +131,10 @@ class USBMS(CLI, Device):
                 self.sync_booklists((metadata, None, None))
 
         self.report_progress(1.0, _('Getting list of books on device...'))
-        #print 'at return', now() - start_time
         return metadata
 
     def upload_books(self, files, names, on_card=None, end_session=True,
                      metadata=None):
-
         path = self._sanity_check(on_card, files)
 
         paths = []
@@ -131,13 +143,12 @@ class USBMS(CLI, Device):
 
         for i, infile in enumerate(files):
             mdata, fname = metadata.next(), names.next()
-            filepath = self.create_upload_path(path, mdata, fname)
+            filepath = self.normalize_path(self.create_upload_path(path, mdata, fname))
-
             paths.append(filepath)
-
+            self.put_file(self.normalize_path(infile), filepath, replace_file=True)
-            self.put_file(infile, filepath, replace_file=True)
             try:
-                self.upload_cover(os.path.dirname(filepath), os.path.splitext(os.path.basename(filepath))[0], mdata)
+                self.upload_cover(os.path.dirname(filepath),
+                                  os.path.splitext(os.path.basename(filepath))[0], mdata)
             except: # Failure to upload cover is not catastrophic
                 import traceback
                 traceback.print_exc()
@@ -145,7 +156,6 @@ class USBMS(CLI, Device):
             self.report_progress((i+1) / float(len(files)), _('Transferring books to device...'))
 
         self.report_progress(1.0, _('Transferring books to device...'))
-
         return zip(paths, cycle([on_card]))
 
     def upload_cover(self, path, filename, metadata):
@@ -166,25 +176,32 @@ class USBMS(CLI, Device):
             blist = 2 if location[1] == 'cardb' else 1 if location[1] == 'carda' else 0
 
             if self._main_prefix:
+                # Normalize path and prefix
+                if self._main_prefix.find('\\') >= 0:
+                    path = path.replace('/', '\\')
+                else:
+                    path = path.replace('\\', '/')
                 prefix = self._main_prefix if path.startswith(self._main_prefix) else None
             if not prefix and self._card_a_prefix:
                 prefix = self._card_a_prefix if path.startswith(self._card_a_prefix) else None
             if not prefix and self._card_b_prefix:
                 prefix = self._card_b_prefix if path.startswith(self._card_b_prefix) else None
+            if prefix is None:
+                print 'in add_books_to_metadata. Prefix is None!', path, self._main_prefix
+                continue
             lpath = path.partition(prefix)[2]
-            if lpath.startswith(os.sep):
+            if lpath.startswith('/') or lpath.startswith('\\'):
-                lpath = lpath[len(os.sep):]
+                lpath = lpath[1:]
-
+            book = self.book_class(prefix, lpath, other=info)
-            book = Book(prefix, lpath, other=info)
+            if book.size is None:
-
+                book.size = os.stat(self.normalize_path(path)).st_size
-            if book not in booklists[blist]:
+            booklists[blist].add_book(book, replace_metadata=True)
-                booklists[blist].append(book)
-
         self.report_progress(1.0, _('Adding books to device metadata listing...'))
 
     def delete_books(self, paths, end_session=True):
         for i, path in enumerate(paths):
             self.report_progress((i+1) / float(len(paths)), _('Removing books from device...'))
+            path = self.normalize_path(path)
             if os.path.exists(path):
                 # Delete the ebook
                 os.unlink(path)
@@ -209,19 +226,19 @@ class USBMS(CLI, Device):
             for bl in booklists:
                 for book in bl:
                     if path.endswith(book.path):
-                        bl.remove(book)
+                        bl.remove_book(book)
         self.report_progress(1.0, _('Removing books from device metadata listing...'))
 
     def sync_booklists(self, booklists, end_session=True):
-        if not os.path.exists(self._main_prefix):
+        if not os.path.exists(self.normalize_path(self._main_prefix)):
-            os.makedirs(self._main_prefix)
+            os.makedirs(self.normalize_path(self._main_prefix))
 
         def write_prefix(prefix, listid):
-            if prefix is not None and isinstance(booklists[listid], BookList):
+            if prefix is not None and isinstance(booklists[listid], self.booklist_class):
                 if not os.path.exists(prefix):
-                    os.makedirs(prefix)
+                    os.makedirs(self.normalize_path(prefix))
                 js = [item.to_json() for item in booklists[listid]]
-                with open(os.path.join(prefix, self.METADATA_CACHE), 'wb') as f:
+                with open(self.normalize_path(os.path.join(prefix, self.METADATA_CACHE)), 'wb') as f:
                     json.dump(js, f, indent=2, encoding='utf-8')
         write_prefix(self._main_prefix, 0)
         write_prefix(self._card_a_prefix, 1)
@@ -229,37 +246,45 @@ class USBMS(CLI, Device):
 
         self.report_progress(1.0, _('Sending metadata to device...'))
 
+    @classmethod
+    def normalize_path(cls, path):
+        if path is None:
+            return None
+        if os.sep == '\\':
+            path = path.replace('/', '\\')
+        else:
+            path = path.replace('\\', '/')
+        return path
+
     @classmethod
     def parse_metadata_cache(cls, prefix, name):
+        bl = []
         js = []
-        bl = BookList()
         need_sync = False
         try:
-            with open(os.path.join(prefix, name), 'rb') as f:
+            with open(cls.normalize_path(os.path.join(prefix, name)), 'rb') as f:
                 js = json.load(f, encoding='utf-8')
             for item in js:
-                lpath = item.get('lpath', None)
+                book = cls.book_class(prefix, item.get('lpath', None))
-                if not lpath or not os.path.exists(os.path.join(prefix, lpath)):
-                    need_sync = True
-                    continue
-                book = Book(prefix, lpath)
                 for key in item.keys():
                     setattr(book, key, item[key])
                 bl.append(book)
         except:
             import traceback
             traceback.print_exc()
-            bl = BookList()
+            bl = []
+            need_sync = True
         return bl, need_sync
 
     @classmethod
     def update_metadata_item(cls, item):
         changed = False
-        size = os.stat(item.path).st_size
+        size = os.stat(cls.normalize_path(item.path)).st_size
         if size != item.size:
             changed = True
             mi = cls.metadata_from_path(item.path)
             item.smart_update(mi)
+            item.size = size
         return item, changed
 
     @classmethod
@@ -278,15 +303,15 @@ class USBMS(CLI, Device):
         from calibre.ebooks.metadata import MetaInformation
 
         if cls.settings().read_metadata or cls.MUST_READ_METADATA:
-            mi = cls.metadata_from_path(os.path.join(prefix, path))
+            mi = cls.metadata_from_path(cls.normalize_path(os.path.join(prefix, path)))
         else:
             from calibre.ebooks.metadata.meta import metadata_from_filename
-            mi = metadata_from_filename(os.path.basename(path),
+            mi = metadata_from_filename(cls.normalize_path(os.path.basename(path)),
                 re.compile(r'^(?P<title>[ \S]+?)[ _]-[ _](?P<author>[ \S]+?)_+\d+'))
 
         if mi is None:
             mi = MetaInformation(os.path.splitext(os.path.basename(path))[0],
                     [_('Unknown')])
-
+        mi.size = os.stat(cls.normalize_path(os.path.join(prefix, path))).st_size
-        book = Book(prefix, path, other=mi)
+        book = cls.book_class(prefix, path, other=mi)
         return book

src/calibre/ebooks/metadata/__init__.py

@@ -254,11 +254,11 @@ class MetaInformation(object):
             setattr(self, x, getattr(mi, x, None))
 
     def print_all_attributes(self):
-        for x in ('author_sort', 'title_sort', 'comments', 'category', 'publisher',
+        for x in ('author', 'author_sort', 'title_sort', 'comments', 'category', 'publisher',
-                  'series', 'series_index', 'rating', 'isbn', 'language',
+                  'series', 'series_index', 'tags', 'rating', 'isbn', 'language',
                   'application_id', 'manifest', 'toc', 'spine', 'guide', 'cover',
                   'book_producer', 'timestamp', 'lccn', 'lcc', 'ddc', 'pubdate',
-                  'rights', 'publication_type', 'uuid',
+                  'rights', 'publication_type', 'uuid', 'tag_order',
                   ):
             prints(x, getattr(self, x, 'None'))
 
@@ -278,7 +278,7 @@ class MetaInformation(object):
                      'isbn', 'application_id', 'manifest', 'spine', 'toc',
                      'cover', 'language', 'guide', 'book_producer',
                      'timestamp', 'lccn', 'lcc', 'ddc', 'pubdate', 'rights',
-                     'publication_type', 'uuid',):
+                     'publication_type', 'uuid', 'tag_order'):
             if hasattr(mi, attr):
                 val = getattr(mi, attr)
                 if val is not None:

src/calibre/gui2/device.py

@@ -25,6 +25,7 @@ from calibre.utils.filenames import ascii_filename
 from calibre.devices.errors import FreeSpaceError
 from calibre.utils.smtp import compose_mail, sendmail, extract_email_address, \
         config as email_config
+from calibre.devices.folder_device.driver import FOLDER_DEVICE
 
 class DeviceJob(BaseJob):
 
@@ -207,6 +208,23 @@ class DeviceManager(Thread):
         return self.create_job(self._get_device_information, done,
                     description=_('Get device information'))
 
+    def connect_to_folder(self, path):
+        dev = FOLDER_DEVICE(path)
+        try:
+            dev.open()
+        except:
+            print 'Unable to open device', dev
+            traceback.print_exc()
+            return False
+        self.connected_device = dev
+        self.connected_slot(True)
+        return True
+
+    def disconnect_folder(self):
+        if self.connected_device is not None:
+            if hasattr(self.connected_device, 'disconnect_from_folder'):
+                self.connected_device.disconnect_from_folder()
+
     def _books(self):
         '''Get metadata from device'''
         mainlist = self.device.books(oncard=None, end_session=False)
@@ -309,6 +327,8 @@ class DeviceAction(QAction):
 class DeviceMenu(QMenu):
 
     fetch_annotations = pyqtSignal()
+    connect_to_folder = pyqtSignal()
+    disconnect_from_folder = pyqtSignal()
 
     def __init__(self, parent=None):
         QMenu.__init__(self, parent)
@@ -404,6 +424,18 @@ class DeviceMenu(QMenu):
         if opts.accounts:
             self.addSeparator()
             self.addMenu(self.email_to_menu)
+
+        self.addSeparator()
+        mitem = self.addAction(_('Connect to folder'))
+        mitem.setEnabled(True)
+        mitem.triggered.connect(lambda x : self.connect_to_folder.emit())
+        self.connect_to_folder_action = mitem
+
+        mitem = self.addAction(_('Disconnect from folder'))
+        mitem.setEnabled(False)
+        mitem.triggered.connect(lambda x : self.disconnect_from_folder.emit())
+        self.disconnect_from_folder_action = mitem
+
         self.addSeparator()
         annot = self.addAction(_('Fetch annotations (experimental)'))
         annot.setEnabled(False)
@@ -523,7 +555,8 @@ class DeviceGUI(object):
             d = ChooseFormatDialog(self, _('Choose format to send to device'),
                                 self.device_manager.device.settings().format_map)
             d.exec_()
-            fmt = d.format().lower()
+            if d.format():
+                fmt = d.format().lower()
         dest, sub_dest = dest.split(':')
         if dest in ('main', 'carda', 'cardb'):
             if not self.device_connected or not self.device_manager:
@@ -821,7 +854,9 @@ class DeviceGUI(object):
 
     def sync_to_device(self, on_card, delete_from_library,
             specific_format=None, send_ids=None, do_auto_convert=True):
-        ids = [self.library_view.model().id(r) for r in self.library_view.selectionModel().selectedRows()] if send_ids is None else send_ids
+        ids = [self.library_view.model().id(r) \
+               for r in self.library_view.selectionModel().selectedRows()] \
+                                if send_ids is None else send_ids
         if not self.device_manager or not ids or len(ids) == 0:
             return
 
@@ -842,8 +877,7 @@ class DeviceGUI(object):
         ids = iter(ids)
         for mi in metadata:
             if mi.cover and os.access(mi.cover, os.R_OK):
-                mi.thumbnail = self.cover_to_thumbnail(open(mi.cover,
+                mi.thumbnail = self.cover_to_thumbnail(open(mi.cover, 'rb').read())
-                    'rb').read())
         imetadata = iter(metadata)
 
         files = [getattr(f, 'name', None) for f in _files]
@@ -890,7 +924,9 @@ class DeviceGUI(object):
                         bad.append(self.library_view.model().db.title(id, index_is_id=True))
 
         if auto != []:
-            format = specific_format if specific_format in list(set(settings.format_map).intersection(set(available_output_formats()))) else None
+            format = specific_format if specific_format in \
+                            list(set(settings.format_map).intersection(set(available_output_formats()))) \
+                            else None
             if not format:
                 for fmt in settings.format_map:
                     if fmt in list(set(settings.format_map).intersection(set(available_output_formats()))):
@@ -995,83 +1031,111 @@ class DeviceGUI(object):
             if changed:
                 self.library_view.model().refresh_ids(list(changed))
 
-    def book_on_device(self, index, format=None, reset=False):
+    def book_on_device(self, id, format=None, reset=False):
         loc = [None, None, None]
 
         if reset:
-            self.book_on_device_cache = None
+            self.book_db_title_cache = None
+            self.book_db_uuid_cache = None
             return
 
-        if self.book_on_device_cache is None:
+        if self.book_db_title_cache is None:
-            self.book_on_device_cache = []
+            self.book_db_title_cache = []
+            self.book_db_uuid_cache = []
             for i, l in enumerate(self.booklists()):
-                self.book_on_device_cache.append({})
+                self.book_db_title_cache.append({})
+                self.book_db_uuid_cache.append(set())
                 for book in l:
                     book_title = book.title.lower() if book.title else ''
                     book_title = re.sub('(?u)\W|[_]', '', book_title)
-                    if book_title not in self.book_on_device_cache[i]:
+                    if book_title not in self.book_db_title_cache[i]:
-                        self.book_on_device_cache[i][book_title] = \
+                        self.book_db_title_cache[i][book_title] = \
                                 {'authors':set(), 'db_ids':set(), 'uuids':set()}
                     book_authors = authors_to_string(book.authors).lower()
                     book_authors = re.sub('(?u)\W|[_]', '', book_authors)
-                    self.book_on_device_cache[i][book_title]['authors'].add(book_authors)
+                    self.book_db_title_cache[i][book_title]['authors'].add(book_authors)
-                    id = getattr(book, 'application_id', None)
+                    db_id = getattr(book, 'application_id', None)
-                    if id is None:
+                    if db_id is None:
-                        id = book.db_id
+                        db_id = book.db_id
-                    if id is not None:
+                    if db_id is not None:
-                        self.book_on_device_cache[i][book_title]['db_ids'].add(id)
+                        self.book_db_title_cache[i][book_title]['db_ids'].add(db_id)
                     uuid = getattr(book, 'uuid', None)
-                    if uuid is None:
+                    if uuid is not None:
-                        self.book_on_device_cache[i][book_title]['uuids'].add(uuid)
+                        self.book_db_uuid_cache[i].add(uuid)
 
-        db = self.library_view.model().db
+        mi = self.library_view.model().db.get_metadata(id, index_is_id=True)
-        db_title = db.title(index, index_is_id=True).lower()
-        db_title = re.sub('(?u)\W|[_]', '', db_title)
-        db_authors = db.authors(index, index_is_id=True)
-        db_authors = db_authors.lower() if db_authors else ''
-        db_authors = re.sub('(?u)\W|[_]', '', db_authors)
-        db_uuid = db.uuid(index, index_is_id=True)
         for i, l in enumerate(self.booklists()):
-            d = self.book_on_device_cache[i].get(db_title, None)
+            if mi.uuid in self.book_db_uuid_cache[i]:
-            if d:
+                loc[i] = True
-                if db_uuid in d['uuids'] or \
+                continue
-                        index in d['db_ids'] or \
+            db_title = re.sub('(?u)\W|[_]', '', mi.title.lower())
-                        db_authors in d['authors']:
+            cache = self.book_db_title_cache[i].get(db_title, None)
+            if cache:
+                if id in cache['db_ids']:
+                    loc[i] = True
+                    break
+                if mi.authors and \
+                        re.sub('(?u)\W|[_]', '', authors_to_string(mi.authors).lower()) \
+                        in cache['authors']:
                     loc[i] = True
                     break
         return loc
 
     def set_books_in_library(self, booklists, reset=False):
         if reset:
-            # First build a self.book_in_library_cache of the library, so the search isn't On**2
+            # First build a cache of the library, so the search isn't On**2
-            self.book_in_library_cache = {}
+            self.db_book_title_cache = {}
-            for id, title in self.library_view.model().db.all_titles():
+            self.db_book_uuid_cache = set()
-                title = re.sub('(?u)\W|[_]', '', title.lower())
+            db = self.library_view.model().db
-                if title not in self.book_in_library_cache:
+            for id in db.data.iterallids():
-                    self.book_in_library_cache[title] = {'authors':set(), 'db_ids':set(), 'uuids':set()}
+                mi = db.get_metadata(id, index_is_id=True)
-                au = self.library_view.model().db.authors(id, index_is_id=True)
+                title = re.sub('(?u)\W|[_]', '', mi.title.lower())
-                authors = au.lower() if au else ''
+                if title not in self.db_book_title_cache:
+                    self.db_book_title_cache[title] = {'authors':{}, 'db_ids':{}}
+                authors = authors_to_string(mi.authors).lower() if mi.authors else ''
                 authors = re.sub('(?u)\W|[_]', '', authors)
-                self.book_in_library_cache[title]['authors'].add(authors)
+                self.db_book_title_cache[title]['authors'][authors] = mi
-                self.book_in_library_cache[title]['db_ids'].add(id)
+                self.db_book_title_cache[title]['db_ids'][mi.application_id] = mi
-                self.book_in_library_cache[title]['uuids'].add(self.library_view.model().db.uuid(id, index_is_id=True))
+                self.db_book_uuid_cache.add(mi.uuid)
 
-        # Now iterate through all the books on the device, setting the in_library field
+        # Now iterate through all the books on the device, setting the
+        # in_library field Fastest and most accurate key is the uuid. Second is
+        # the application_id, which is really the db key, but as this can
+        # accidentally match across libraries we also verify the title. The
+        # db_id exists on Sony devices. Fallback is title and author match
+        resend_metadata = False
         for booklist in booklists:
             for book in booklist:
+                if getattr(book, 'uuid', None) in self.db_book_uuid_cache:
+                    book.in_library = True
+                    continue
+
                 book_title = book.title.lower() if book.title else ''
                 book_title = re.sub('(?u)\W|[_]', '', book_title)
                 book.in_library = False
-                d = self.book_in_library_cache.get(book_title, None)
+                d = self.db_book_title_cache.get(book_title, None)
                 if d is not None:
-                    if getattr(book, 'uuid', None) in d['uuids'] or \
+                    if getattr(book, 'application_id', None) in d['db_ids']:
-                            getattr(book, 'application_id', None) in d['db_ids']:
                         book.in_library = True
+                        book.smart_update(d['db_ids'][book.application_id])
+                        resend_metadata = True
                         continue
                     if book.db_id in d['db_ids']:
                         book.in_library = True
+                        book.smart_update(d['db_ids'][book.db_id])
+                        resend_metadata = True
                         continue
                     book_authors = authors_to_string(book.authors).lower() if book.authors else ''
                     book_authors = re.sub('(?u)\W|[_]', '', book_authors)
                     if book_authors in d['authors']:
                         book.in_library = True
+                        book.smart_update(d['authors'][book_authors])
+                        resend_metadata = True
+                # Set author_sort if it isn't already
+                asort = getattr(book, 'author_sort', None)
+                if not asort:
+                    pass
+        if resend_metadata:
+            # Correcting metadata cache on device.
+            if self.device_manager.is_device_connected:
+                self.device_manager.sync_booklists(None, booklists)

src/calibre/gui2/library.py

@@ -17,7 +17,7 @@ from PyQt4.QtCore import QAbstractTableModel, QVariant, Qt, pyqtSignal, \
                          SIGNAL, QObject, QSize, QModelIndex, QDate
 
 from calibre import strftime
-from calibre.ebooks.metadata import fmt_sidx, authors_to_string
+from calibre.ebooks.metadata import fmt_sidx, authors_to_string, string_to_authors
 from calibre.ebooks.metadata.meta import set_metadata as _set_metadata
 from calibre.gui2 import NONE, TableView, config, error_dialog, UNDEFINED_QDATE
 from calibre.gui2.dialogs.comments_dialog import CommentsDialog
@@ -371,7 +371,7 @@ class BooksModel(QAbstractTableModel):
     def set_device_connected(self, is_connected):
         self.device_connected = is_connected
         self.read_config()
-        self.refresh(reset=True)
+        self.db.refresh_ondevice()
         self.database_changed.emit(self.db)
 
     def set_book_on_device_func(self, func):
@@ -1378,7 +1378,17 @@ class DeviceBooksModel(BooksModel):
         def libcmp(x, y):
             x, y = self.db[x].in_library, self.db[y].in_library
             return cmp(x, y)
-        fcmp = strcmp('title_sorter') if col == 0 else strcmp('authors') if col == 1 else \
+        def authorcmp(x, y):
+            ax = getattr(self.db[x], 'author_sort', None)
+            ay = getattr(self.db[y], 'author_sort', None)
+            if ax and ay:
+                x = ax
+                y = ay
+            else:
+                x, y = authors_to_string(self.db[x].authors), \
+                                authors_to_string(self.db[y].authors)
+            return cmp(x, y)
+        fcmp = strcmp('title_sorter') if col == 0 else authorcmp if col == 1 else \
                sizecmp if col == 2 else datecmp if col == 3 else tagscmp if col == 4 else libcmp
         self.map.sort(cmp=fcmp, reverse=descending)
         if len(self.map) == len(self.db):
@@ -1446,9 +1456,9 @@ class DeviceBooksModel(BooksModel):
                 au = self.db[self.map[row]].authors
                 if not au:
                     au = self.unknown
-                if role == Qt.EditRole:
+#                if role == Qt.EditRole:
-                    return QVariant(authors_to_string(au))
+#                    return QVariant(au)
-                return QVariant(" & ".join(au))
+                return QVariant(authors_to_string(au))
             elif col == 2:
                 size = self.db[self.map[row]].size
                 return QVariant(BooksView.human_readable(size))
@@ -1501,7 +1511,7 @@ class DeviceBooksModel(BooksModel):
                 self.db[idx].title = val
                 self.db[idx].title_sorter = val
             elif col == 1:
-                self.db[idx].authors = val
+                self.db[idx].authors = string_to_authors(val)
             elif col == 4:
                 tags = [i.strip() for i in val.split(',')]
                 tags = [t for t in tags if t]

src/calibre/gui2/ui.py

@@ -669,6 +669,15 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
         MainWindow.resizeEvent(self, ev)
         self.search.setMaximumWidth(self.width()-150)
 
+    def connect_to_folder(self):
+        dir = choose_dir(self, 'Select Device Folder', 'Select folder to open')
+        if dir is not None:
+            self.device_manager.connect_to_folder(dir)
+            self._sync_menu.disconnect_from_folder_action.setEnabled(True)
+
+    def disconnect_from_folder(self):
+        self.device_manager.disconnect_folder()
+
     def _sync_action_triggered(self, *args):
         m = getattr(self, '_sync_menu', None)
         if m is not None:
@@ -681,6 +690,8 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
                 SIGNAL('sync(PyQt_PyObject, PyQt_PyObject, PyQt_PyObject)'),
                 self.dispatch_sync_event)
         self._sync_menu.fetch_annotations.connect(self.fetch_annotations)
+        self._sync_menu.connect_to_folder.connect(self.connect_to_folder)
+        self._sync_menu.disconnect_from_folder.connect(self.disconnect_from_folder)
 
     def add_spare_server(self, *args):
         self.spare_servers.append(Server(limit=int(config['worker_limit']/2.0)))
@@ -939,6 +950,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
         Called when a device is connected to the computer.
         '''
         if connected:
+            self._sync_menu.connect_to_folder_action.setEnabled(False)
             self.device_manager.get_device_information(\
                     Dispatcher(self.info_read))
             self.set_default_thumbnail(\
@@ -952,8 +964,10 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
                     self.device_manager.device)
             self.location_view.model().device_connected(self.device_manager.device)
             self.eject_action.setEnabled(True)
-            self.refresh_ondevice_info (device_connected = True)
+            self.refresh_ondevice_info (device_connected = True, reset_only = True)
         else:
+            self._sync_menu.connect_to_folder_action.setEnabled(True)
+            self._sync_menu.disconnect_from_folder_action.setEnabled(False)
             self.save_device_view_settings()
             self.device_connected = False
             self._sync_menu.enable_device_actions(False)
@@ -1022,10 +1036,11 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
 
     ############################################################################
     ### Force the library view to refresh, taking into consideration books information
-    def refresh_ondevice_info(self, device_connected):
+    def refresh_ondevice_info(self, device_connected, reset_only = False):
-        # Save current column widths because we might be turning on OnDevice
-        self.library_view.write_settings()
         self.book_on_device(None, reset=True)
+        if reset_only:
+            return
+        self.library_view.write_settings()
         self.library_view.model().set_device_connected(device_connected)
     ############################################################################
 
@@ -1508,6 +1523,11 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
                     sm = view.selectionModel()
                     sm.select(ci, sm.Select)
         else:
+            if not confirm('<p>'+_('The selected books will be '
+                                   '<b>permanently deleted</b> '
+                                   'from your device. Are you sure?')
+                                +'</p>', 'library_delete_books', self):
+                return
             if self.stack.currentIndex() == 1:
                 view = self.memory_view
             elif self.stack.currentIndex() == 2:

src/calibre/library/caches.py

@@ -557,6 +557,12 @@ class ResultCache(SearchQueryParser):
     def count(self):
         return len(self._map)
 
+    def refresh_ondevice(self, db):
+        ondevice_col = self.FIELD_MAP['ondevice']
+        for item in self._data:
+            if item is not None:
+                item[ondevice_col] = db.book_on_device_string(item[0])
+
     def refresh(self, db, field=None, ascending=True):
         temp = db.conn.get('SELECT * FROM meta2')
         self._data = list(itertools.repeat(None, temp[-1][0]+2)) if temp else []

src/calibre/library/database2.py

@@ -245,6 +245,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         self.has_id  = self.data.has_id
         self.count   = self.data.count
 
+        self.refresh_ondevice = functools.partial(self.data.refresh_ondevice, self)
+
         self.refresh()
         self.last_update_check = self.last_modified()
 
@@ -470,14 +472,14 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             im = PILImage.open(f)
             im.convert('RGB').save(path, 'JPEG')
 
-    def book_on_device(self, index):
+    def book_on_device(self, id):
         if callable(self.book_on_device_func):
-            return self.book_on_device_func(index)
+            return self.book_on_device_func(id)
         return None
 
-    def book_on_device_string(self, index):
+    def book_on_device_string(self, id):
         loc = []
-        on = self.book_on_device(index)
+        on = self.book_on_device(id)
         if on is not None:
             m, a, b = on
             if m is not None: