Remove PRS500 driver and initial implementation of SONY XML cache update

src/calibre/customize/builtins.py

@@ -442,7 +442,6 @@ from calibre.devices.irexdr.driver import IREXDR1000, IREXDR800
 from calibre.devices.jetbook.driver import JETBOOK
 from calibre.devices.kindle.driver import KINDLE, KINDLE2, KINDLE_DX
 from calibre.devices.nook.driver import NOOK
-from calibre.devices.prs500.driver import PRS500
 from calibre.devices.prs505.driver import PRS505, PRS700
 from calibre.devices.android.driver import ANDROID, S60
 from calibre.devices.nokia.driver import N770, N810
@@ -512,7 +511,6 @@ plugins += [
     NOOK,
     PRS505,
     PRS700,
-    PRS500,
     ANDROID,
     S60,
     N770,

src/calibre/devices/interface.py

@@ -418,3 +418,16 @@ class BookList(list):
         '''
         raise NotImplementedError()
 
+    def get_collections(self, collection_attributes):
+        '''
+        Return a dictionary of collections created from collection_attributes.
+        Each entry in the dictionary is of the form collection name:[list of
+        books]
+
+        The list of books is sorted by book title, except for collections
+        created from series, in which case series_index is used.
+
+        :param collection_attributes: A list of attributes of the Book object
+        '''
+        raise NotImplementedError()
+

src/calibre/devices/prs505/driver.py

@@ -71,7 +71,7 @@ class PRS505(USBMS):
         return fname
 
     def initialize_XML_cache(self):
-        paths = {}
+        paths, prefixes = {}, {}
         for prefix, path, source_id in [
                 ('main', MEDIA_XML, 0),
                 ('card_a', CACHE_XML, 1),
@@ -80,10 +80,11 @@ class PRS505(USBMS):
             prefix = getattr(self, '_%s_prefix'%prefix)
             if prefix is not None and os.path.exists(prefix):
                 paths[source_id] = os.path.join(prefix, *(path.split('/')))
+                prefixes[source_id] = prefix
                 d = os.path.dirname(paths[source_id])
                 if not os.path.exists(d):
                     os.makedirs(d)
-        return XMLCache(paths)
+        return XMLCache(paths, prefixes)
 
     def books(self, oncard=None, end_session=True):
         bl = USBMS.books(self, oncard=oncard, end_session=end_session)
@@ -96,10 +97,15 @@ class PRS505(USBMS):
         blists = {}
         for i in c.paths:
             blists[i] = booklists[i]
-        c.update(blists)
+        opts = self.settings()
+        collections = ['series', 'tags']
+        if opts.extra_customization:
+            collections = opts.extra_customization.split(',')
+
+        c.update(blists, collections)
         c.write()
 
-        USBMS.sync_booklists(self, booklists, end_session)
+        USBMS.sync_booklists(self, booklists, end_session=end_session)
 
 class PRS700(PRS505):
 

src/calibre/devices/prs505/sony_cache.py

@@ -5,17 +5,18 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os
+import os, time
 from pprint import pprint
 from base64 import b64decode
+from uuid import uuid4
 
 from lxml import etree
 
-from calibre import prints
+from calibre import prints, guess_type
 from calibre.devices.errors import DeviceError
 from calibre.constants import DEBUG
 from calibre.ebooks.chardet import xml_to_unicode
-from calibre.ebooks.metadata import string_to_authors
+from calibre.ebooks.metadata import string_to_authors, authors_to_string
 
 EMPTY_CARD_CACHE = '''\
 <?xml version="1.0" encoding="UTF-8"?>
@@ -23,12 +24,43 @@ EMPTY_CARD_CACHE = '''\
 </cache>
 '''
 
+MIME_MAP   = {
+                "lrf" : "application/x-sony-bbeb",
+                'lrx' : 'application/x-sony-bbeb',
+                "rtf" : "application/rtf",
+                "pdf" : "application/pdf",
+                "txt" : "text/plain" ,
+                'epub': 'application/epub+zip',
+              }
+
+DAY_MAP   = dict(Sun=0, Mon=1, Tue=2, Wed=3, Thu=4, Fri=5, Sat=6)
+MONTH_MAP = dict(Jan=1, Feb=2, Mar=3, Apr=4, May=5, Jun=6, Jul=7, Aug=8, Sep=9, Oct=10, Nov=11, Dec=12)
+INVERSE_DAY_MAP = dict(zip(DAY_MAP.values(), DAY_MAP.keys()))
+INVERSE_MONTH_MAP = dict(zip(MONTH_MAP.values(), MONTH_MAP.keys()))
+
+def strptime(src):
+    src = src.strip()
+    src = src.split()
+    src[0] = str(DAY_MAP[src[0][:-1]])+','
+    src[2] = str(MONTH_MAP[src[2]])
+    return time.strptime(' '.join(src), '%w, %d %m %Y %H:%M:%S %Z')
+
+def strftime(epoch, zone=time.gmtime):
+    src = time.strftime("%w, %d %m %Y %H:%M:%S GMT", zone(epoch)).split()
+    src[0] = INVERSE_DAY_MAP[int(src[0][:-1])]+','
+    src[2] = INVERSE_MONTH_MAP[int(src[2])]
+    return ' '.join(src)
+
+def uuid():
+    return str(uuid4()).replace('-', '', 1).upper()
+
 class XMLCache(object):
 
-    def __init__(self, paths):
+    def __init__(self, paths, prefixes):
         if DEBUG:
             pprint(paths)
         self.paths = paths
+        self.prefixes = prefixes
         parser = etree.XMLParser(recover=True)
         self.roots = {}
         for source_id, path in paths.items():
@@ -50,7 +82,9 @@ class XMLCache(object):
 
         recs = self.roots[0].xpath('//*[local-name()="records"]')
         if not recs:
-            raise DeviceError('The SONY XML database is corrupted (no <records>)')
+            raise DeviceError('The SONY XML database is corrupted (no'
+                    ' <records>). Try disconnecting an reconnecting'
+                    ' your reader.')
         self.record_roots = {}
         self.record_roots.update(self.roots)
         self.record_roots[0] = recs[0]
@@ -75,11 +109,63 @@ class XMLCache(object):
         for i, root in self.record_roots.items():
             self.purge_broken_playlist_items(root)
             for playlist in root.xpath('//*[local-name()="playlist"]'):
-                if len(playlist) == 0:
+                if len(playlist) == 0 or not playlist.get('title', None):
                     if DEBUG:
-                        prints('Removing playlist:', playlist.get('id', None))
+                        prints('Removing playlist:', playlist.get('id', None),
+                                playlist.get('title', None))
                     playlist.getparent().remove(playlist)
 
+    def ensure_unique_playlist_titles(self):
+        for i, root in self.record_roots.items():
+            seen = set([])
+            for playlist in root.xpath('//*[local-name()="playlist"]'):
+                title = playlist.get('title', None)
+                if title is None:
+                    title = _('Unnamed')
+                    playlist.set('title', title)
+                if title in seen:
+                    for i in range(2, 1000):
+                        if title+str(i) not in seen:
+                            title = title+str(i)
+                            playlist.set('title', title)
+                            break
+                else:
+                    seen.add(title)
+
+    def get_playlist_map(self):
+        ans = {}
+        self.ensure_unique_playlist_titles()
+        self.prune_empty_playlists()
+        for i, root in self.record_roots.items():
+            for playlist in root.xpath('//*[local-name()="playlist"]'):
+                items = []
+                for item in playlist:
+                    id_ = item.get('id', None)
+                    records = root.xpath(
+                        '//*[local-name()="text" and @id="%s"]'%id_)
+                    if records:
+                        items.append(records[0])
+                ans[i] = {playlist.get('title'):items}
+        return ans
+
+    def get_or_create_playlist(self, bl_idx, title):
+        root = self.record_roots[bl_idx]
+        for playlist in root.xpath('//*[local-name()="playlist"]'):
+            if playlist.get('title', None) == title:
+                return playlist
+        ans = root.makelement('{%s}playlist'%self.namespaces[bl_idx],
+                nsmap=root.nsmap, attrib={
+                    'uuid' : uuid(),
+                    'title': title,
+                    'id'   : str(self.max_id(root)+1),
+                    'sourceid': '1'
+                    })
+        tail = '\n\t\t' if bl_idx == 0 else '\n\t'
+        ans.tail = tail
+        if len(root) > 0:
+            root.iterchildren(reversed=True).next().tail = tail
+        root.append(ans)
+        return ans
     # }}}
 
     def fix_ids(self): # {{{
@@ -189,11 +275,107 @@ class XMLCache(object):
                     break
     # }}}
 
-    def update(self, booklists):
+    # Update XML Cache {{{
-        pass
+    def update(self, booklists, collections_attributes):
+        playlist_map = self.get_playlist_map()
+
+        for i, booklist in booklists.items():
+            root = self.record_roots[i]
+            for book in booklist:
+                path = os.path.join(self.prefixes[i], *(book.lpath.split('/')))
+                record = self.book_by_lpath(book.lpath, root)
+                if record is None:
+                    record = self.create_text_record(root, i, book.lpath)
+                self.update_record(record, book, path, i)
+                bl_pmap = playlist_map[i]
+                self.update_playlists(i, root, booklist, bl_pmap,
+                        collections_attributes)
+
+            tail = '\n\t' if i == 0 else '\n'
+            if len(root) > 0:
+                root.iterchildren(reversed=True).next().tail = tail
+
+        self.fix_ids()
+
+    def update_playlists(self, bl_index, root, booklist, playlist_map,
+            collections_attributes):
+        collections = booklist.get_collections(collections_attributes)
+        for category, books in collections:
+            records = [self.book_by_lpath(b.lpath) for b in books]
+            # Remove any books that were not found, although this
+            # *should* never happen
+            if DEBUG and None in records:
+                prints('WARNING: Some elements in the JSON cache were not'
+                        'found in the XML cache')
+            records = [x for x in records if x is not None]
+            for rec in records:
+                if rec.get('id', None) is None:
+                    rec.set('id', str(self.max_id(root)+1))
+            ids = [x.get('id', None) for x in records]
+            if None in ids:
+                if DEBUG:
+                    prints('WARNING: Some <text> elements do not have ids')
+                    ids = [x for x in ids if x is not None]
+
+            playlist = self.get_or_create_playlist(bl_index, category)
+            playlist_ids = []
+            for item in playlist:
+                id_ = item.get('id', None)
+                if id_ is not None:
+                    playlist_ids.append(id_)
+            for item in list(playlist):
+                playlist.remove(item)
+
+            extra_ids = [x for x in playlist_ids if x not in ids]
+            tail = '\n\t\t\t' if bl_index == 0 else '\n\t\t'
+            playlist.tail = tail
+            for id_ in ids + extra_ids:
+                item = playlist.makeelement(
+                        '{%s}item'%self.namespaces[bl_index],
+                        nsmap=playlist.nsmap, attrib={'id':id_})
+                item.tail = tail
+            if len(playlist) > 0:
+                root.iterchildren(reversed=True).next().tail = tail[:-1]
+
+
+    def create_text_record(self, root, bl_id, lpath):
+        namespace = self.namespaces[bl_id]
+        id_ = self.max_id(root)+1
+        attrib = {
+                'page':'0', 'part':'0','pageOffset':'0','scale':'0',
+                'id':str(id_), 'sourceid':'1', 'path':lpath}
+        ans = root.makeelement('{%s}text'%namespace, attrib=attrib, nsmap=root.nsmap)
+        tail = '\n\t\t' if bl_id == 0 else '\n\t'
+        ans.tail = tail
+        if len(root) > 0:
+            root.iterchildren(reversed=True).next().tail = tail
+        root.append(ans)
+        return ans
+
+    def update_text_record(self, record, book, path, bl_index):
+        timestamp = 'ctime' if bl_index == 0 else 'mtime'
+        timestamp = getattr(os.path, 'get'+timestamp)(path)
+        date = strftime(timestamp)
+        record.set('date', date)
+        record.set('size', os.stat(path).st_size)
+        record.set('title', book.title)
+        record.set('author', authors_to_string(book.authors))
+        ext = os.path.splitext(path)[1]
+        if ext:
+            ext = ext[1:].lower()
+            mime = MIME_MAP.get(ext, None)
+            if mime is None:
+                mime = guess_type('a.'+ext)[0]
+                if mime is not None:
+                    record.set('mime', mime)
+        if 'sourceid' not in record.attrib:
+            record.set('sourceid', '1')
+        if 'id' not in record.attrib:
+            num = self.max_id(record.getroottree().getroot())
+            record.set('id', str(num+1))
+    # }}}
 
     def write(self):
-        return
         for i, path in self.paths.items():
             raw = etree.tostring(self.roots[i], encoding='utf-8',
                     xml_declaration=True)

src/calibre/devices/usbms/books.py

@@ -4,9 +4,7 @@ __license__ = 'GPL 3'
 __copyright__ = '2009, John Schember <john@nachtimwald.com>'
 __docformat__ = 'restructuredtext en'
 
-import os
+import os, re, time, sys
-import re
-import time
 
 from calibre.ebooks.metadata import MetaInformation
 from calibre.devices.mime import mime_type_ext
@@ -110,6 +108,9 @@ class Book(MetaInformation):
             if isbytestring(val):
                 enc = filesystem_encoding if attr == 'lpath' else preferred_encoding
                 val = val.decode(enc, 'replace')
+            elif isinstance(val, (list, tuple)):
+                val = [x.decode(preferred_encoding, 'replace') if
+                        isbytestring(x) else x for x in val]
             json[attr] = val
         return json
 
@@ -129,3 +130,34 @@ class BookList(_BookList):
 
     def remove_book(self, book):
         self.remove(book)
+
+    def get_collections(self, collection_attributes):
+        collections = {}
+        series_categories = set([])
+        for attr in collection_attributes:
+            for book in self:
+                val = getattr(book, attr, None)
+                if not val: continue
+                if isbytestring(val):
+                    val = val.decode(preferred_encoding, 'replace')
+                if isinstance(val, (list, tuple)):
+                    val = list(val)
+                elif isinstance(val, unicode):
+                    val = [val]
+                for category in val:
+                    if category not in collections:
+                        collections[category] = []
+                    collections[category].append(book)
+                    if attr == 'series':
+                        series_categories.add(category)
+        for category, books in collections.items():
+            def tgetter(x):
+                return getattr(x, 'title_sort', 'zzzz')
+            books.sort(cmp=lambda x,y:cmp(tgetter(x), tgetter(y)))
+            if category in series_categories:
+                # Ensures books are sub sorted by title
+                def getter(x):
+                    return getattr(x, 'series_index', sys.maxint)
+                books.sort(cmp=lambda x,y:cmp(getter(x), getter(y)))
+        return collections
+

src/calibre/gui2/wizard/__init__.py

@@ -78,18 +78,12 @@ class KindleDX(Kindle):
     name = 'Kindle DX'
     id = 'kindledx'
 
-class Sony500(Device):
+class Sony505(Device):
 
     output_profile = 'sony'
-    name = 'SONY PRS 500'
+    name = 'SONY Reader 6" and Touch Editions'
-    output_format = 'LRF'
-    manufacturer = 'SONY'
-    id = 'prs500'
-
-class Sony505(Sony500):
-
     output_format = 'EPUB'
-    name = 'SONY Reader 6" and Touch Edition'
+    manufacturer = 'SONY'
     id = 'prs505'
 
 class Kobo(Device):