MTP: Implement getting list of books and their metadata from device

src/calibre/devices/mtp/base.py

@@ -7,10 +7,17 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-from functools import wraps
+import re
+from functools import wraps, partial
 
+from calibre import prints
+from calibre.constants import DEBUG
 from calibre.devices.interface import DevicePlugin
 
+def debug(*args, **kwargs):
+    if DEBUG:
+        prints('MTP:', *args, **kwargs)
+
 def synchronous(func):
     @wraps(func)
     def synchronizer(self, *args, **kwargs):
@@ -53,4 +60,28 @@ class MTPDeviceBase(DevicePlugin):
         # return False
         return False
 
+    def build_template_regexp(self):
+        return None
+        # TODO: Implement this
+        def replfunc(match, seen=None):
+            v = match.group(1)
+            if v in ['authors', 'author_sort']:
+                v = 'author'
+            if v in ('title', 'series', 'series_index', 'isbn', 'author'):
+                if v not in seen:
+                    seen.add(v)
+                    return '(?P<' + v + '>.+?)'
+            return '(.+?)'
+        s = set()
+        f = partial(replfunc, seen=s)
+        template = None
+        try:
+            template = self.save_template().rpartition('/')[2]
+            return re.compile(re.sub('{([^}]*)}', f, template) + '([_\d]*$)')
+        except:
+            prints(u'Failed to parse template: %r'%template)
+            template = u'{title} - {authors}'
+            return re.compile(re.sub('{([^}]*)}', f, template) + '([_\d]*$)')
+
+
 

src/calibre/devices/mtp/books.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import os
+
+from calibre.devices.interface import BookList as BL
+from calibre.ebooks.metadata.book.base import Metadata
+from calibre.ebooks.metadata.book.json_codec import JsonCodec
+
+class BookList(BL):
+
+    def __init__(self, storage_id):
+        self.storage_id = storage_id
+
+    def supports_collections(self):
+        return False
+
+class Book(Metadata):
+
+    def __init__(self, storage_id, lpath, other=None):
+        Metadata.__init__(self, _('Unknown'), other=other)
+        self.storage_id, self.lpath = storage_id, lpath
+        self.lpath = self.lpath.replace(os.sep, '/')
+        self.mtp_relpath = tuple([icu_lower(x) for x in self.lpath.split('/')])
+
+    def matches_file(self, mtp_file):
+        return (self.storage_id == mtp_file.storage_id and
+                self.mtp_relpath == mtp_file.mtp_relpath)
+
+class JSONCodec(JsonCodec):
+    pass
+

src/calibre/devices/mtp/driver.py

@@ -7,10 +7,13 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import json, pprint
+import json, pprint, traceback
 from io import BytesIO
 
+from calibre import prints
 from calibre.constants import iswindows, numeric_version
+from calibre.devices.mtp.base import debug
+from calibre.ptempfile import SpooledTemporaryFile
 from calibre.utils.config import from_json, to_json
 from calibre.utils.date import now, isoformat
 
@@ -25,6 +28,7 @@ class MTP_DEVICE(BASE):
 
     METADATA_CACHE = 'metadata.calibre'
     DRIVEINFO = 'driveinfo.calibre'
+    CAN_SET_METADATA = []
 
     def _update_drive_info(self, storage, location_code, name=None):
         import uuid
@@ -81,6 +85,98 @@ class MTP_DEVICE(BASE):
         self._update_drive_info(self.filesystem_cache.storage(sid),
                 location_code, name=name)
 
+    def books(self, oncard=None, end_session=True):
+        from calibre.devices.mtp.books import JSONCodec
+        from calibre.devices.mtp.books import BookList, Book
+        sid = {'carda':self._carda_id, 'cardb':self._cardb_id}.get(oncard,
+                self._main_id)
+        if sid is None:
+            return BookList(None)
+
+        bl = BookList(sid)
+        # If True then there is a mismatch between the ebooks on the device and
+        # the metadata cache
+        need_sync = False
+        all_books = list(self.filesystem_cache.iterebooks(sid))
+        steps = len(all_books) + 2
+        count = 0
+
+        self.report_progress(0, _('Reading metadata from device'))
+        # Read the cache if it exists
+        storage = self.filesystem_cache.storage(sid)
+        cache = storage.find_path((self.METADATA_CACHE,))
+        if cache is not None:
+            json_codec = JSONCodec()
+            try:
+                stream = self.get_file(cache)
+                json_codec.decode_from_file(stream, bl, Book, sid)
+            except:
+                need_sync = True
+
+        relpath_cache = {b.mtp_relpath:i for i, b in enumerate(bl)}
+
+        for mtp_file in all_books:
+            count += 1
+            relpath = mtp_file.mtp_relpath
+            idx = relpath_cache.get(relpath, None)
+            if idx is not None:
+                cached_metadata = bl[idx]
+                del relpath_cache[relpath]
+                if cached_metadata.size == mtp_file.size:
+                    debug('Using cached metadata for',
+                            '/'.join(mtp_file.full_path))
+                    continue # No need to update metadata
+                book = cached_metadata
+            else:
+                book = Book(sid, '/'.join(relpath))
+                bl.append(book)
+
+            need_sync = True
+            self.report_progress(count/steps, _('Reading metadata from %s')%
+                    ('/'.join(relpath)))
+            try:
+                book.smart_update(self.read_file_metadata(mtp_file))
+                debug('Read metadata for', '/'.join(mtp_file.full_path))
+            except:
+                prints('Failed to read metadata from',
+                        '/'.join(mtp_file.full_path))
+                traceback.print_exc()
+            book.size = mtp_file.size
+
+        # Remove books in the cache that no longer exist
+        for idx in sorted(relpath_cache.itervalues(), reverse=True):
+            del bl[idx]
+            need_sync = True
+
+        if need_sync:
+            self.report_progress(count/steps, _('Updating metadata cache on device'))
+            self.write_metadata_cache(storage, bl)
+        self.report_progress(1, _('Finished reading metadata from device'))
+
+    def read_file_metadata(self, mtp_file):
+        from calibre.ebooks.metadata.meta import get_metadata
+        from calibre.customize.ui import quick_metadata
+        ext = mtp_file.name.rpartition('.')[-1].lower()
+        stream = self.get_file(mtp_file)
+        with quick_metadata:
+            return get_metadata(stream, stream_type=ext,
+                    force_read_metadata=True,
+                    pattern=self.build_template_regexp())
+
+    def write_metadata_cache(self, storage, bl):
+        from calibre.devices.mtp.books import JSONCodec
+
+        if bl.storage_id != storage.storage_id:
+            # Just a sanity check, should never happen
+            return
+
+        json_codec = JSONCodec()
+        stream = SpooledTemporaryFile(10*(1024**2))
+        json_codec.encode_to_file(stream, bl)
+        size = stream.tell()
+        stream.seek(0)
+        self.put_file(storage, self.METADATA_CACHE, stream, size)
+
 if __name__ == '__main__':
     dev = MTP_DEVICE(None)
     dev.startup()
@@ -92,8 +188,9 @@ if __name__ == '__main__':
         cd = dev.detect_managed_devices(devs)
         if cd is None:
             raise ValueError('Failed to detect MTP device')
+        dev.set_progress_reporter(prints)
         dev.open(cd, None)
-        pprint.pprint(dev.get_device_information())
+        dev.books()
     finally:
         dev.shutdown()
 

src/calibre/devices/mtp/filesystem_cache.py

@@ -14,6 +14,9 @@ from future_builtins import map
 
 from calibre import human_readable, prints, force_unicode
 from calibre.utils.icu import sort_key, lower
+from calibre.ebooks import BOOK_EXTENSIONS
+
+bexts = frozenset(BOOK_EXTENSIONS)
 
 class FileOrFolder(object):
 
@@ -50,6 +53,9 @@ class FileOrFolder(object):
         if self.storage_id == self.object_id:
             self.storage_prefix = 'mtp:::%s:::'%self.persistent_id
 
+        self.is_ebook = (not self.is_folder and
+                self.name.rpartition('.')[-1].lower() in bexts)
+
     def __repr__(self):
         name = 'Folder' if self.is_folder else 'File'
         try:
@@ -147,6 +153,9 @@ class FileOrFolder(object):
             parent = c
         return parent
 
+    @property
+    def mtp_relpath(self):
+        return tuple(x.lower() for x in self.full_path[1:])
 
 class FilesystemCache(object):
 
@@ -192,4 +201,9 @@ class FilesystemCache(object):
             if e.storage_id == storage_id:
                 return e
 
+    def iterebooks(self, storage_id):
+        for x in self.id_map.itervalues():
+            if x.storage_id == storage_id and x.is_ebook:
+                yield x
+
 

src/calibre/devices/mtp/unix/driver.py

@@ -17,7 +17,6 @@ from calibre.constants import plugins
 from calibre.ptempfile import SpooledTemporaryFile
 from calibre.devices.errors import OpenFailed, DeviceError
 from calibre.devices.mtp.base import MTPDeviceBase, synchronous
-from calibre.devices.mtp.filesystem_cache import FilesystemCache
 
 MTPDevice = namedtuple('MTPDevice', 'busnum devnum vendor_id product_id '
         'bcd serial manufacturer product')
@@ -175,6 +174,7 @@ class MTP_DEVICE(MTPDeviceBase):
     @property
     def filesystem_cache(self):
         if self._filesystem_cache is None:
+            from calibre.devices.mtp.filesystem_cache import FilesystemCache
             with self.lock:
                 storage, all_items, all_errs = [], [], []
                 for sid, capacity in zip([self._main_id, self._carda_id,

src/calibre/devices/mtp/windows/driver.py

@@ -17,7 +17,6 @@ from calibre.constants import plugins, __appname__, numeric_version
 from calibre.ptempfile import SpooledTemporaryFile
 from calibre.devices.errors import OpenFailed, DeviceError
 from calibre.devices.mtp.base import MTPDeviceBase
-from calibre.devices.mtp.filesystem_cache import FilesystemCache
 
 class ThreadingViolation(Exception):
 
@@ -143,6 +142,7 @@ class MTP_DEVICE(MTPDeviceBase):
     @property
     def filesystem_cache(self):
         if self._filesystem_cache is None:
+            from calibre.devices.mtp.filesystem_cache import FilesystemCache
             ts = self.total_space()
             all_storage = []
             items = []