MTP: Cross platform filesystem cache

src/calibre/__init__.py

@@ -674,7 +674,7 @@ def get_download_filename(url, cookie_file=None):
 
     return filename
 
-def human_readable(size):
+def human_readable(size, sep=' '):
     """ Convert a size in bytes into a human readable form """
     divisor, suffix = 1, "B"
     for i, candidate in enumerate(('B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB')):
@@ -686,7 +686,7 @@ def human_readable(size):
         size = size[:size.find(".")+2]
     if size.endswith('.0'):
         size = size[:-2]
-    return size + " " + suffix
+    return size + sep + suffix
 
 def remove_bracketed_text(src,
         brackets={u'(':u')', u'[':u']', u'{':u'}'}):

src/calibre/devices/mtp/filesystem_cache.py

@@ -7,10 +7,85 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
+import weakref, sys
+from operator import attrgetter
+from future_builtins import map
+
+from calibre import human_readable, prints, force_unicode
+from calibre.utils.icu import sort_key
+
+class FileOrFolder(object):
+
+    def __init__(self, entry, fs_cache, all_storage_ids):
+        self.object_id = entry['id']
+        self.is_folder = entry['is_folder']
+        self.name = force_unicode(entry.get('name', '___'), 'utf-8')
+        self.persistent_id = entry.get('persistent_id', self.object_id)
+        self.size = entry.get('size', 0)
+        # self.parent_id is None for storage objects
+        self.parent_id = entry.get('parent_id', None)
+        if self.parent_id == 0:
+            sid = entry['storage_id']
+            if sid not in all_storage_ids:
+                sid = all_storage_ids[0]
+            self.parent_id = sid
+        self.is_hidden = entry.get('is_hidden', False)
+        self.is_system = entry.get('is_system', False)
+        self.can_delete = entry.get('can_delete', True)
+
+        self.files = []
+        self.folders = []
+        fs_cache.id_map[self.object_id] = self
+        self.fs_cache = weakref.ref(fs_cache)
+
+    @property
+    def id_map(self):
+        return self.fs_cache().id_map
+
+    @property
+    def parent(self):
+        return None if self.parent_id is None else self.id_map[self.parent_id]
+
+    def __iter__(self):
+        for e in self.folders:
+            yield e
+        for e in self.files:
+            yield e
+
+    def dump(self, prefix='', out=sys.stdout):
+        c = '+' if self.is_folder else '-'
+        data = ('%s children'%(sum(map(len, (self.files, self.folders))))
+            if self.is_folder else human_readable(self.size))
+        line = '%s%s %s [id:%s %s]'%(prefix, c, self.name, self.object_id, data)
+        prints(line, file=out)
+        for c in (self.folders, self.files):
+            for e in sorted(c, key=lambda x:sort_key(x.name)):
+                e.dump(prefix=prefix+'  ', out=out)
+
 class FilesystemCache(object):
 
-    def __init__(self, storage_map):
+    def __init__(self, all_storage, entries):
-        self.tree = {}
+        self.entries = []
-        for storage_id, id_map in storage_map.iteritems():
+        self.id_map = {}
-            self.tree[storage_id] = self.build_tree(id_map)
+
+        for storage in all_storage:
+            e = FileOrFolder(storage, self, [])
+            self.entries.append(e)
+
+        self.entries.sort(key=attrgetter('object_id'))
+        all_storage_ids = [x.object_id for x in self.entries]
+
+        for entry in entries:
+            FileOrFolder(entry, self, all_storage_ids)
+
+        for item in self.id_map.itervalues():
+            p = item.parent
+            if p is not None:
+                t = p.folders if item.is_folder else p.files
+                t.append(item)
+
+    def dump(self, out=sys.stdout):
+        for e in self.entries:
+            e.dump(out=out)
+
 

src/calibre/devices/mtp/unix/driver.py

@@ -9,77 +9,13 @@ __docformat__ = 'restructuredtext en'
 
 import time, operator
 from threading import RLock
-from itertools import chain
-from collections import deque, OrderedDict
 from io import BytesIO
 
-from calibre import prints
 from calibre.devices.errors import OpenFailed, DeviceError
 from calibre.devices.mtp.base import MTPDeviceBase, synchronous
+from calibre.devices.mtp.filesystem_cache import FilesystemCache
 from calibre.devices.mtp.unix.detect import MTPDetect
 
-class FilesystemCache(object):
-
-    def __init__(self, files, folders):
-        self.files = files
-        self.folders = folders
-        self.file_id_map = {f['id']:f for f in files}
-        self.folder_id_map = {f['id']:f for f in self.iterfolders(set_level=0)}
-
-        # Set the parents of each file
-        self.files_in_root = OrderedDict()
-        for f in files:
-            parents = deque()
-            pid = f['parent_id']
-            while pid is not None and pid > 0:
-                try:
-                    parent = self.folder_id_map[pid]
-                except KeyError:
-                    break
-                parents.appendleft(pid)
-                pid = parent['parent_id']
-            f['parents'] = parents
-            if not parents:
-                self.files_in_root[f['id']] = f
-
-        # Set the files in each folder
-        for f in self.iterfolders():
-            f['files'] = [i for i in files if i['parent_id'] ==
-                    f['id']]
-
-        # Decode the file and folder names
-        for f in chain(files, folders):
-            try:
-                name = f['name'].decode('utf-8')
-            except UnicodeDecodeError:
-                name = 'undecodable_%d'%f['id']
-            f['name'] = name
-
-    def iterfolders(self, folders=None, set_level=None):
-        clevel = None if set_level is None else set_level + 1
-        if folders is None:
-            folders = self.folders
-        for f in folders:
-            if set_level is not None:
-                f['level'] = set_level
-            yield f
-            for c in f['children']:
-                for child in self.iterfolders([c], set_level=clevel):
-                    yield child
-
-    def dump_filesystem(self):
-        indent = 2
-        for f in self.iterfolders():
-            prefix = ' '*(indent*f['level'])
-            prints(prefix, '+', f['name'], 'id=%s'%f['id'])
-            for leaf in f['files']:
-                prints(prefix, ' '*indent, '-', leaf['name'],
-                        'id=%d'%leaf['id'], 'size=%d'%leaf['size'],
-                        'modtime=%d'%leaf['modtime'])
-        for leaf in self.files_in_root.itervalues():
-            prints('-', leaf['name'], 'id=%d'%leaf['id'],
-                    'size=%d'%leaf['size'], 'modtime=%d'%leaf['modtime'])
-
 class MTP_DEVICE(MTPDeviceBase):
 
     supported_platforms = ['linux']
@@ -87,7 +23,7 @@ class MTP_DEVICE(MTPDeviceBase):
     def __init__(self, *args, **kwargs):
         MTPDeviceBase.__init__(self, *args, **kwargs)
         self.dev = None
-        self.filesystem_cache = None
+        self._filesystem_cache = None
         self.lock = RLock()
         self.blacklisted_devices = set()
 
@@ -129,7 +65,7 @@ class MTP_DEVICE(MTPDeviceBase):
 
     @synchronous
     def post_yank_cleanup(self):
-        self.dev = self.filesystem_cache = self.current_friendly_name = None
+        self.dev = self._filesystem_cache = self.current_friendly_name = None
 
     @synchronous
     def startup(self):
@@ -140,7 +76,7 @@ class MTP_DEVICE(MTPDeviceBase):
 
     @synchronous
     def shutdown(self):
-        self.dev = self.filesystem_cache = None
+        self.dev = self._filesystem_cache = None
 
     def format_errorstack(self, errs):
         return '\n'.join(['%d:%s'%(code, msg.decode('utf-8', 'replace')) for
@@ -148,7 +84,7 @@ class MTP_DEVICE(MTPDeviceBase):
 
     @synchronous
     def open(self, connected_device, library_uuid):
-        self.dev = self.filesystem_cache = None
+        self.dev = self._filesystem_cache = None
         def blacklist_device():
             d = connected_device
             self.blacklisted_devices.add((d.busnum, d.devnum, d.vendor_id,
@@ -179,11 +115,12 @@ class MTP_DEVICE(MTPDeviceBase):
             self._carda_id = storage[1]['id']
         if len(storage) > 2:
             self._cardb_id = storage[2]['id']
-        self.current_friendly_name = self.dev.name
+        self.current_friendly_name = self.dev.friendly_name
 
-    @synchronous
+    @property
-    def read_filesystem_cache(self):
+    def filesystem_cache(self):
-        try:
+        if self._filesystem_cache is None:
+            with self.lock:
                 files, errs = self.dev.get_filelist(self)
                 if errs and not files:
                     raise DeviceError('Failed to read files from device. Underlying errors:\n'
@@ -192,10 +129,27 @@ class MTP_DEVICE(MTPDeviceBase):
                 if errs and not folders:
                     raise DeviceError('Failed to read folders from device. Underlying errors:\n'
                             +self.format_errorstack(errs))
-            self.filesystem_cache = FilesystemCache(files, folders)
+                storage = []
-        except:
+                for sid, capacity in zip([self._main_id, self._carda_id,
-            self.dev = self._main_id = self._carda_id = self._cardb_id = None
+                    self._cardb_id], self.total_space()):
-            raise
+                    if sid is not None:
+                        name = _('Unknown')
+                        for x in self.dev.storage_info:
+                            if x['id'] == sid:
+                                name = x['name']
+                                break
+                        storage.append({'id':sid, 'size':capacity,
+                            'is_folder':True, 'name':name})
+                all_folders = []
+                def recurse(f):
+                    all_folders.append(f)
+                    for c in f['children']:
+                        recurse(c)
+
+                for f in folders: recurse(f)
+                self._filesystem_cache = FilesystemCache(storage,
+                        all_folders+files)
+        return self._filesystem_cache
 
     @synchronous
     def get_device_information(self, end_session=True):
@@ -246,7 +200,6 @@ if __name__ == '__main__':
     devs = linux_scanner()
     mtp_devs = dev.detect(devs)
     dev.open(list(mtp_devs)[0], 'xxx')
-    dev.read_filesystem_cache()
     d = dev.dev
     print ("Opened device:", dev.get_gui_name())
     print ("Storage info:")
@@ -257,7 +210,7 @@ if __name__ == '__main__':
     # fname = b'moose.txt'
     # src = BytesIO(raw)
     # print (d.put_file(dev._main_id, 0, fname, src, len(raw), PR()))
-    dev.filesystem_cache.dump_filesystem()
+    dev.filesystem_cache.dump()
     # with open('/tmp/flint.epub', 'wb') as f:
     #     print(d.get_file(786, f, PR()))
     # print()

src/calibre/devices/mtp/unix/libmtp.c

@@ -315,7 +315,7 @@ libmtp_Device_storage_info(libmtp_Device *self, void *closure) {
                 "capacity", storage->MaxCapacity,
                 "freespace_bytes", storage->FreeSpaceInBytes,
                 "freespace_objects", storage->FreeSpaceInObjects,
-                "storage_desc", storage->StorageDescription,
+                "name", storage->StorageDescription,
                 "volume_id", storage->VolumeIdentifier
         );
 
@@ -358,13 +358,14 @@ libmtp_Device_get_filelist(libmtp_Device *self, PyObject *args, PyObject *kwargs
     }
 
     for (f=tf; f != NULL; f=f->next) {
-        fo = Py_BuildValue("{s:k,s:k,s:k,s:s,s:K,s:k}",
+        fo = Py_BuildValue("{s:k,s:k,s:k,s:s,s:K,s:k,s:O}",
                 "id", f->item_id,
                 "parent_id", f->parent_id,
                 "storage_id", f->storage_id,
                 "name", f->filename,
                 "size", f->filesize,
-                "modtime", f->modificationdate
+                "modtime", f->modificationdate,
+                "is_folder", Py_False
         );
         if (fo == NULL || PyList_Append(ans, fo) != 0) break;
         Py_DECREF(fo);
@@ -393,11 +394,12 @@ int folderiter(LIBMTP_folder_t *f, PyObject *parent) {
     children = PyList_New(0);
     if (children == NULL) { PyErr_NoMemory(); return 1;}
 
-    folder = Py_BuildValue("{s:k,s:k,s:k,s:s,s:N}",
+    folder = Py_BuildValue("{s:k,s:k,s:k,s:s,s:O,s:N}",
             "id", f->folder_id,
             "parent_id", f->parent_id,
             "storage_id", f->storage_id,
             "name", f->name,
+            "is_folder", Py_True,
             "children", children);
     if (folder == NULL) return 1;
     PyList_Append(parent, folder);

src/calibre/devices/mtp/windows/content_enumeration.cpp

@@ -28,7 +28,7 @@ static IPortableDeviceKeyCollection* create_filesystem_properties_collection() {
     ADDPROP(WPD_OBJECT_PARENT_ID);
     ADDPROP(WPD_OBJECT_PERSISTENT_UNIQUE_ID);
     ADDPROP(WPD_OBJECT_NAME);
-    ADDPROP(WPD_OBJECT_SYNC_ID);
+    // ADDPROP(WPD_OBJECT_SYNC_ID);
     ADDPROP(WPD_OBJECT_ISSYSTEM);
     ADDPROP(WPD_OBJECT_ISHIDDEN);
     ADDPROP(WPD_OBJECT_CAN_DELETE);
@@ -93,7 +93,7 @@ static void set_properties(PyObject *obj, IPortableDeviceValues *values) {
 
     set_string_property(obj, WPD_OBJECT_PARENT_ID, "parent_id", values);
     set_string_property(obj, WPD_OBJECT_NAME, "name", values);
-    set_string_property(obj, WPD_OBJECT_SYNC_ID, "sync_id", values);
+    // set_string_property(obj, WPD_OBJECT_SYNC_ID, "sync_id", values);
     set_string_property(obj, WPD_OBJECT_PERSISTENT_UNIQUE_ID, "persistent_id", values);
 
     set_bool_property(obj, WPD_OBJECT_ISHIDDEN, "is_hidden", values);

src/calibre/devices/mtp/windows/driver.py

@@ -9,19 +9,27 @@ __docformat__ = 'restructuredtext en'
 
 import time, threading
 from functools import wraps
+from future_builtins import zip
+from itertools import chain
 
 from calibre import as_unicode, prints
 from calibre.constants import plugins, __appname__, numeric_version
 from calibre.ptempfile import SpooledTemporaryFile
 from calibre.devices.errors import OpenFailed
 from calibre.devices.mtp.base import MTPDeviceBase
+from calibre.devices.mtp.filesystem_cache import FilesystemCache
+
+class ThreadingViolation(Exception):
+
+    def __init__(self):
+        Exception.__init__('You cannot use the MTP driver from a thread other than the '
+                ' thread in which startup() was called')
 
 def same_thread(func):
     @wraps(func)
     def check_thread(self, *args, **kwargs):
         if self.start_thread is not threading.current_thread():
-            raise Exception('You cannot use %s from a thread other than the '
+            raise ThreadingViolation()
-                ' thread in which startup() was called'%self.__class__.__name__)
         return func(self, *args, **kwargs)
     return check_thread
 
@@ -42,6 +50,7 @@ class MTP_DEVICE(MTPDeviceBase):
         self.wpd = self.wpd_error = None
         self._main_id = self._carda_id = self._cardb_id = None
         self.start_thread = None
+        self._filesystem_cache = None
 
     def startup(self):
         self.start_thread = threading.current_thread()
@@ -59,7 +68,7 @@ class MTP_DEVICE(MTPDeviceBase):
 
     @same_thread
     def shutdown(self):
-        self.dev = self.filesystem_cache = self.start_thread = None
+        self.dev = self._filesystem_cache = self.start_thread = None
         if self.wpd is not None:
             self.wpd.uninit()
 
@@ -130,11 +139,33 @@ class MTP_DEVICE(MTPDeviceBase):
 
         return True
 
+    @property
+    def filesystem_cache(self):
+        if self._filesystem_cache is None:
+            ts = self.total_space()
+            all_storage = []
+            items = []
+            for storage_id, capacity in zip([self._main_id, self._carda_id,
+                self._cardb_id], ts):
+                if storage_id is None: continue
+                name = _('Unknown')
+                for s in self.dev.data['storage']:
+                    if s['id'] == storage_id:
+                        name = s['name']
+                        break
+                storage = {'id':storage_id, 'size':capacity, 'name':name,
+                        'is_folder':True}
+                id_map = self.dev.get_filesystem(storage_id)
+                all_storage.append(storage)
+                items.append(id_map.itervalues())
+            self._filesystem_cache = FilesystemCache(all_storage, chain(*items))
+        return self._filesystem_cache
+
     @same_thread
     def post_yank_cleanup(self):
         self.currently_connected_pnp_id = self.current_friendly_name = None
         self._main_id = self._carda_id = self._cardb_id = None
-        self.dev = self.filesystem_cache = None
+        self.dev = self._filesystem_cache = None
 
     @same_thread
     def eject(self):
@@ -142,11 +173,11 @@ class MTP_DEVICE(MTPDeviceBase):
         self.ejected_devices.add(self.currently_connected_pnp_id)
         self.currently_connected_pnp_id = self.current_friendly_name = None
         self._main_id = self._carda_id = self._cardb_id = None
-        self.dev = self.filesystem_cache = None
+        self.dev = self._filesystem_cache = None
 
     @same_thread
     def open(self, connected_device, library_uuid):
-        self.dev = self.filesystem_cache = None
+        self.dev = self._filesystem_cache = None
         try:
             self.dev = self.wpd.Device(connected_device)
         except self.wpd.WPDError:

src/calibre/devices/mtp/windows/remote.py

@@ -70,10 +70,10 @@ def main():
         print ('Connected to:', dev.get_gui_name())
         print ('Total space', dev.total_space())
         print ('Free space', dev.free_space())
-        # pprint.pprint(dev.dev.get_filesystem(dev._main_id))
+        dev.filesystem_cache.dump()
-        print ('Fetching file: oFF (198214 bytes)')
+        # print ('Fetching file: oFF (198214 bytes)')
-        stream = dev.get_file('oFF')
+        # stream = dev.get_file('oFF')
-        print ("Fetched size: ", stream.tell())
+        # print ("Fetched size: ", stream.tell())
     finally:
         dev.shutdown()