MTP driver: Add code to dump the filesystem for debugging

src/calibre/devices/mtp/unix/driver.py

@@ -10,7 +10,10 @@ __docformat__ = 'restructuredtext en'
 import time, operator
 from threading import RLock
 from functools import wraps
+from itertools import chain
+from collections import deque
 
+from calibre import prints
 from calibre.devices.errors import OpenFailed
 from calibre.devices.mtp.base import MTPDeviceBase
 from calibre.devices.mtp.unix.detect import MTPDetect
@@ -22,6 +25,61 @@ def synchronous(func):
             return func(self, *args, **kwargs)
     return synchronizer
 
+class FilesystemCache(object):
+
+    def __init__(self, files, folders):
+        self.files = files
+        self.folders = folders
+        self.file_id_map = {f['id']:f for f in files}
+        self.folder_id_map = {f['id']:f for f in self.iterfolders(set_level=0)}
+
+        # Set the parents of each file
+        for f in files:
+            parents = deque()
+            pid = f['parent_id']
+            while pid is not None and pid > 0:
+                try:
+                    parent = self.folder_id_map[pid]
+                except KeyError:
+                    break
+                parents.appendleft(pid)
+                pid = parent['parent_id']
+
+        # Set the files in each folder
+        for f in self.iterfolders():
+            f['files'] = [i for i in files if i['parent_id'] ==
+                    f['id']]
+
+        # Decode the file and folder names
+        for f in chain(files, folders):
+            try:
+                name = f['name'].decode('utf-8')
+            except UnicodeDecodeError:
+                name = 'undecodable_%d'%f['id']
+            f['name'] = name
+
+    def iterfolders(self, folders=None, set_level=None):
+        clevel = None if set_level is None else set_level + 1
+        if folders is None:
+            folders = self.folders
+        for f in folders:
+            if set_level is not None:
+                f['level'] = set_level
+            yield f
+            for c in f['children']:
+                for child in self.iterfolders([c], set_level=clevel):
+                    yield child
+
+    def dump_filesystem(self):
+        indent = 2
+        for f in self.iterfolders():
+            prefix = ' '*(indent*f['level'])
+            prints(prefix, '+', f['name'], 'id=%s'%f['id'])
+            for leaf in f['files']:
+                prints(prefix, ' '*indent, '-', leaf['name'],
+                        'id=%d'%leaf['id'], 'size=%d'%leaf['size'],
+                        'modtime=%d'%leaf['modtime'])
+
 class MTP_DEVICE(MTPDeviceBase):
 
     supported_platforms = ['linux']
@@ -30,6 +88,7 @@ class MTP_DEVICE(MTPDeviceBase):
         MTPDeviceBase.__init__(self, *args, **kwargs)
         self.detect = MTPDetect()
         self.dev = None
+        self.filesystem_cache = None
         self.lock = RLock()
         self.blacklisted_devices = set()
 
@@ -79,8 +138,13 @@ class MTP_DEVICE(MTPDeviceBase):
     def shutdown(self):
         self.dev = None
 
+    def format_errorstack(self, errs):
+        return '\n'.join(['%d:%s'%(code, msg.decode('utf-8', 'replace')) for
+            code, msg in errs])
+
     @synchronous
     def open(self, connected_device, library_uuid):
+        self.dev = self.filesystem_cache = None
         def blacklist_device():
             d = connected_device
             self.blacklisted_devices.add((d.busnum, d.devnum, d.vendor_id,
@@ -112,6 +176,16 @@ class MTP_DEVICE(MTPDeviceBase):
         if len(storage) > 2:
             self._cardb_id = storage[2]['id']
 
+        files, errs = self.dev.get_filelist(self)
+        if errs and not files:
+            raise OpenFailed('Failed to read files from device. Underlying errors:\n'
+                    +self.format_errorstack(errs))
+        folders, errs = self.dev.get_folderlist()
+        if errs and not folders:
+            raise OpenFailed('Failed to read folders from device. Underlying errors:\n'
+                    +self.format_errorstack(errs))
+        self.filesystem_cache = FilesystemCache(files, folders)
+
     @synchronous
     def get_device_information(self, end_session=True):
         d = self.dev
@@ -160,8 +234,5 @@ if __name__ == '__main__':
     print ("Storage info:")
     pprint(d.storage_info)
     print("Free space:", dev.free_space())
-    files, errs = d.get_filelist(dev)
+    dev.filesystem_cache.dump_filesystem()
-    pprint((len(files), errs))
-    folders, errs = d.get_folderlist()
-    pprint((len(folders), errs))
 

src/calibre/devices/mtp/unix/libmtp.c

@@ -301,7 +301,7 @@ libmtp_Device_get_filelist(libmtp_Device *self, PyObject *args, PyObject *kwargs
                 "id", f->item_id,
                 "parent_id", f->parent_id,
                 "storage_id", f->storage_id,
-                "filename", f->filename,
+                "name", f->filename,
                 "size", f->filesize,
                 "modtime", f->modificationdate
         );
@@ -334,7 +334,7 @@ int folderiter(LIBMTP_folder_t *f, PyObject *parent) {
 
     folder = Py_BuildValue("{s:k,s:k,s:k,s:s,s:N}",
             "id", f->folder_id,
-            "parent_d", f->parent_id,
+            "parent_id", f->parent_id,
             "storage_id", f->storage_id,
             "name", f->name,
             "children", children);