MTP driver: Allow ignoring any folder on the device

MTP driver: Allow ignoring any folder on the device, not just top level
folders. For newly connected devices, scan
/Android/data/com.amazon.kindle for books by default (newer versions of
the Kindle app place downloaded files there).

src/calibre/devices/mtp/driver.py

@@ -70,20 +70,34 @@ class MTP_DEVICE(BASE):
 
         return self._prefs
 
-    def is_folder_ignored(self, storage_or_storage_id, name,
+    def is_folder_ignored(self, storage_or_storage_id, path,
                           ignored_folders=None):
         storage_id = unicode(getattr(storage_or_storage_id, 'object_id',
                              storage_or_storage_id))
-        name = icu_lower(name)
+        lpath = tuple(icu_lower(name) for name in path)
         if ignored_folders is None:
             ignored_folders = self.get_pref('ignored_folders')
         if storage_id in ignored_folders:
-            return name in {icu_lower(x) for x in ignored_folders[storage_id]}
+            # Use the users ignored folders settings
+            return '/'.join(lpath) in {icu_lower(x) for x in ignored_folders[storage_id]}
 
-        return name in {
+        # Implement the default ignore policy
-            'alarms', 'android', 'dcim', 'movies', 'music', 'notifications',
+
+        # Top level ignores
+        if lpath[0] in {
+            'alarms', 'dcim', 'movies', 'music', 'notifications',
             'pictures', 'ringtones', 'samsung', 'sony', 'htc', 'bluetooth',
-            'games', 'lost.dir', 'video', 'whatsapp', 'image', 'com.zinio.mobile.android.reader'}
+            'games', 'lost.dir', 'video', 'whatsapp', 'image', 'com.zinio.mobile.android.reader'}:
+            return True
+
+        if len(lpath) > 1 and lpath[0] == 'android':
+            # Ignore everything in Android apart from a few select folders
+            if lpath[1] != 'data':
+                return True
+            if len(lpath) > 2 and lpath[2] != 'com.amazon.kindle':
+                return True
+
+        return False
 
     def configure_for_kindle_app(self):
         proxy = self.prefs
@@ -398,8 +412,8 @@ class MTP_DEVICE(BASE):
 
         for infile, fname, mi in izip(files, names, metadata):
             path = self.create_upload_path(prefix, mi, fname, routing)
-            if path and self.is_folder_ignored(storage, path[0]):
+            if path and self.is_folder_ignored(storage, path):
-                raise MTPInvalidSendPathError(path[0])
+                raise MTPInvalidSendPathError('/'.join(path))
             parent = self.ensure_parent(storage, path)
             if hasattr(infile, 'read'):
                 pos = infile.tell()
@@ -549,6 +563,3 @@ if __name__ == '__main__':
         print ('Prefix for main mem:', dev.prefix_for_location(None))
     finally:
         dev.shutdown()
-
-
-

src/calibre/devices/mtp/unix/driver.py

@@ -21,6 +21,7 @@ from calibre.devices.mtp.base import MTPDeviceBase, synchronous, debug
 MTPDevice = namedtuple('MTPDevice', 'busnum devnum vendor_id product_id '
         'bcd serial manufacturer product')
 
+null = object()
 def fingerprint(d):
     return MTPDevice(d.busnum, d.devnum, d.vendor_id, d.product_id, d.bcd,
             d.serial, d.manufacturer, d.product)
@@ -230,13 +231,23 @@ class MTP_DEVICE(MTPDeviceBase):
         ans += pprint.pformat(storage)
         return ans
 
-    def _filesystem_callback(self, entry, level):
+    def _filesystem_callback(self, fs_map, entry, level):
         name = entry.get('name', '')
         self.filesystem_callback(_('Found object: %s')%name)
-        if (level == 0 and
+        fs_map[entry.get('id', null)] = entry
-            self.is_folder_ignored(self._currently_getting_sid, name)):
+        path = [name]
-            return False
+        pid = entry.get('parent_id', 0)
-        return True
+        while pid != 0 and pid in fs_map:
+            parent = fs_map[pid]
+            path.append(parent.get('name', ''))
+            pid = parent.get('parent_id', 0)
+            if fs_map.get(pid, None) is parent:
+                break  # An object is its own parent
+        path = tuple(reversed(path))
+        ok = not self.is_folder_ignored(self._currently_getting_sid, path)
+        if not ok:
+            debug('Ignored object: %s' % '/'.join(path))
+        return ok
 
     @property
     def filesystem_cache(self):
@@ -260,7 +271,7 @@ class MTP_DEVICE(MTPDeviceBase):
                         'is_system':True})
                     self._currently_getting_sid = unicode(sid)
                     items, errs = self.dev.get_filesystem(sid,
-                            self._filesystem_callback)
+                            partial(self._filesystem_callback, {}))
                     all_items.extend(items), all_errs.extend(errs)
                 if not all_items and all_errs:
                     raise DeviceError(

src/calibre/devices/mtp/windows/driver.py

@@ -18,6 +18,8 @@ from calibre.ptempfile import SpooledTemporaryFile
 from calibre.devices.errors import OpenFailed, DeviceError, BlacklistedDevice
 from calibre.devices.mtp.base import MTPDeviceBase, debug
 
+null = object()
+
 class ThreadingViolation(Exception):
 
     def __init__(self):
@@ -219,14 +221,26 @@ class MTP_DEVICE(MTPDeviceBase):
 
         return True
 
-    def _filesystem_callback(self, obj, level):
+    def _filesystem_callback(self, fs_map, obj, level):
-        n = obj.get('name', '')
+        name = obj.get('name', '')
-        msg = _('Found object: %s')%n
+        self.filesystem_callback(_('Found object: %s')%name)
-        if (level == 0 and
+        if not obj.get('is_folder', False):
-            self.is_folder_ignored(self._currently_getting_sid, n)):
             return False
-        self.filesystem_callback(msg)
+        fs_map[obj.get('id', null)] = obj
-        return obj.get('is_folder', False)
+        path = [name]
+        pid = obj.get('parent_id', 0)
+        while pid != 0 and pid in fs_map:
+            parent = fs_map[pid]
+            path.append(parent.get('name', ''))
+            pid = parent.get('parent_id', 0)
+            if fs_map.get(pid, None) is parent:
+                break  # An object is its own parent
+
+        path = tuple(reversed(path))
+        ok = not self.is_folder_ignored(self._currently_getting_sid, path)
+        if not ok:
+            debug('Ignored object: %s' % '/'.join(path))
+        return ok
 
     @property
     def filesystem_cache(self):
@@ -249,8 +263,8 @@ class MTP_DEVICE(MTPDeviceBase):
                 storage = {'id':storage_id, 'size':capacity, 'name':name,
                         'is_folder':True, 'can_delete':False, 'is_system':True}
                 self._currently_getting_sid = unicode(storage_id)
-                id_map = self.dev.get_filesystem(storage_id,
+                id_map = self.dev.get_filesystem(storage_id, partial(
-                        self._filesystem_callback)
+                        self._filesystem_callback, {}))
                 for x in id_map.itervalues():
                     x['storage_id'] = storage_id
                 all_storage.append(storage)

src/calibre/gui2/device_drivers/mtp_config.py

@@ -19,7 +19,7 @@ from calibre.ebooks import BOOK_EXTENSIONS
 from calibre.gui2 import error_dialog
 from calibre.gui2.dialogs.template_dialog import TemplateDialog
 from calibre.utils.date import parse_date
-from calibre.gui2.device_drivers.mtp_folder_browser import Browser, TopLevel
+from calibre.gui2.device_drivers.mtp_folder_browser import Browser, IgnoredFolders
 
 class FormatsConfig(QWidget): # {{{
 
@@ -424,7 +424,7 @@ class MTPConfig(QTabWidget):
         d.exec_()
 
     def change_ignored_folders(self):
-        d = TopLevel(self.device,
+        d = IgnoredFolders(self.device,
                      self.current_ignored_folders, parent=self)
         if d.exec_() == d.Accepted:
             self.current_ignored_folders = d.ignored_folders

src/calibre/gui2/device_drivers/mtp_folder_browser.py

@@ -10,12 +10,11 @@ __docformat__ = 'restructuredtext en'
 from operator import attrgetter
 
 from PyQt4.Qt import (QTabWidget, QTreeWidget, QTreeWidgetItem, Qt, QDialog,
-        QDialogButtonBox, QVBoxLayout, QSize, pyqtSignal, QIcon, QLabel,
+        QDialogButtonBox, QVBoxLayout, QSize, pyqtSignal, QIcon, QLabel)
-        QListWidget, QListWidgetItem)
 
 from calibre.gui2 import file_icon_provider
 
-def item(f, parent):
+def browser_item(f, parent):
     name = f.name
     if not f.is_folder:
         name += ' [%s]'%f.last_mod_string
@@ -31,22 +30,24 @@ def item(f, parent):
 
 class Storage(QTreeWidget):
 
-    def __init__(self, storage, show_files):
+    def __init__(self, storage, show_files=False, item_func=browser_item):
         QTreeWidget.__init__(self)
+        self.item_func = item_func
         self.show_files = show_files
         self.create_children(storage, self)
         self.name = storage.name
         self.object_id = storage.persistent_id
         self.setMinimumHeight(350)
         self.setHeaderHidden(True)
+        self.storage = storage
 
     def create_children(self, f, parent):
         for child in sorted(f.folders, key=attrgetter('name')):
-            i = item(child, parent)
+            i = self.item_func(child, parent)
             self.create_children(child, i)
         if self.show_files:
             for child in sorted(f.files, key=attrgetter('name')):
-                i = item(child, parent)
+                i = self.item_func(child, parent)
 
     @property
     def current_item(self):
@@ -96,14 +97,14 @@ class Browser(QDialog):
     def current_item(self):
         return self.folders.current_item
 
-class TopLevel(QDialog):
+class IgnoredFolders(QDialog):
 
     def __init__(self, dev, ignored_folders=None, parent=None):
         QDialog.__init__(self, parent)
         self.l = l = QVBoxLayout()
         self.setLayout(l)
         self.la = la = QLabel('<p>'+ _('<b>Scanned folders:</b>') + ' ' +
-            _('You can select which top level folders calibre will '
+            _('You can select which folders calibre will '
               'scan when searching this device for books.'))
         la.setWordWrap(True)
         l.addWidget(la)
@@ -112,17 +113,18 @@ class TopLevel(QDialog):
         self.widgets = []
 
         for storage in dev.filesystem_cache.entries:
-            w = QListWidget(self)
+            self.dev = dev
-            w.storage = storage
+            w = Storage(storage, item_func=self.create_item)
+            del self.dev
             self.tabs.addTab(w, storage.name)
             self.widgets.append(w)
-            for child in sorted(storage.folders, key=attrgetter('name')):
+            w.itemChanged.connect(self.item_changed)
-                i = QListWidgetItem(child.name)
+
-                i.setFlags(Qt.ItemIsUserCheckable | Qt.ItemIsEnabled)
+        self.la2 = la = QLabel(_(
-                i.setCheckState(Qt.Unchecked if
+            'If you a select a previously unselected folder, any sub-folders'
-                    dev.is_folder_ignored(storage, child.name,
+            ' will not be visible until you restart calibre.'))
-                            ignored_folders=ignored_folders) else Qt.Checked)
+        l.addWidget(la)
-                w.addItem(i)
+        la.setWordWrap(True)
 
         self.bb = QDialogButtonBox(QDialogButtonBox.Ok |
                                    QDialogButtonBox.Cancel)
@@ -136,29 +138,68 @@ class TopLevel(QDialog):
         self.setWindowTitle(_('Choose folders to scan'))
         self.setWindowIcon(QIcon(I('devices/tablet.png')))
 
-        self.resize(500, 500)
+        self.resize(600, 500)
+
+    def item_changed(self, item, column):
+        w = item.treeWidget()
+        root = w.invisibleRootItem()
+        w.itemChanged.disconnect(self.item_changed)
+        try:
+            if item.checkState(0) == Qt.Checked:
+                # Ensure that the parents of this item are checked
+                p = item.parent()
+                while p is not None and p is not root:
+                    p.setCheckState(0, Qt.Checked)
+                    p = p.parent()
+            # Set the state of all descendants to the same state as this item
+            for child in self.iterchildren(item):
+                child.setCheckState(0, item.checkState(0))
+        finally:
+            w.itemChanged.connect(self.item_changed)
+
+    def iterchildren(self, node):
+        ' Iterate over all descendants of node '
+        for i in xrange(node.childCount()):
+            child = node.child(i)
+            yield child
+            for gc in self.iterchildren(child):
+                yield gc
+
+    def create_item(self, f, parent):
+        name = f.name
+        ans = QTreeWidgetItem(parent, [name])
+        ans.setData(0, Qt.UserRole, '/'.join(f.full_path[1:]))
+        ans.setFlags(Qt.ItemIsUserCheckable | Qt.ItemIsEnabled)
+        ans.setCheckState(0,
+            Qt.Unchecked if self.dev.is_folder_ignored(f.storage_id, f.full_path[1:]) else Qt.Checked)
+        ans.setData(0, Qt.DecorationRole, file_icon_provider().icon_from_ext('dir'))
+        return ans
 
     def select_all(self):
         w = self.tabs.currentWidget()
-        for i in xrange(w.count()):
+        for i in xrange(w.invisibleRootItem().childCount()):
-            x = w.item(i)
+            c = w.invisibleRootItem().child(i)
-            x.setCheckState(Qt.Checked)
+            c.setCheckState(0, Qt.Checked)
 
     def select_none(self):
         w = self.tabs.currentWidget()
-        for i in xrange(w.count()):
+        for i in xrange(w.invisibleRootItem().childCount()):
-            x = w.item(i)
+            c = w.invisibleRootItem().child(i)
-            x.setCheckState(Qt.Unchecked)
+            c.setCheckState(0, Qt.Unchecked)
 
     @property
     def ignored_folders(self):
         ans = {}
         for w in self.widgets:
-            ans[unicode(w.storage.object_id)] = folders = []
+            folders = set()
-            for i in xrange(w.count()):
+            for node in self.iterchildren(w.invisibleRootItem()):
-                x = w.item(i)
+                if node.checkState(0) == Qt.Checked:
-                if x.checkState() != Qt.Checked:
+                    continue
-                    folders.append(unicode(x.text()))
+                path = unicode(node.data(0, Qt.UserRole).toString())
+                parent = path.rpartition('/')[0]
+                if '/' not in path or icu_lower(parent) not in folders:
+                    folders.add(icu_lower(path))
+            ans[unicode(w.storage.storage_id)] = list(folders)
         return ans
 
 def setup_device():
@@ -184,17 +225,17 @@ def browse():
     dev.shutdown()
     return d.current_item
 
-def top_level():
+def ignored_folders():
     from calibre.gui2 import Application
     app = Application([])
     app
     dev = setup_device()
-    d = TopLevel(dev, None)
+    d = IgnoredFolders(dev)
     d.exec_()
     dev.shutdown()
     return d.ignored_folders
 
 if __name__ == '__main__':
-    # print (browse())
+    print (browse())
-    print ('Ignored:', top_level())
+    # print ('Ignored:', ignored_folders())