Merge from trunk

src/calibre/devices/interface.py

@@ -297,7 +297,7 @@ class DevicePlugin(Plugin):
 
         :return: (device name, device version, software version on device, mime type)
                  The tuple can optionally have a fifth element, which is a
-                 drive information diction. See usbms.driver for an example.
+                 drive information dictionary. See usbms.driver for an example.
 
         """
         raise NotImplementedError()
@@ -607,7 +607,7 @@ class BookList(list):
         pass
 
     def supports_collections(self):
-        ''' Return True if the the device supports collections for this book list. '''
+        ''' Return True if the device supports collections for this book list. '''
         raise NotImplementedError()
 
     def add_book(self, book, replace_metadata):

src/calibre/devices/mtp/base.py

@@ -8,9 +8,8 @@ __copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
 from calibre.devices.interface import DevicePlugin
-from calibre.devices.usbms.deviceconfig import DeviceConfig
 
-class MTPDeviceBase(DeviceConfig, DevicePlugin):
+class MTPDeviceBase(DevicePlugin):
     name = 'SmartDevice App Interface'
     gui_name = _('MTP Device')
     icon = I('devices/galaxy_s3.png')
@@ -28,7 +27,14 @@ class MTPDeviceBase(DeviceConfig, DevicePlugin):
 
     BACKLOADING_ERROR_MESSAGE = None
 
+    def __init__(self, *args, **kwargs):
+        DevicePlugin.__init__(self, *args, **kwargs)
+        self.progress_reporter = None
+
     def reset(self, key='-1', log_packets=False, report_progress=None,
             detected_device=None):
         pass
 
+    def set_progress_reporter(self, report_progress):
+        self.progress_reporter = report_progress
+

src/calibre/devices/mtp/unix/driver.py

@@ -7,7 +7,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import time
+import time, operator
 from threading import RLock
 from functools import wraps
 
@@ -33,6 +33,19 @@ class MTP_DEVICE(MTPDeviceBase):
         self.lock = RLock()
         self.blacklisted_devices = set()
 
+    def report_progress(self, sent, total):
+        try:
+            p = int(sent/total * 100)
+        except ZeroDivisionError:
+            p = 100
+        if self.progress_reporter is not None:
+            self.progress_reporter(p)
+
+    @synchronous
+    def get_gui_name(self):
+        if self.dev is None or not self.dev.friendly_name: return self.name
+        return self.dev.friendly_name
+
     @synchronous
     def is_usb_connected(self, devices_on_system, debug=False,
             only_presence=False):
@@ -62,6 +75,10 @@ class MTP_DEVICE(MTPDeviceBase):
     def post_yank_cleanup(self):
         self.dev = None
 
+    @synchronous
+    def shutdown(self):
+        self.dev = None
+
     @synchronous
     def open(self, connected_device, library_uuid):
         def blacklist_device():
@@ -69,18 +86,82 @@ class MTP_DEVICE(MTPDeviceBase):
             self.blacklisted_devices.add((d.busnum, d.devnum, d.vendor_id,
                     d.product_id, d.bcd, d.serial))
         try:
-            self.detect.create_device(connected_device)
+            self.dev = self.detect.create_device(connected_device)
         except ValueError:
             # Give the device some time to settle
             time.sleep(2)
             try:
-                self.detect.create_device(connected_device)
+                self.dev = self.detect.create_device(connected_device)
             except ValueError:
                 # Black list this device so that it is ignored for the
                 # remainder of this session.
                 blacklist_device()
-                raise OpenFailed('%s is not a MTP device'%connected_device)
+                raise OpenFailed('%s is not a MTP device'%(connected_device,))
         except TypeError:
             blacklist_device()
             raise OpenFailed('')
 
+        storage = sorted(self.dev.storage_info, key=operator.itemgetter('id'))
+        if not storage:
+            blacklist_device()
+            raise OpenFailed('No storage found for device %s'%(connected_device,))
+        self._main_id = storage[0]['id']
+        self._carda_id = self._cardb_id = None
+        if len(storage) > 1:
+            self._carda_id = storage[1]['id']
+        if len(storage) > 2:
+            self._cardb_id = storage[2]['id']
+
+    @synchronous
+    def get_device_information(self, end_session=True):
+        d = self.dev
+        return (d.friendly_name, d.device_version, d.device_version, '')
+
+    @synchronous
+    def card_prefix(self, end_session=True):
+        ans = [None, None]
+        if self._carda_id is not None:
+            ans[0] = 'mtp:%d:'%self._carda_id
+        if self._cardb_id is not None:
+            ans[1] = 'mtp:%d:'%self._cardb_id
+        return tuple(ans)
+
+    @synchronous
+    def total_space(self, end_session=True):
+        ans = [0, 0, 0]
+        for s in self.dev.storage_info:
+            i = {self._main_id:0, self._carda_id:1,
+                    self._cardb_id:2}.get(s['id'], None)
+            if i is not None:
+                ans[i] = s['capacity']
+        return tuple(ans)
+
+    @synchronous
+    def free_space(self, end_session=True):
+        self.dev.update_storage_info()
+        ans = [0, 0, 0]
+        for s in self.dev.storage_info:
+            i = {self._main_id:0, self._carda_id:1,
+                    self._cardb_id:2}.get(s['id'], None)
+            if i is not None:
+                ans[i] = s['freespace_bytes']
+        return tuple(ans)
+
+
+if __name__ == '__main__':
+    from pprint import pprint
+    dev = MTP_DEVICE(None)
+    from calibre.devices.scanner import linux_scanner
+    devs = linux_scanner()
+    mtp_devs = dev.detect(devs)
+    dev.open(list(mtp_devs)[0], 'xxx')
+    d = dev.dev
+    print ("Opened device:", dev.get_gui_name())
+    print ("Storage info:")
+    pprint(d.storage_info)
+    print("Free space:", dev.free_space())
+    files, errs = d.get_filelist(dev)
+    pprint((len(files), errs))
+    folders, errs = d.get_folderlist()
+    pprint((len(folders), errs))
+

src/calibre/devices/mtp/unix/libmtp.c

@@ -6,6 +6,66 @@
 
 #include "devices.h"
 
+// Macros and utilities
+#define ENSURE_DEV(rval) \
+    if (self->device == NULL) { \
+        PyErr_SetString(PyExc_ValueError, "This device has not been initialized."); \
+        return rval; \
+    }
+
+#define ENSURE_STORAGE(rval) \
+    if (self->device->storage == NULL) { \
+        PyErr_SetString(PyExc_RuntimeError, "The device has no storage information."); \
+        return rval; \
+    }
+
+// Storage types
+#define ST_Undefined            0x0000
+#define ST_FixedROM             0x0001
+#define ST_RemovableROM         0x0002
+#define ST_FixedRAM             0x0003
+#define ST_RemovableRAM         0x0004
+
+// Storage Access capability
+#define AC_ReadWrite            0x0000
+#define AC_ReadOnly             0x0001
+#define AC_ReadOnly_with_Object_Deletion    0x0002
+
+typedef struct {
+    PyObject *obj;
+    PyThreadState *state;
+} ProgressCallback;
+
+static int report_progress(uint64_t const sent, uint64_t const total, void const *const data) {
+    PyObject *res;
+    ProgressCallback *cb;
+
+    cb = (ProgressCallback *)data;
+    if (cb->obj != NULL) {
+        PyEval_RestoreThread(cb->state);
+        res = PyObject_CallMethod(cb->obj, "report_progress", "KK", sent, total);
+        Py_XDECREF(res);
+        cb->state = PyEval_SaveThread();
+    }
+    return 0;
+}
+
+static void dump_errorstack(LIBMTP_mtpdevice_t *dev, PyObject *list) {
+    LIBMTP_error_t *stack;
+    PyObject *err;
+
+    for(stack = LIBMTP_Get_Errorstack(dev); stack != NULL; stack=stack->next) {
+        err = Py_BuildValue("Is", stack->errornumber, stack->error_text);
+        if (err == NULL) break;
+        PyList_Append(list, err);
+        Py_DECREF(err);
+    }
+
+    LIBMTP_Clear_Errorstack(dev);
+}
+
+// }}}
+
 // Device object definition {{{
 typedef struct {
     PyObject_HEAD
@@ -20,6 +80,7 @@ typedef struct {
 
 } libmtp_Device;
 
+// Device.__init__() {{{
 static void
 libmtp_Device_dealloc(libmtp_Device* self)
 {
@@ -69,7 +130,9 @@ libmtp_Device_init(libmtp_Device *self, PyObject *args, PyObject *kwds)
         }
     }
 
-    dev = LIBMTP_Open_Raw_Device_Uncached(&rawdev);
+    // Note that contrary to what the libmtp docs imply, we cannot use
+    // LIBMTP_Open_Raw_Device_Uncached as using it causes file listing to fail
+    dev = LIBMTP_Open_Raw_Device(&rawdev);
     Py_END_ALLOW_THREADS;
 
     if (dev == NULL) { 
@@ -119,44 +182,217 @@ libmtp_Device_init(libmtp_Device *self, PyObject *args, PyObject *kwds)
 
     return 0;
 }
+// }}}
 
-// Collator.friendly_name {{{
+// Device.friendly_name {{{
 static PyObject *
 libmtp_Device_friendly_name(libmtp_Device *self, void *closure) {
-    return Py_BuildValue("O", self->friendly_name);
+    Py_INCREF(self->friendly_name); return self->friendly_name;
 } // }}}
 
-// Collator.manufacturer_name {{{
+// Device.manufacturer_name {{{
 static PyObject *
 libmtp_Device_manufacturer_name(libmtp_Device *self, void *closure) {
-    return Py_BuildValue("O", self->manufacturer_name);
+    Py_INCREF(self->manufacturer_name); return self->manufacturer_name;
 } // }}}
 
-// Collator.model_name {{{
+// Device.model_name {{{
 static PyObject *
 libmtp_Device_model_name(libmtp_Device *self, void *closure) {
-    return Py_BuildValue("O", self->model_name);
+    Py_INCREF(self->model_name); return self->model_name;
 } // }}}
 
-// Collator.serial_number {{{
+// Device.serial_number {{{
 static PyObject *
 libmtp_Device_serial_number(libmtp_Device *self, void *closure) {
-    return Py_BuildValue("O", self->serial_number);
+    Py_INCREF(self->serial_number); return self->serial_number;
 } // }}}
 
-// Collator.device_version {{{
+// Device.device_version {{{
 static PyObject *
 libmtp_Device_device_version(libmtp_Device *self, void *closure) {
-    return Py_BuildValue("O", self->device_version);
+    Py_INCREF(self->device_version); return self->device_version;
 } // }}}
 
-// Collator.ids {{{
+// Device.ids {{{
 static PyObject *
 libmtp_Device_ids(libmtp_Device *self, void *closure) {
-    return Py_BuildValue("O", self->ids);
+    Py_INCREF(self->ids); return self->ids;
+} // }}}
+
+// Device.update_storage_info() {{{
+static PyObject*
+libmtp_Device_update_storage_info(libmtp_Device *self, PyObject *args, PyObject *kwargs) {
+    ENSURE_DEV(NULL);
+    if (LIBMTP_Get_Storage(self->device, LIBMTP_STORAGE_SORTBY_NOTSORTED) < 0) {
+        PyErr_SetString(PyExc_RuntimeError, "Failed to get storage infor for device.");
+        return NULL;
+    }
+    Py_RETURN_NONE;
+}
+// }}}
+
+// Device.storage_info {{{
+static PyObject *
+libmtp_Device_storage_info(libmtp_Device *self, void *closure) {
+    PyObject *ans, *loc;
+    LIBMTP_devicestorage_t *storage;
+    ENSURE_DEV(NULL); ENSURE_STORAGE(NULL);
+
+    ans = PyList_New(0);
+    if (ans == NULL) { PyErr_NoMemory(); return NULL; }
+
+    for (storage = self->device->storage; storage != NULL; storage = storage->next) {
+        // Ignore read only storage
+        if (storage->StorageType == ST_FixedROM || storage->StorageType == ST_RemovableROM) continue;
+        // Storage IDs with the lower 16 bits 0x0000 are not supposed to be
+        // writeable.
+        if ((storage->id & 0x0000FFFFU) == 0x00000000U) continue;
+        // Also check the access capability to avoid e.g. deletable only storages
+        if (storage->AccessCapability == AC_ReadOnly || storage->AccessCapability == AC_ReadOnly_with_Object_Deletion) continue;
+
+        loc = Py_BuildValue("{s:k,s:O,s:K,s:K,s:K,s:s,s:s}", 
+                "id", storage->id, 
+                "removable", ((storage->StorageType == ST_RemovableRAM) ? Py_True : Py_False),
+                "capacity", storage->MaxCapacity,
+                "freespace_bytes", storage->FreeSpaceInBytes,
+                "freespace_objects", storage->FreeSpaceInObjects,
+                "storage_desc", storage->StorageDescription,
+                "volume_id", storage->VolumeIdentifier
+        );
+
+        if (loc == NULL) return NULL; 
+        if (PyList_Append(ans, loc) != 0) return NULL;
+        Py_DECREF(loc);
+
+    }
+
+    return ans;
+} // }}}
+
+// Device.get_filelist {{{
+static PyObject *
+libmtp_Device_get_filelist(libmtp_Device *self, PyObject *args, PyObject *kwargs) {
+    PyObject *ans, *fo, *callback = NULL, *errs;
+    ProgressCallback cb;
+    LIBMTP_file_t *f, *tf;
+
+    ENSURE_DEV(NULL); ENSURE_STORAGE(NULL);
+
+
+    if (!PyArg_ParseTuple(args, "|O", &callback)) return NULL;
+    cb.obj = callback;
+
+    ans = PyList_New(0);
+    errs = PyList_New(0);
+    if (ans == NULL || errs == NULL) { PyErr_NoMemory(); return NULL; }
+
+    cb.state = PyEval_SaveThread();
+    tf = LIBMTP_Get_Filelisting_With_Callback(self->device, report_progress, &cb);
+    PyEval_RestoreThread(cb.state);
+
+    if (tf == NULL) { 
+        dump_errorstack(self->device, errs);
+        return Py_BuildValue("NN", ans, errs);
+    }
+
+    for (f=tf; f != NULL; f=f->next) {
+        fo = Py_BuildValue("{s:k,s:k,s:k,s:s,s:K,s:k}",
+                "id", f->item_id,
+                "parent_id", f->parent_id,
+                "storage_id", f->storage_id,
+                "filename", f->filename,
+                "size", f->filesize,
+                "modtime", f->modificationdate
+        );
+        if (fo == NULL || PyList_Append(ans, fo) != 0) break;
+        Py_DECREF(fo);
+    }
+
+    // Release memory
+    f = tf;
+    while (f != NULL) {
+        tf = f; f = f->next; LIBMTP_destroy_file_t(tf);
+    }
+
+    if (callback != NULL) {
+        // Bug in libmtp where it does not call callback with 100%
+        fo = PyObject_CallMethod(callback, "report_progress", "KK", PyList_Size(ans), PyList_Size(ans));
+        Py_XDECREF(fo);
+    }
+
+    return Py_BuildValue("NN", ans, errs);
+} // }}}
+
+// Device.get_folderlist {{{
+
+int folderiter(LIBMTP_folder_t *f, PyObject *parent) {
+    PyObject *folder, *children;
+
+    children = PyList_New(0);
+    if (children == NULL) { PyErr_NoMemory(); return 1;}
+
+    folder = Py_BuildValue("{s:k,s:k,s:k,s:s,s:N}",
+            "id", f->folder_id,
+            "parent_d", f->parent_id,
+            "storage_id", f->storage_id,
+            "name", f->name,
+            "children", children);
+    if (folder == NULL) return 1;
+    PyList_Append(parent, folder);
+    Py_DECREF(folder);
+
+    if (f->sibling != NULL) {
+        if (folderiter(f->sibling, parent)) return 1;
+    }
+
+    if (f->child != NULL) {
+        if (folderiter(f->child, children)) return 1;
+    }
+
+    return 0;
+}
+
+static PyObject *
+libmtp_Device_get_folderlist(libmtp_Device *self, PyObject *args, PyObject *kwargs) {
+    PyObject *ans, *errs;
+    LIBMTP_folder_t *f;
+
+    ENSURE_DEV(NULL); ENSURE_STORAGE(NULL);
+
+    ans = PyList_New(0);
+    errs = PyList_New(0);
+    if (errs == NULL || ans == NULL) { PyErr_NoMemory(); return NULL; }
+
+    Py_BEGIN_ALLOW_THREADS;
+    f = LIBMTP_Get_Folder_List(self->device);
+    Py_END_ALLOW_THREADS;
+
+    if (f == NULL) {
+        dump_errorstack(self->device, errs);
+        return Py_BuildValue("NN", ans, errs);
+    }
+
+    if (folderiter(f, ans)) return NULL;
+    LIBMTP_destroy_folder_t(f);
+
+    return Py_BuildValue("NN", ans, errs);
+
 } // }}}
 
 static PyMethodDef libmtp_Device_methods[] = {
+    {"update_storage_info", (PyCFunction)libmtp_Device_update_storage_info, METH_VARARGS,
+     "update_storage_info() -> Reread the storage info from the device (total, space, free space, storage locations, etc.)"
+    },
+
+    {"get_filelist", (PyCFunction)libmtp_Device_get_filelist, METH_VARARGS,
+     "get_filelist(callback=None) -> Get the list of files on the device. callback must be an object that has a method named 'report_progress(current, total)'. Returns files, errors."
+    },
+
+    {"get_folderlist", (PyCFunction)libmtp_Device_get_folderlist, METH_VARARGS,
+     "get_folderlist() -> Get the list of folders on the device. Returns files, erros."
+    },
+
     {NULL}  /* Sentinel */
 };
 
@@ -191,6 +427,11 @@ static PyGetSetDef libmtp_Device_getsetters[] = {
      (char *)"The ids of the device (busnum, devnum, vendor_id, product_id, usb_serialnum)",
      NULL},
 
+    {(char *)"storage_info",
+     (getter)libmtp_Device_storage_info, NULL,
+     (char *)"Information about the storage locations on the device. Returns a list of dictionaries where each dictionary corresponds to the LIBMTP_devicestorage_struct.",
+     NULL},
+
     {NULL}  /* Sentinel */
 };