WPD: Implement reading files from device

src/calibre/devices/mtp/windows/content_enumeration.cpp

@@ -14,7 +14,7 @@
 namespace wpd {
 
 static IPortableDeviceKeyCollection* create_filesystem_properties_collection() { // {{{
-    IPortableDeviceKeyCollection *properties;
+    IPortableDeviceKeyCollection *properties = NULL;
     HRESULT hr;
 
     Py_BEGIN_ALLOW_THREADS;
@@ -370,7 +370,7 @@ end:
 } 
 // }}}
 
-PyObject* wpd::get_filesystem(IPortableDevice *device, const wchar_t *storage_id, IPortableDevicePropertiesBulk *bulk_properties) {
+PyObject* wpd::get_filesystem(IPortableDevice *device, const wchar_t *storage_id, IPortableDevicePropertiesBulk *bulk_properties) { // {{{
     PyObject *folders = NULL;
     IPortableDevicePropVariantCollection *object_ids = NULL;
     IPortableDeviceContent *content = NULL;
@@ -399,6 +399,105 @@ end:
     if (object_ids != NULL) object_ids->Release();
 
     return folders;
-}
+} // }}}
+
+PyObject* wpd::get_file(IPortableDevice *device, const wchar_t *object_id, PyObject *dest, PyObject *callback) { // {{{
+    IPortableDeviceContent *content = NULL;
+    IPortableDeviceResources *resources = NULL;
+    IPortableDeviceProperties *devprops = NULL;
+    IPortableDeviceValues *values = NULL;
+    IPortableDeviceKeyCollection *properties = NULL;
+    IStream *stream = NULL;
+    HRESULT hr;
+    DWORD bufsize = 4096;
+    char *buf = NULL;
+    ULONG bytes_read = 0, total_read = 0;
+    BOOL ok = FALSE;
+    PyObject *res = NULL;
+    ULONGLONG filesize = 0;
+
+    Py_BEGIN_ALLOW_THREADS;
+    hr = device->Content(&content);
+    Py_END_ALLOW_THREADS;
+    if (FAILED(hr)) { hresult_set_exc("Failed to create content interface", hr); goto end; }
+
+    Py_BEGIN_ALLOW_THREADS;
+    hr = content->Properties(&devprops);
+    Py_END_ALLOW_THREADS;
+    if (FAILED(hr)) { hresult_set_exc("Failed to get IPortableDeviceProperties interface", hr); goto end; }
+
+    Py_BEGIN_ALLOW_THREADS;
+    hr = CoCreateInstance(CLSID_PortableDeviceKeyCollection, NULL,
+            CLSCTX_INPROC_SERVER, IID_PPV_ARGS(&properties));
+    Py_END_ALLOW_THREADS;
+    if (FAILED(hr)) { hresult_set_exc("Failed to create filesystem properties collection", hr); goto end; }
+    hr = properties->Add(WPD_OBJECT_SIZE);
+    if (FAILED(hr)) { hresult_set_exc("Failed to add filesize property to properties collection", hr); goto end; }
+
+    Py_BEGIN_ALLOW_THREADS;
+    hr = devprops->GetValues(object_id, properties, &values);
+    Py_END_ALLOW_THREADS;
+    if (FAILED(hr)) { hresult_set_exc("Failed to get filesize for object", hr); goto end; }
+    hr = values->GetUnsignedLargeIntegerValue(WPD_OBJECT_SIZE, &filesize);
+    if (FAILED(hr)) { hresult_set_exc("Failed to get filesize from values collection", hr); goto end; }
+
+    Py_BEGIN_ALLOW_THREADS;
+    hr = content->Transfer(&resources);
+    Py_END_ALLOW_THREADS;
+    if (FAILED(hr)) { hresult_set_exc("Failed to create resources interface", hr); goto end; }
+
+    Py_BEGIN_ALLOW_THREADS;
+    hr = resources->GetStream(object_id, WPD_RESOURCE_DEFAULT, STGM_READ, &bufsize, &stream);
+    Py_END_ALLOW_THREADS;
+    if (FAILED(hr)) { 
+        if (HRESULT_FROM_WIN32(ERROR_BUSY) == hr) {
+            PyErr_SetString(WPDFileBusy, "Object is in use");
+        } else hresult_set_exc("Failed to create stream interface to read from object", hr); 
+        goto end; 
+    }
+
+    buf = (char *)calloc(bufsize+10, 1);
+    if (buf == NULL) { PyErr_NoMemory(); goto end; }
+
+    while (TRUE) {
+        bytes_read = 0;
+        Py_BEGIN_ALLOW_THREADS;
+        hr = stream->Read(buf, bufsize, &bytes_read);
+        Py_END_ALLOW_THREADS;
+        total_read = total_read + bytes_read;
+        if (hr == STG_E_ACCESSDENIED) { 
+            PyErr_SetString(PyExc_IOError, "Read access is denied to this object"); break; 
+        } else if (hr == S_OK || hr == S_FALSE) {
+            if (bytes_read > 0) {
+                res = PyObject_CallMethod(dest, "write", "s#", buf, bytes_read);
+                if (res == NULL) break;
+                Py_DECREF(res); res = NULL;
+                if (callback != NULL) Py_XDECREF(PyObject_CallFunction(callback, "kK", total_read, filesize));
+            }
+        } else { hresult_set_exc("Failed to read file from device", hr); break; }
+
+        if (hr == S_FALSE || bytes_read < bufsize) { 
+            ok = TRUE; 
+            Py_XDECREF(PyObject_CallMethod(dest, "flush", NULL));
+            break;
+        }
+    }
+
+    if (ok && total_read != filesize) {
+        ok = FALSE;
+        PyErr_SetString(WPDError, "Failed to read all data from file");
+    }
+
+end:
+    if (content != NULL) content->Release();
+    if (devprops != NULL) devprops->Release();
+    if (resources != NULL) resources->Release();
+    if (stream != NULL) stream->Release();
+    if (values != NULL) values->Release();
+    if (properties != NULL) properties->Release();
+    if (buf != NULL) free(buf);
+    if (!ok) return NULL;
+    Py_RETURN_NONE;
+} // }}}
 
 } // namespace wpd

src/calibre/devices/mtp/windows/device.cpp

@@ -78,7 +78,7 @@ update_data(Device *self, PyObject *args, PyObject *kwargs) {
 // get_filesystem() {{{
 static PyObject*
 py_get_filesystem(Device *self, PyObject *args, PyObject *kwargs) {
-    PyObject *storage_id, *ans = NULL;
+    PyObject *storage_id;
     wchar_t *storage;
 
     if (!PyArg_ParseTuple(args, "O", &storage_id)) return NULL;
@@ -88,6 +88,21 @@ py_get_filesystem(Device *self, PyObject *args, PyObject *kwargs) {
     return wpd::get_filesystem(self->device, storage, self->bulk_properties);
 } // }}}
 
+// get_file() {{{
+static PyObject*
+py_get_file(Device *self, PyObject *args, PyObject *kwargs) {
+    PyObject *object_id, *stream, *callback = NULL;
+    wchar_t *object;
+
+    if (!PyArg_ParseTuple(args, "OO|O", &object_id, &stream, &callback)) return NULL;
+    object = unicode_to_wchar(object_id);
+    if (object == NULL) return NULL;
+
+    if (callback == NULL || !PyCallable_Check(callback)) callback = NULL;
+
+    return wpd::get_file(self->device, object, stream, callback);
+} // }}}
+
 static PyMethodDef Device_methods[] = {
     {"update_data", (PyCFunction)update_data, METH_VARARGS,
      "update_data() -> Reread the basic device data from the device (total, space, free space, storage locations, etc.)"
@@ -97,6 +112,10 @@ static PyMethodDef Device_methods[] = {
      "get_filesystem(storage_id) -> Get all files/folders on the storage identified by storage_id. Tries to use bulk operations when possible."
     },
 
+    {"get_file", (PyCFunction)py_get_file, METH_VARARGS,
+     "get_file(object_id, stream, callback=None) -> Get the file identified by object_id from the device. The file is written to the stream object, which must be a file like object. If callback is not None, it must be a callable that accepts two arguments: (bytes_read, total_size). It will be called after each chunk is read from the device. Note that it can be called multiple times with the same values."
+    },
+
     {NULL}
 };
 

src/calibre/devices/mtp/windows/driver.py

@@ -12,6 +12,7 @@ from threading import RLock
 
 from calibre import as_unicode, prints
 from calibre.constants import plugins, __appname__, numeric_version
+from calibre.ptempfile import SpooledTemporaryFile
 from calibre.devices.errors import OpenFailed
 from calibre.devices.mtp.base import MTPDeviceBase, synchronous
 
@@ -196,5 +197,12 @@ class MTP_DEVICE(MTPDeviceBase):
                 ans[i] = s['free_space']
         return tuple(ans)
 
-
+    def get_file(self, object_id, callback=None):
+        stream = SpooledTemporaryFile(5*1024*1024)
+        try:
+            self.dev.get_file(object_id, stream, callback)
+        except self.wpd.WPDFileBusy:
+            time.sleep(2)
+            self.dev.get_file(object_id, stream, callback)
+        return stream
 

src/calibre/devices/mtp/windows/global.h

@@ -20,7 +20,7 @@
 namespace wpd {
 
 // Module exception types
-extern PyObject *WPDError, *NoWPD;
+extern PyObject *WPDError, *NoWPD, *WPDFileBusy;
 
 // The global device manager
 extern IPortableDeviceManager *portable_device_manager;
@@ -57,6 +57,7 @@ extern IPortableDeviceValues* get_client_information();
 extern IPortableDevice* open_device(const wchar_t *pnp_id, IPortableDeviceValues *client_information);
 extern PyObject* get_device_information(IPortableDevice *device, IPortableDevicePropertiesBulk **bulk_properties);
 extern PyObject* get_filesystem(IPortableDevice *device, const wchar_t *storage_id, IPortableDevicePropertiesBulk *bulk_properties);
+extern PyObject* get_file(IPortableDevice *device, const wchar_t *object_id, PyObject *dest, PyObject *callback);
 
 }
 

src/calibre/devices/mtp/windows/remote.py

@@ -7,8 +7,8 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import subprocess, sys, os, pprint, signal, time, glob
+import subprocess, sys, os, pprint, signal, time, glob, io
-pprint
+pprint, io
 
 def build(mod='wpd'):
     master = subprocess.Popen('ssh -MN getafix'.split())
@@ -70,7 +70,10 @@ def main():
         print ('Connected to:', dev.get_gui_name())
         print ('Total space', dev.total_space())
         print ('Free space', dev.free_space())
-        pprint.pprint(dev.dev.get_filesystem(dev._main_id))
+        # pprint.pprint(dev.dev.get_filesystem(dev._main_id))
+        print ('Fetching file: oFF (198214 bytes)')
+        stream = dev.get_file('oFF')
+        print ("Fetched size: ", stream.tell())
     finally:
         dev.shutdown()
 

src/calibre/devices/mtp/windows/wpd.cpp

@@ -10,7 +10,7 @@
 using namespace wpd;
 
 // Module exception types
-PyObject *wpd::WPDError = NULL, *wpd::NoWPD = NULL;
+PyObject *wpd::WPDError = NULL, *wpd::NoWPD = NULL, *wpd::WPDFileBusy = NULL;
 
 // The global device manager
 IPortableDeviceManager *wpd::portable_device_manager = NULL;
@@ -199,6 +199,9 @@ initwpd(void) {
     NoWPD = PyErr_NewException("wpd.NoWPD", NULL, NULL);
     if (NoWPD == NULL) return;
 
+    WPDFileBusy = PyErr_NewException("wpd.WPDFileBusy", NULL, NULL);
+    if (WPDFileBusy == NULL) return;
+
     Py_INCREF(&DeviceType);
     PyModule_AddObject(m, "Device", (PyObject *)&DeviceType);