From 35eb01234ddfe2af803b18613d284111bd407346 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 21 Aug 2012 17:27:12 +0530 Subject: [PATCH] WPD: Implement reading files from device --- .../mtp/windows/content_enumeration.cpp | 105 +++++++++++++++++- src/calibre/devices/mtp/windows/device.cpp | 21 +++- src/calibre/devices/mtp/windows/driver.py | 10 +- src/calibre/devices/mtp/windows/global.h | 3 +- src/calibre/devices/mtp/windows/remote.py | 9 +- src/calibre/devices/mtp/windows/wpd.cpp | 5 +- 6 files changed, 143 insertions(+), 10 deletions(-) diff --git a/src/calibre/devices/mtp/windows/content_enumeration.cpp b/src/calibre/devices/mtp/windows/content_enumeration.cpp index 4fe20523cd..29d227d710 100644 --- a/src/calibre/devices/mtp/windows/content_enumeration.cpp +++ b/src/calibre/devices/mtp/windows/content_enumeration.cpp @@ -14,7 +14,7 @@ namespace wpd { static IPortableDeviceKeyCollection* create_filesystem_properties_collection() { // {{{ - IPortableDeviceKeyCollection *properties; + IPortableDeviceKeyCollection *properties = NULL; HRESULT hr; Py_BEGIN_ALLOW_THREADS; @@ -370,7 +370,7 @@ end: } // }}} -PyObject* wpd::get_filesystem(IPortableDevice *device, const wchar_t *storage_id, IPortableDevicePropertiesBulk *bulk_properties) { +PyObject* wpd::get_filesystem(IPortableDevice *device, const wchar_t *storage_id, IPortableDevicePropertiesBulk *bulk_properties) { // {{{ PyObject *folders = NULL; IPortableDevicePropVariantCollection *object_ids = NULL; IPortableDeviceContent *content = NULL; @@ -399,6 +399,105 @@ end: if (object_ids != NULL) object_ids->Release(); return folders; -} +} // }}} + +PyObject* wpd::get_file(IPortableDevice *device, const wchar_t *object_id, PyObject *dest, PyObject *callback) { // {{{ + IPortableDeviceContent *content = NULL; + IPortableDeviceResources *resources = NULL; + IPortableDeviceProperties *devprops = NULL; + IPortableDeviceValues *values = NULL; + IPortableDeviceKeyCollection *properties = NULL; + IStream *stream = NULL; + HRESULT hr; + DWORD bufsize = 4096; + char *buf = NULL; + ULONG bytes_read = 0, total_read = 0; + BOOL ok = FALSE; + PyObject *res = NULL; + ULONGLONG filesize = 0; + + Py_BEGIN_ALLOW_THREADS; + hr = device->Content(&content); + Py_END_ALLOW_THREADS; + if (FAILED(hr)) { hresult_set_exc("Failed to create content interface", hr); goto end; } + + Py_BEGIN_ALLOW_THREADS; + hr = content->Properties(&devprops); + Py_END_ALLOW_THREADS; + if (FAILED(hr)) { hresult_set_exc("Failed to get IPortableDeviceProperties interface", hr); goto end; } + + Py_BEGIN_ALLOW_THREADS; + hr = CoCreateInstance(CLSID_PortableDeviceKeyCollection, NULL, + CLSCTX_INPROC_SERVER, IID_PPV_ARGS(&properties)); + Py_END_ALLOW_THREADS; + if (FAILED(hr)) { hresult_set_exc("Failed to create filesystem properties collection", hr); goto end; } + hr = properties->Add(WPD_OBJECT_SIZE); + if (FAILED(hr)) { hresult_set_exc("Failed to add filesize property to properties collection", hr); goto end; } + + Py_BEGIN_ALLOW_THREADS; + hr = devprops->GetValues(object_id, properties, &values); + Py_END_ALLOW_THREADS; + if (FAILED(hr)) { hresult_set_exc("Failed to get filesize for object", hr); goto end; } + hr = values->GetUnsignedLargeIntegerValue(WPD_OBJECT_SIZE, &filesize); + if (FAILED(hr)) { hresult_set_exc("Failed to get filesize from values collection", hr); goto end; } + + Py_BEGIN_ALLOW_THREADS; + hr = content->Transfer(&resources); + Py_END_ALLOW_THREADS; + if (FAILED(hr)) { hresult_set_exc("Failed to create resources interface", hr); goto end; } + + Py_BEGIN_ALLOW_THREADS; + hr = resources->GetStream(object_id, WPD_RESOURCE_DEFAULT, STGM_READ, &bufsize, &stream); + Py_END_ALLOW_THREADS; + if (FAILED(hr)) { + if (HRESULT_FROM_WIN32(ERROR_BUSY) == hr) { + PyErr_SetString(WPDFileBusy, "Object is in use"); + } else hresult_set_exc("Failed to create stream interface to read from object", hr); + goto end; + } + + buf = (char *)calloc(bufsize+10, 1); + if (buf == NULL) { PyErr_NoMemory(); goto end; } + + while (TRUE) { + bytes_read = 0; + Py_BEGIN_ALLOW_THREADS; + hr = stream->Read(buf, bufsize, &bytes_read); + Py_END_ALLOW_THREADS; + total_read = total_read + bytes_read; + if (hr == STG_E_ACCESSDENIED) { + PyErr_SetString(PyExc_IOError, "Read access is denied to this object"); break; + } else if (hr == S_OK || hr == S_FALSE) { + if (bytes_read > 0) { + res = PyObject_CallMethod(dest, "write", "s#", buf, bytes_read); + if (res == NULL) break; + Py_DECREF(res); res = NULL; + if (callback != NULL) Py_XDECREF(PyObject_CallFunction(callback, "kK", total_read, filesize)); + } + } else { hresult_set_exc("Failed to read file from device", hr); break; } + + if (hr == S_FALSE || bytes_read < bufsize) { + ok = TRUE; + Py_XDECREF(PyObject_CallMethod(dest, "flush", NULL)); + break; + } + } + + if (ok && total_read != filesize) { + ok = FALSE; + PyErr_SetString(WPDError, "Failed to read all data from file"); + } + +end: + if (content != NULL) content->Release(); + if (devprops != NULL) devprops->Release(); + if (resources != NULL) resources->Release(); + if (stream != NULL) stream->Release(); + if (values != NULL) values->Release(); + if (properties != NULL) properties->Release(); + if (buf != NULL) free(buf); + if (!ok) return NULL; + Py_RETURN_NONE; +} // }}} } // namespace wpd diff --git a/src/calibre/devices/mtp/windows/device.cpp b/src/calibre/devices/mtp/windows/device.cpp index 0a03b9e735..d79db0a2d3 100644 --- a/src/calibre/devices/mtp/windows/device.cpp +++ b/src/calibre/devices/mtp/windows/device.cpp @@ -78,7 +78,7 @@ update_data(Device *self, PyObject *args, PyObject *kwargs) { // get_filesystem() {{{ static PyObject* py_get_filesystem(Device *self, PyObject *args, PyObject *kwargs) { - PyObject *storage_id, *ans = NULL; + PyObject *storage_id; wchar_t *storage; if (!PyArg_ParseTuple(args, "O", &storage_id)) return NULL; @@ -88,6 +88,21 @@ py_get_filesystem(Device *self, PyObject *args, PyObject *kwargs) { return wpd::get_filesystem(self->device, storage, self->bulk_properties); } // }}} +// get_file() {{{ +static PyObject* +py_get_file(Device *self, PyObject *args, PyObject *kwargs) { + PyObject *object_id, *stream, *callback = NULL; + wchar_t *object; + + if (!PyArg_ParseTuple(args, "OO|O", &object_id, &stream, &callback)) return NULL; + object = unicode_to_wchar(object_id); + if (object == NULL) return NULL; + + if (callback == NULL || !PyCallable_Check(callback)) callback = NULL; + + return wpd::get_file(self->device, object, stream, callback); +} // }}} + static PyMethodDef Device_methods[] = { {"update_data", (PyCFunction)update_data, METH_VARARGS, "update_data() -> Reread the basic device data from the device (total, space, free space, storage locations, etc.)" @@ -97,6 +112,10 @@ static PyMethodDef Device_methods[] = { "get_filesystem(storage_id) -> Get all files/folders on the storage identified by storage_id. Tries to use bulk operations when possible." }, + {"get_file", (PyCFunction)py_get_file, METH_VARARGS, + "get_file(object_id, stream, callback=None) -> Get the file identified by object_id from the device. The file is written to the stream object, which must be a file like object. If callback is not None, it must be a callable that accepts two arguments: (bytes_read, total_size). It will be called after each chunk is read from the device. Note that it can be called multiple times with the same values." + }, + {NULL} }; diff --git a/src/calibre/devices/mtp/windows/driver.py b/src/calibre/devices/mtp/windows/driver.py index fcfc415c90..4809ea5054 100644 --- a/src/calibre/devices/mtp/windows/driver.py +++ b/src/calibre/devices/mtp/windows/driver.py @@ -12,6 +12,7 @@ from threading import RLock from calibre import as_unicode, prints from calibre.constants import plugins, __appname__, numeric_version +from calibre.ptempfile import SpooledTemporaryFile from calibre.devices.errors import OpenFailed from calibre.devices.mtp.base import MTPDeviceBase, synchronous @@ -196,5 +197,12 @@ class MTP_DEVICE(MTPDeviceBase): ans[i] = s['free_space'] return tuple(ans) - + def get_file(self, object_id, callback=None): + stream = SpooledTemporaryFile(5*1024*1024) + try: + self.dev.get_file(object_id, stream, callback) + except self.wpd.WPDFileBusy: + time.sleep(2) + self.dev.get_file(object_id, stream, callback) + return stream diff --git a/src/calibre/devices/mtp/windows/global.h b/src/calibre/devices/mtp/windows/global.h index e58fafa1d4..47f0786249 100644 --- a/src/calibre/devices/mtp/windows/global.h +++ b/src/calibre/devices/mtp/windows/global.h @@ -20,7 +20,7 @@ namespace wpd { // Module exception types -extern PyObject *WPDError, *NoWPD; +extern PyObject *WPDError, *NoWPD, *WPDFileBusy; // The global device manager extern IPortableDeviceManager *portable_device_manager; @@ -57,6 +57,7 @@ extern IPortableDeviceValues* get_client_information(); extern IPortableDevice* open_device(const wchar_t *pnp_id, IPortableDeviceValues *client_information); extern PyObject* get_device_information(IPortableDevice *device, IPortableDevicePropertiesBulk **bulk_properties); extern PyObject* get_filesystem(IPortableDevice *device, const wchar_t *storage_id, IPortableDevicePropertiesBulk *bulk_properties); +extern PyObject* get_file(IPortableDevice *device, const wchar_t *object_id, PyObject *dest, PyObject *callback); } diff --git a/src/calibre/devices/mtp/windows/remote.py b/src/calibre/devices/mtp/windows/remote.py index a6502b991b..6f883f8baf 100644 --- a/src/calibre/devices/mtp/windows/remote.py +++ b/src/calibre/devices/mtp/windows/remote.py @@ -7,8 +7,8 @@ __license__ = 'GPL v3' __copyright__ = '2012, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import subprocess, sys, os, pprint, signal, time, glob -pprint +import subprocess, sys, os, pprint, signal, time, glob, io +pprint, io def build(mod='wpd'): master = subprocess.Popen('ssh -MN getafix'.split()) @@ -70,7 +70,10 @@ def main(): print ('Connected to:', dev.get_gui_name()) print ('Total space', dev.total_space()) print ('Free space', dev.free_space()) - pprint.pprint(dev.dev.get_filesystem(dev._main_id)) + # pprint.pprint(dev.dev.get_filesystem(dev._main_id)) + print ('Fetching file: oFF (198214 bytes)') + stream = dev.get_file('oFF') + print ("Fetched size: ", stream.tell()) finally: dev.shutdown() diff --git a/src/calibre/devices/mtp/windows/wpd.cpp b/src/calibre/devices/mtp/windows/wpd.cpp index 561eeb1bbc..51a55b97ac 100644 --- a/src/calibre/devices/mtp/windows/wpd.cpp +++ b/src/calibre/devices/mtp/windows/wpd.cpp @@ -10,7 +10,7 @@ using namespace wpd; // Module exception types -PyObject *wpd::WPDError = NULL, *wpd::NoWPD = NULL; +PyObject *wpd::WPDError = NULL, *wpd::NoWPD = NULL, *wpd::WPDFileBusy = NULL; // The global device manager IPortableDeviceManager *wpd::portable_device_manager = NULL; @@ -199,6 +199,9 @@ initwpd(void) { NoWPD = PyErr_NewException("wpd.NoWPD", NULL, NULL); if (NoWPD == NULL) return; + WPDFileBusy = PyErr_NewException("wpd.WPDFileBusy", NULL, NULL); + if (WPDFileBusy == NULL) return; + Py_INCREF(&DeviceType); PyModule_AddObject(m, "Device", (PyObject *)&DeviceType);