diff --git a/setup/extensions.py b/setup/extensions.py index 2df62636ae..f43dc1eef0 100644 --- a/setup/extensions.py +++ b/setup/extensions.py @@ -172,13 +172,14 @@ if iswindows: [ 'calibre/devices/mtp/windows/utils.cpp', 'calibre/devices/mtp/windows/device_enumeration.cpp', + 'calibre/devices/mtp/windows/content_enumeration.cpp', 'calibre/devices/mtp/windows/device.cpp', 'calibre/devices/mtp/windows/wpd.cpp', ], headers=[ 'calibre/devices/mtp/windows/global.h', ], - libraries=['ole32', 'portabledeviceguids'], + libraries=['ole32', 'portabledeviceguids', 'user32'], # needs_ddk=True, cflags=['/X'] ), diff --git a/src/calibre/devices/mtp/windows/content_enumeration.cpp b/src/calibre/devices/mtp/windows/content_enumeration.cpp new file mode 100644 index 0000000000..859b6c9efa --- /dev/null +++ b/src/calibre/devices/mtp/windows/content_enumeration.cpp @@ -0,0 +1,329 @@ +/* + * content_enumeration.cpp + * Copyright (C) 2012 Kovid Goyal + * + * Distributed under terms of the GPL3 license. + */ + +#include "global.h" + +#include + +#define ADDPROP(x) hr = properties->Add(x); if (FAILED(hr)) { hresult_set_exc("Failed to add property to filesystem properties collection", hr); properties->Release(); return NULL; } + +namespace wpd { + +static IPortableDeviceKeyCollection* create_filesystem_properties_collection() { // {{{ + IPortableDeviceKeyCollection *properties; + HRESULT hr; + + Py_BEGIN_ALLOW_THREADS; + hr = CoCreateInstance(CLSID_PortableDeviceKeyCollection, NULL, + CLSCTX_INPROC_SERVER, IID_PPV_ARGS(&properties)); + Py_END_ALLOW_THREADS; + + if (FAILED(hr)) { hresult_set_exc("Failed to create filesystem properties collection", hr); return NULL; } + + ADDPROP(WPD_OBJECT_CONTENT_TYPE); + ADDPROP(WPD_OBJECT_PARENT_ID); + ADDPROP(WPD_OBJECT_PERSISTENT_UNIQUE_ID); + ADDPROP(WPD_OBJECT_NAME); + ADDPROP(WPD_OBJECT_SYNC_ID); + ADDPROP(WPD_OBJECT_ISSYSTEM); + ADDPROP(WPD_OBJECT_ISHIDDEN); + ADDPROP(WPD_OBJECT_CAN_DELETE); + ADDPROP(WPD_OBJECT_SIZE); + + return properties; + +} // }}} + +// Convert properties from COM to python {{{ +static void set_string_property(PyObject *dict, REFPROPERTYKEY key, const char *pykey, IPortableDeviceValues *properties) { + HRESULT hr; + wchar_t *property = NULL; + PyObject *val; + + hr = properties->GetStringValue(key, &property); + if (SUCCEEDED(hr)) { + val = wchar_to_unicode(property); + if (val != NULL) { + PyDict_SetItemString(dict, pykey, val); + Py_DECREF(val); + } + CoTaskMemFree(property); + } +} + +static void set_bool_property(PyObject *dict, REFPROPERTYKEY key, const char *pykey, IPortableDeviceValues *properties) { + BOOL ok = 0; + HRESULT hr; + + hr = properties->GetBoolValue(key, &ok); + if (SUCCEEDED(hr)) + PyDict_SetItemString(dict, pykey, (ok)?Py_True:Py_False); +} + +static void set_size_property(PyObject *dict, REFPROPERTYKEY key, const char *pykey, IPortableDeviceValues *properties) { + ULONGLONG val = 0; + HRESULT hr; + PyObject *pval; + + hr = properties->GetUnsignedLargeIntegerValue(key, &val); + + if (SUCCEEDED(hr)) { + pval = PyInt_FromSsize_t((Py_ssize_t)val); + if (pval != NULL) { + PyDict_SetItemString(dict, pykey, pval); + Py_DECREF(pval); + } + } +} + +static void set_content_type_property(PyObject *dict, IPortableDeviceValues *properties) { + GUID guid = GUID_NULL; + BOOL is_folder = 0; + + if (SUCCEEDED(properties->GetGuidValue(WPD_OBJECT_CONTENT_TYPE, &guid)) && IsEqualGUID(guid, WPD_CONTENT_TYPE_FOLDER)) is_folder = 1; + PyDict_SetItemString(dict, "is_folder", (is_folder) ? Py_True : Py_False); +} +// }}} + +class GetBulkCallback : public IPortableDevicePropertiesBulkCallback { + +public: + PyObject *items; + HANDLE complete; + ULONG self_ref; + PyThreadState *thread_state; + + GetBulkCallback(PyObject *items_dict, HANDLE ev) : items(items_dict), complete(ev), self_ref(1), thread_state(NULL) {} + ~GetBulkCallback() {} + + HRESULT __stdcall OnStart(REFGUID Context) { return S_OK; } + + HRESULT __stdcall OnEnd(REFGUID Context, HRESULT hrStatus) { SetEvent(this->complete); return S_OK; } + + ULONG __stdcall AddRef() { InterlockedIncrement((long*) &self_ref); return self_ref; } + + ULONG __stdcall Release() { + ULONG refcnt = self_ref - 1; + if (InterlockedDecrement((long*) &self_ref) == 0) { delete this; return 0; } + return refcnt; + } + + HRESULT __stdcall QueryInterface(REFIID riid, LPVOID* obj) { + HRESULT hr = S_OK; + if (obj == NULL) { hr = E_INVALIDARG; return hr; } + + if ((riid == IID_IUnknown) || (riid == IID_IPortableDevicePropertiesBulkCallback)) { + AddRef(); + *obj = this; + } + else { + *obj = NULL; + hr = E_NOINTERFACE; + } + return hr; + } + + HRESULT __stdcall OnProgress(REFGUID Context, IPortableDeviceValuesCollection* values) { + DWORD num = 0, i; + wchar_t *property = NULL; + IPortableDeviceValues *properties = NULL; + PyObject *temp, *obj; + HRESULT hr; + + if (SUCCEEDED(values->GetCount(&num))) { + PyEval_RestoreThread(this->thread_state); + for (i = 0; i < num; i++) { + hr = values->GetAt(i, &properties); + if (SUCCEEDED(hr)) { + + hr = properties->GetStringValue(WPD_OBJECT_ID, &property); + if (!SUCCEEDED(hr)) continue; + temp = wchar_to_unicode(property); + CoTaskMemFree(property); property = NULL; + if (temp == NULL) continue; + obj = PyDict_GetItem(this->items, temp); + if (obj == NULL) { + obj = Py_BuildValue("{s:O}", "id", temp); + if (obj == NULL) continue; + PyDict_SetItem(this->items, temp, obj); + Py_DECREF(obj); // We want a borrowed reference to obj + } + Py_DECREF(temp); + + set_content_type_property(obj, properties); + + set_string_property(obj, WPD_OBJECT_PARENT_ID, "parent_id", properties); + set_string_property(obj, WPD_OBJECT_NAME, "name", properties); + set_string_property(obj, WPD_OBJECT_SYNC_ID, "sync_id", properties); + set_string_property(obj, WPD_OBJECT_PERSISTENT_UNIQUE_ID, "persistent_id", properties); + + set_bool_property(obj, WPD_OBJECT_ISHIDDEN, "is_hidden", properties); + set_bool_property(obj, WPD_OBJECT_CAN_DELETE, "can_delete", properties); + set_bool_property(obj, WPD_OBJECT_ISSYSTEM, "is_system", properties); + + set_size_property(obj, WPD_OBJECT_SIZE, "size", properties); + + properties->Release(); properties = NULL; + } + } // end for loop + this->thread_state = PyEval_SaveThread(); + } + + return S_OK; + } + +}; + +static PyObject* bulk_get_filesystem(IPortableDevice *device, IPortableDevicePropertiesBulk *bulk_properties, const wchar_t *storage_id, IPortableDevicePropVariantCollection *object_ids) { + PyObject *folders = NULL, *ret = NULL; + GUID guid_context = GUID_NULL; + HANDLE ev = NULL; + IPortableDeviceKeyCollection *properties; + GetBulkCallback *callback = NULL; + HRESULT hr; + DWORD wait_result; + int pump_result; + BOOL ok = TRUE; + + ev = CreateEvent(NULL, FALSE, FALSE, NULL); + if (ev == NULL) return PyErr_NoMemory(); + + folders = PyDict_New(); + if (folders == NULL) {PyErr_NoMemory(); goto end;} + + properties = create_filesystem_properties_collection(); + if (properties == NULL) goto end; + + callback = new (std::nothrow) GetBulkCallback(folders, ev); + if (callback == NULL) { PyErr_NoMemory(); goto end; } + + hr = bulk_properties->QueueGetValuesByObjectList(object_ids, properties, callback, &guid_context); + if (FAILED(hr)) { hresult_set_exc("Failed to queue bulk property retrieval", hr); goto end; } + + hr = bulk_properties->Start(guid_context); + if (FAILED(hr)) { hresult_set_exc("Failed to start bulk operation", hr); goto end; } + + callback->thread_state = PyEval_SaveThread(); + while (TRUE) { + Py_BEGIN_ALLOW_THREADS; + wait_result = MsgWaitForMultipleObjects(1, &(callback->complete), FALSE, 60000, QS_ALLEVENTS); + Py_END_ALLOW_THREADS; + if (wait_result == WAIT_OBJECT_0) { + break; // Event was signalled, bulk operation complete + } else if (wait_result == WAIT_OBJECT_0 + 1) { // Messages need to be dispatched + pump_result = pump_waiting_messages(); + if (pump_result == 1) { PyErr_SetString(PyExc_RuntimeError, "Application has been asked to quit."); ok = FALSE; break;} + } else if (wait_result == WAIT_TIMEOUT) { + // 60 seconds with no updates, looks bad + PyErr_SetString(WPDError, "The device seems to have hung."); ok = FALSE; break; + } else if (wait_result == WAIT_ABANDONED_0) { + // This should never happen + PyErr_SetString(WPDError, "An unknown error occurred (mutex abandoned)"); ok = FALSE; break; + } else { + // The wait failed for some reason + PyErr_SetFromWindowsErr(0); ok = FALSE; break; + } + } + PyEval_RestoreThread(callback->thread_state); + if (!ok) { + // We deliberately leak the callback object to prevent any crashes in case COM tries to call methods on it during a future pump_waiting_messages() + PyDict_Clear(folders); + folders = NULL; + callback = NULL; + ev = NULL; + } +end: + if (folders != NULL) { + ret = PyDict_Values(folders); + Py_DECREF(folders); + } + if (ev != NULL) CloseHandle(ev); + if (properties != NULL) properties->Release(); + if (callback != NULL) callback->Release(); + + return ret; +} + +static BOOL find_all_objects_in(IPortableDeviceContent *content, IPortableDevicePropVariantCollection *object_ids, const wchar_t *parent_id) { + /* + * Find all children of the object identified by parent_id, recursively. + * The child ids are put into object_ids. Returns False if any errors + * occurred (also sets the python exception). + */ + IEnumPortableDeviceObjectIDs *children; + HRESULT hr = S_OK, hr2 = S_OK; + PWSTR child_ids[10]; + DWORD fetched, i; + PROPVARIANT pv; + BOOL ok = 1; + + PropVariantInit(&pv); + pv.vt = VT_LPWSTR; + + Py_BEGIN_ALLOW_THREADS; + hr = content->EnumObjects(0, parent_id, NULL, &children); + Py_END_ALLOW_THREADS; + + if (FAILED(hr)) {hresult_set_exc("Failed to get children from device", hr); ok = 0; goto end;} + + hr = S_OK; + + while (hr == S_OK) { + Py_BEGIN_ALLOW_THREADS; + hr = children->Next(10, child_ids, &fetched); + Py_END_ALLOW_THREADS; + if (SUCCEEDED(hr)) { + for(i = 0; i < fetched; i++) { + pv.pwszVal = child_ids[i]; + hr2 = object_ids->Add(&pv); + pv.pwszVal = NULL; + if (FAILED(hr2)) { hresult_set_exc("Failed to add child ids to propvariantcollection", hr2); break; } + ok = find_all_objects_in(content, object_ids, child_ids[i]); + if (!ok) break; + } + for (i = 0; i < fetched; i++) { CoTaskMemFree(child_ids[i]); child_ids[i] = NULL; } + if (FAILED(hr2) || !ok) { ok = 0; goto end; } + } + } + +end: + if (children != NULL) children->Release(); + PropVariantClear(&pv); + return ok; +} + +PyObject* wpd::get_filesystem(IPortableDevice *device, const wchar_t *storage_id, IPortableDevicePropertiesBulk *bulk_properties) { + PyObject *folders = NULL; + IPortableDevicePropVariantCollection *object_ids = NULL; + IPortableDeviceContent *content = NULL; + HRESULT hr; + BOOL ok; + + Py_BEGIN_ALLOW_THREADS; + hr = device->Content(&content); + Py_END_ALLOW_THREADS; + if (FAILED(hr)) { hresult_set_exc("Failed to create content interface", hr); goto end; } + + Py_BEGIN_ALLOW_THREADS; + hr = CoCreateInstance(CLSID_PortableDevicePropVariantCollection, NULL, + CLSCTX_INPROC_SERVER, IID_PPV_ARGS(&object_ids)); + Py_END_ALLOW_THREADS; + if (FAILED(hr)) { hresult_set_exc("Failed to create propvariantcollection", hr); goto end; } + + ok = find_all_objects_in(content, object_ids, storage_id); + if (!ok) goto end; + + if (bulk_properties != NULL) folders = bulk_get_filesystem(device, bulk_properties, storage_id, object_ids); + +end: + if (content != NULL) content->Release(); + if (object_ids != NULL) object_ids->Release(); + + return folders; +} + +} // namespace wpd diff --git a/src/calibre/devices/mtp/windows/device.cpp b/src/calibre/devices/mtp/windows/device.cpp index 98038764a7..0a03b9e735 100644 --- a/src/calibre/devices/mtp/windows/device.cpp +++ b/src/calibre/devices/mtp/windows/device.cpp @@ -9,7 +9,7 @@ extern IPortableDevice* wpd::open_device(const wchar_t *pnp_id, IPortableDeviceValues *client_information); extern IPortableDeviceValues* wpd::get_client_information(); -extern PyObject* wpd::get_device_information(IPortableDevice *device); +extern PyObject* wpd::get_device_information(IPortableDevice *device, IPortableDevicePropertiesBulk **pb); using namespace wpd; // Device.__init__() {{{ @@ -19,6 +19,8 @@ dealloc(Device* self) if (self->pnp_id != NULL) free(self->pnp_id); self->pnp_id = NULL; + if (self->bulk_properties != NULL) { self->bulk_properties->Release(); self->bulk_properties = NULL; } + if (self->device != NULL) { Py_BEGIN_ALLOW_THREADS; self->device->Close(); self->device->Release(); @@ -44,12 +46,17 @@ init(Device *self, PyObject *args, PyObject *kwds) self->pnp_id = unicode_to_wchar(pnp_id); if (self->pnp_id == NULL) return -1; + self->bulk_properties = NULL; + self->client_information = get_client_information(); if (self->client_information != NULL) { self->device = open_device(self->pnp_id, self->client_information); if (self->device != NULL) { - self->device_information = get_device_information(self->device); - if (self->device_information != NULL) ret = 0; + self->device_information = get_device_information(self->device, &(self->bulk_properties)); + if (self->device_information != NULL) { + ret = 0; + } + } } @@ -62,17 +69,34 @@ init(Device *self, PyObject *args, PyObject *kwds) static PyObject* update_data(Device *self, PyObject *args, PyObject *kwargs) { PyObject *di = NULL; - di = get_device_information(self->device); + di = get_device_information(self->device, NULL); if (di == NULL) return NULL; Py_XDECREF(self->device_information); self->device_information = di; Py_RETURN_NONE; } // }}} +// get_filesystem() {{{ +static PyObject* +py_get_filesystem(Device *self, PyObject *args, PyObject *kwargs) { + PyObject *storage_id, *ans = NULL; + wchar_t *storage; + + if (!PyArg_ParseTuple(args, "O", &storage_id)) return NULL; + storage = unicode_to_wchar(storage_id); + if (storage == NULL) return NULL; + + return wpd::get_filesystem(self->device, storage, self->bulk_properties); +} // }}} + static PyMethodDef Device_methods[] = { {"update_data", (PyCFunction)update_data, METH_VARARGS, "update_data() -> Reread the basic device data from the device (total, space, free space, storage locations, etc.)" }, + {"get_filesystem", (PyCFunction)py_get_filesystem, METH_VARARGS, + "get_filesystem(storage_id) -> Get all files/folders on the storage identified by storage_id. Tries to use bulk operations when possible." + }, + {NULL} }; diff --git a/src/calibre/devices/mtp/windows/device_enumeration.cpp b/src/calibre/devices/mtp/windows/device_enumeration.cpp index 775464d4c8..90bc437be1 100644 --- a/src/calibre/devices/mtp/windows/device_enumeration.cpp +++ b/src/calibre/devices/mtp/windows/device_enumeration.cpp @@ -171,8 +171,9 @@ PyObject* get_storage_info(IPortableDevice *device) { // {{{ Py_DECREF(so); } } - } - } + } + for (i = 0; i < fetched; i ++) { CoTaskMemFree(object_ids[i]); object_ids[i] = NULL;} + }// if(SUCCEEDED(hr)) } ans = storage; @@ -185,9 +186,10 @@ end: return ans; } // }}} -PyObject* get_device_information(IPortableDevice *device) { // {{{ +PyObject* get_device_information(IPortableDevice *device, IPortableDevicePropertiesBulk **pb) { // {{{ IPortableDeviceContent *content = NULL; IPortableDeviceProperties *properties = NULL; + IPortableDevicePropertiesBulk *properties_bulk = NULL; IPortableDeviceKeyCollection *keys = NULL; IPortableDeviceValues *values = NULL; IPortableDeviceCapabilities *capabilities = NULL; @@ -336,10 +338,17 @@ PyObject* get_device_information(IPortableDevice *device) { // {{{ } + Py_BEGIN_ALLOW_THREADS; + hr = properties->QueryInterface(IID_PPV_ARGS(&properties_bulk)); + Py_END_ALLOW_THREADS; + PyDict_SetItemString(ans, "has_bulk_properties", (FAILED(hr)) ? Py_False: Py_True); + if (pb != NULL) *pb = (SUCCEEDED(hr)) ? properties_bulk : NULL; + end: if (keys != NULL) keys->Release(); if (values != NULL) values->Release(); if (properties != NULL) properties->Release(); + if (properties_bulk != NULL && pb == NULL) properties_bulk->Release(); if (content != NULL) content->Release(); if (capabilities != NULL) capabilities->Release(); if (categories != NULL) categories->Release(); diff --git a/src/calibre/devices/mtp/windows/global.h b/src/calibre/devices/mtp/windows/global.h index 75712a18f5..cbf489d424 100644 --- a/src/calibre/devices/mtp/windows/global.h +++ b/src/calibre/devices/mtp/windows/global.h @@ -42,6 +42,7 @@ typedef struct { IPortableDeviceValues *client_information; IPortableDevice *device; PyObject *device_information; + IPortableDevicePropertiesBulk *bulk_properties; } Device; extern PyTypeObject DeviceType; @@ -49,10 +50,13 @@ extern PyTypeObject DeviceType; // Utility functions PyObject *hresult_set_exc(const char *msg, HRESULT hr); wchar_t *unicode_to_wchar(PyObject *o); +PyObject *wchar_to_unicode(wchar_t *o); +int pump_waiting_messages(); extern IPortableDeviceValues* get_client_information(); extern IPortableDevice* open_device(const wchar_t *pnp_id, IPortableDeviceValues *client_information); -extern PyObject* get_device_information(IPortableDevice *device); +extern PyObject* get_device_information(IPortableDevice *device, IPortableDevicePropertiesBulk **bulk_properties); +extern PyObject* get_filesystem(IPortableDevice *device, const wchar_t *storage_id, IPortableDevicePropertiesBulk *bulk_properties); } diff --git a/src/calibre/devices/mtp/windows/remote.py b/src/calibre/devices/mtp/windows/remote.py index b3c25d0216..13e15764b2 100644 --- a/src/calibre/devices/mtp/windows/remote.py +++ b/src/calibre/devices/mtp/windows/remote.py @@ -66,6 +66,7 @@ def main(): # pprint.pprint(dev.detected_devices) print ('Trying to connect to:', pnp_id) dev.open(pnp_id, '') + pprint.pprint(dev.dev.data) print ('Connected to:', dev.get_gui_name()) print ('Total space', dev.total_space()) print ('Free space', dev.free_space()) diff --git a/src/calibre/devices/mtp/windows/utils.cpp b/src/calibre/devices/mtp/windows/utils.cpp index 527885a6f5..243bcc0f59 100644 --- a/src/calibre/devices/mtp/windows/utils.cpp +++ b/src/calibre/devices/mtp/windows/utils.cpp @@ -33,6 +33,7 @@ PyObject *wpd::hresult_set_exc(const char *msg, HRESULT hr) { wchar_t *wpd::unicode_to_wchar(PyObject *o) { wchar_t *buf; Py_ssize_t len; + if (o == NULL) return NULL; if (!PyUnicode_Check(o)) {PyErr_Format(PyExc_TypeError, "The python object must be a unicode object"); return NULL;} len = PyUnicode_GET_SIZE(o); buf = (wchar_t *)calloc(len+2, sizeof(wchar_t)); @@ -42,3 +43,30 @@ wchar_t *wpd::unicode_to_wchar(PyObject *o) { return buf; } +PyObject *wpd::wchar_to_unicode(wchar_t *o) { + PyObject *ans; + if (o == NULL) return NULL; + ans = PyUnicode_FromWideChar(o, wcslen(o)); + if (ans == NULL) PyErr_NoMemory(); + return ans; +} + +int wpd::pump_waiting_messages() { + UINT firstMsg = 0, lastMsg = 0; + MSG msg; + int result = 0; + // Read all of the messages in this next loop, + // removing each message as we read it. + while (PeekMessage(&msg, NULL, firstMsg, lastMsg, PM_REMOVE)) { + // If it's a quit message, we're out of here. + if (msg.message == WM_QUIT) { + result = 1; + break; + } + // Otherwise, dispatch the message. + DispatchMessage(&msg); + } // End of PeekMessage while loop + + return result; +} + diff --git a/src/calibre/devices/mtp/windows/wpd.cpp b/src/calibre/devices/mtp/windows/wpd.cpp index 2fb908ebd3..561eeb1bbc 100644 --- a/src/calibre/devices/mtp/windows/wpd.cpp +++ b/src/calibre/devices/mtp/windows/wpd.cpp @@ -22,7 +22,7 @@ wpd::ClientInfo wpd::client_info = {NULL, 0, 0, 0}; extern IPortableDeviceValues* wpd::get_client_information(); extern IPortableDevice* wpd::open_device(const wchar_t *pnp_id, IPortableDeviceValues *client_information); -extern PyObject* wpd::get_device_information(IPortableDevice *device); +extern PyObject* wpd::get_device_information(IPortableDevice *device, IPortableDevicePropertiesBulk **bulk_properties); // Module startup/shutdown {{{ static PyObject * @@ -151,7 +151,7 @@ wpd_device_info(PyObject *self, PyObject *args) { if (client_information != NULL) { device = open_device(pnp_id, client_information); if (device != NULL) { - ans = get_device_information(device); + ans = get_device_information(device, NULL); } }