Merge from trunk

src/calibre/devices/interface.py

@@ -487,7 +487,8 @@ class DevicePlugin(Plugin):
         Set the device name in the driveinfo file to 'name'. This setting will
         persist until the file is re-created or the name is changed again.
 
-        Non-disk devices will ignore this request.
+        Non-disk devices should implement this method based on the location
+        codes returned by the get_device_information() method.
         '''
         pass
 

src/calibre/devices/mtp/base.py

@@ -26,11 +26,6 @@ class MTPDeviceBase(DevicePlugin):
     author = 'Kovid Goyal'
     version = (1, 0, 0)
 
-    # Invalid USB vendor information so the scanner will never match
-    VENDOR_ID                   = [0xffff]
-    PRODUCT_ID                  = [0xffff]
-    BCD                         = [0xffff]
-
     THUMBNAIL_HEIGHT = 128
     CAN_SET_METADATA = []
 
@@ -51,4 +46,10 @@ class MTPDeviceBase(DevicePlugin):
     def get_gui_name(self):
         return self.current_friendly_name or self.name
 
+    def is_usb_connected(self, devices_on_system, debug=False,
+            only_presence=False):
+        # We manage device presence ourselves, so this method should always
+        # return False
+        return False
+
 

src/calibre/devices/mtp/filesystem_cache.py

@@ -8,11 +8,12 @@ __copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
 import weakref, sys
+from collections import deque
 from operator import attrgetter
 from future_builtins import map
 
 from calibre import human_readable, prints, force_unicode
-from calibre.utils.icu import sort_key
+from calibre.utils.icu import sort_key, lower
 
 class FileOrFolder(object):
 
@@ -20,15 +21,19 @@ class FileOrFolder(object):
         self.object_id = entry['id']
         self.is_folder = entry['is_folder']
         self.name = force_unicode(entry.get('name', '___'), 'utf-8')
+        self.storage_id = entry.get('storage_id', None)
         self.persistent_id = entry.get('persistent_id', self.object_id)
         self.size = entry.get('size', 0)
         # self.parent_id is None for storage objects
         self.parent_id = entry.get('parent_id', None)
         if self.parent_id == 0:
-            sid = entry['storage_id']
+            sid = self.storage_id
             if sid not in all_storage_ids:
                 sid = all_storage_ids[0]
             self.parent_id = sid
+        if self.parent_id is None and self.storage_id is None:
+            # A storage object
+            self.storage_id = self.object_id
         self.is_hidden = entry.get('is_hidden', False)
         self.is_system = entry.get('is_system', False)
         self.can_delete = entry.get('can_delete', True)
@@ -46,12 +51,28 @@ class FileOrFolder(object):
     def parent(self):
         return None if self.parent_id is None else self.id_map[self.parent_id]
 
+    @property
+    def full_path(self):
+        parts = deque()
+        parts.append(self.name)
+        p = self.parent
+        while p is not None:
+            parts.appendleft(p.name)
+            p = p.parent
+        return tuple(parts)
+
     def __iter__(self):
         for e in self.folders:
             yield e
         for e in self.files:
             yield e
 
+    def add_child(self, entry):
+        ans = FileOrFolder(entry, self.id_map)
+        t = self.folders if ans.is_folder else self.files
+        t.append(ans)
+        return ans
+
     def dump(self, prefix='', out=sys.stdout):
         c = '+' if self.is_folder else '-'
         data = ('%s children'%(sum(map(len, (self.files, self.folders))))
@@ -62,6 +83,20 @@ class FileOrFolder(object):
             for e in sorted(c, key=lambda x:sort_key(x.name)):
                 e.dump(prefix=prefix+'  ', out=out)
 
+    def folder_named(self, name):
+        name = lower(name)
+        for e in self.folders:
+            if e.name and lower(e.name) == name:
+                return e
+        return None
+
+    def file_named(self, name):
+        name = lower(name)
+        for e in self.files:
+            if e.name and lower(e.name) == name:
+                return e
+        return None
+
 class FilesystemCache(object):
 
     def __init__(self, all_storage, entries):
@@ -79,7 +114,17 @@ class FilesystemCache(object):
             FileOrFolder(entry, self, all_storage_ids)
 
         for item in self.id_map.itervalues():
-            p = item.parent
+            try:
+                p = item.parent
+            except KeyError:
+                # Parent does not exist, set the parent to be the storage
+                # object
+                sid = p.storage_id
+                if sid not in all_storage_ids:
+                    sid = all_storage_ids[0]
+                item.parent_id = sid
+                p = item.parent
+
             if p is not None:
                 t = p.folders if item.is_folder else p.files
                 t.append(item)

src/calibre/devices/mtp/unix/detect.py

@@ -1,71 +0,0 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
-from __future__ import (unicode_literals, division, absolute_import,
-                        print_function)
-
-__license__   = 'GPL v3'
-__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
-__docformat__ = 'restructuredtext en'
-
-from calibre.constants import plugins
-
-class MTPDetect(object):
-
-    def __init__(self):
-        p = plugins['libmtp']
-        self.libmtp = p[0]
-        if self.libmtp is None:
-            print ('Failed to load libmtp, MTP device detection disabled')
-            print (p[1])
-        self.cache = {}
-
-    def __call__(self, devices):
-        '''
-        Given a list of devices as returned by LinuxScanner, return the set of
-        devices that are likely to be MTP devices. This class maintains a cache
-        to minimize USB polling. Note that detection is partially based on a
-        list of known vendor and product ids. This is because polling some
-        older devices causes problems. Therefore, if this method identifies a
-        device as MTP, it is not actually guaranteed that it will be a working
-        MTP device.
-        '''
-        # First drop devices that have been disconnected from the cache
-        connected_devices = {(d.busnum, d.devnum, d.vendor_id, d.product_id,
-            d.bcd, d.serial) for d in devices}
-        for d in tuple(self.cache.iterkeys()):
-            if d not in connected_devices:
-                del self.cache[d]
-
-        # Since is_mtp_device() can cause USB traffic by probing the device, we
-        # cache its result
-        mtp_devices = set()
-        if self.libmtp is None:
-            return mtp_devices
-
-        for d in devices:
-            ans = self.cache.get((d.busnum, d.devnum, d.vendor_id, d.product_id,
-                    d.bcd, d.serial), None)
-            if ans is None:
-                ans = self.libmtp.is_mtp_device(d.busnum, d.devnum,
-                        d.vendor_id, d.product_id)
-                self.cache[(d.busnum, d.devnum, d.vendor_id, d.product_id,
-                    d.bcd, d.serial)] = ans
-            if ans:
-                mtp_devices.add(d)
-        return mtp_devices
-
-    def create_device(self, connected_device):
-        d = connected_device
-        return self.libmtp.Device(d.busnum, d.devnum, d.vendor_id,
-                d.product_id, d.manufacturer, d.product, d.serial)
-
-if __name__ == '__main__':
-    from calibre.devices.scanner import linux_scanner
-    mtp_detect = MTPDetect()
-    devs = mtp_detect(linux_scanner())
-    print ('Found %d MTP devices:'%len(devs))
-    for dev in devs:
-        print (dev, 'at busnum=%d and devnum=%d'%(dev.busnum, dev.devnum))
-        print()
-
-

src/calibre/devices/mtp/unix/driver.py

@@ -10,11 +10,19 @@ __docformat__ = 'restructuredtext en'
 import time, operator
 from threading import RLock
 from io import BytesIO
+from collections import namedtuple
 
+from calibre.constants import plugins
 from calibre.devices.errors import OpenFailed, DeviceError
 from calibre.devices.mtp.base import MTPDeviceBase, synchronous
 from calibre.devices.mtp.filesystem_cache import FilesystemCache
-from calibre.devices.mtp.unix.detect import MTPDetect
+
+MTPDevice = namedtuple('MTPDevice', 'busnum devnum vendor_id product_id '
+        'bcd serial manufacturer product')
+
+def fingerprint(d):
+    return MTPDevice(d.busnum, d.devnum, d.vendor_id, d.product_id, d.bcd,
+            d.serial, d.manufacturer, d.product)
 
 class MTP_DEVICE(MTPDeviceBase):
 
@@ -22,13 +30,18 @@ class MTP_DEVICE(MTPDeviceBase):
 
     def __init__(self, *args, **kwargs):
         MTPDeviceBase.__init__(self, *args, **kwargs)
+        self.libmtp = None
+        self.detect_cache = {}
+
         self.dev = None
         self._filesystem_cache = None
         self.lock = RLock()
         self.blacklisted_devices = set()
+        self.ejected_devices = set()
+        self.currently_connected_dev = None
 
     def set_debug_level(self, lvl):
-        self.detect.libmtp.set_debug_level(lvl)
+        self.libmtp.set_debug_level(lvl)
 
     def report_progress(self, sent, total):
         try:
@@ -39,40 +52,67 @@ class MTP_DEVICE(MTPDeviceBase):
             self.progress_reporter(p)
 
     @synchronous
-    def is_usb_connected(self, devices_on_system, debug=False,
+    def detect_managed_devices(self, devices_on_system):
-            only_presence=False):
+        if self.libmtp is None: return None
-
         # First remove blacklisted devices.
-        devs = []
+        devs = set()
         for d in devices_on_system:
-            if (d.busnum, d.devnum, d.vendor_id,
+            fp = fingerprint(d)
-                d.product_id, d.bcd, d.serial) not in self.blacklisted_devices:
+            if fp not in self.blacklisted_devices:
-                devs.append(d)
+                devs.add(fp)
 
-        devs = self.detect(devs)
+        # Clean up ejected devices
-        if self.dev is not None:
+        self.ejected_devices = devs.intersection(self.ejected_devices)
-            # Check if the currently opened device is still connected
+
-            ids = self.dev.ids
+        # Check if the currently connected device is still present
-            found = False
+        if self.currently_connected_dev is not None:
-            for d in devs:
+            return (self.currently_connected_dev if
-                if ( (d.busnum, d.devnum, d.vendor_id, d.product_id, d.serial)
+                    self.currently_connected_dev in devs else None)
-                        == ids ):
+
-                    found = True
+        # Remove ejected devices
-                    break
+        devs = devs - self.ejected_devices
-            return found
+
-        # Check if any MTP capable device is present
+        # Now check for MTP devices
-        return len(devs) > 0
+        cache = self.detect_cache
+        for d in devs:
+            ans = cache.get(d, None)
+            if ans is None:
+                ans = self.libmtp.is_mtp_device(d.busnum, d.devnum,
+                        d.vendor_id, d.product_id)
+                cache[d] = ans
+            if ans:
+                return d
+
+        return None
+
+    @synchronous
+    def create_device(self, connected_device):
+        d = connected_device
+        return self.libmtp.Device(d.busnum, d.devnum, d.vendor_id,
+                d.product_id, d.manufacturer, d.product, d.serial)
+
+    @synchronous
+    def eject(self):
+        if self.currently_connected_dev is None: return
+        self.ejected_devices.add(self.currently_connected_dev)
+        self.post_yank_cleanup()
 
     @synchronous
     def post_yank_cleanup(self):
         self.dev = self._filesystem_cache = self.current_friendly_name = None
+        self.currently_connected_dev = None
 
     @synchronous
     def startup(self):
-        self.detect = MTPDetect()
+        p = plugins['libmtp']
-        for x in vars(self.detect.libmtp):
+        self.libmtp = p[0]
+        if self.libmtp is None:
+            print ('Failed to load libmtp, MTP device detection disabled')
+            print (p[1])
+
+        for x in vars(self.libmtp):
             if x.startswith('LIBMTP'):
-                setattr(self, x, getattr(self.detect.libmtp, x))
+                setattr(self, x, getattr(self.libmtp, x))
 
     @synchronous
     def shutdown(self):
@@ -85,29 +125,25 @@ class MTP_DEVICE(MTPDeviceBase):
     @synchronous
     def open(self, connected_device, library_uuid):
         self.dev = self._filesystem_cache = None
-        def blacklist_device():
-            d = connected_device
-            self.blacklisted_devices.add((d.busnum, d.devnum, d.vendor_id,
-                    d.product_id, d.bcd, d.serial))
         try:
-            self.dev = self.detect.create_device(connected_device)
+            self.dev = self.create_device(connected_device)
-        except ValueError:
+        except self.libmtp.MTPError:
             # Give the device some time to settle
             time.sleep(2)
             try:
-                self.dev = self.detect.create_device(connected_device)
+                self.dev = self.create_device(connected_device)
-            except ValueError:
+            except self.libmtp.MTPError:
                 # Black list this device so that it is ignored for the
                 # remainder of this session.
-                blacklist_device()
+                self.blacklisted_devices.add(connected_device)
                 raise OpenFailed('%s is not a MTP device'%(connected_device,))
         except TypeError:
-            blacklist_device()
+            self.blacklisted_devices.add(connected_device)
             raise OpenFailed('')
 
         storage = sorted(self.dev.storage_info, key=operator.itemgetter('id'))
         if not storage:
-            blacklist_device()
+            self.blacklisted_devices.add(connected_device)
             raise OpenFailed('No storage found for device %s'%(connected_device,))
         self._main_id = storage[0]['id']
         self._carda_id = self._cardb_id = None
@@ -186,6 +222,16 @@ class MTP_DEVICE(MTPDeviceBase):
                 ans[i] = s['freespace_bytes']
         return tuple(ans)
 
+    @synchronous
+    def create_folder(self, parent_id, name):
+        parent = self.filesystem_cache.id_map[parent_id]
+        if not parent.is_folder:
+            raise ValueError('%s is not a folder'%parent.full_path)
+        e = parent.folder_named(name)
+        if e is not None:
+            return e
+        ans = self.dev.create_folder(parent.storage_id, parent_id, name)
+        return parent.add_child(ans)
 
 if __name__ == '__main__':
     BytesIO
@@ -198,8 +244,8 @@ if __name__ == '__main__':
     dev.startup()
     from calibre.devices.scanner import linux_scanner
     devs = linux_scanner()
-    mtp_devs = dev.detect(devs)
+    cd = dev.detect_managed_devices(devs)
-    dev.open(list(mtp_devs)[0], 'xxx')
+    dev.open(cd, 'xxx')
     d = dev.dev
     print ("Opened device:", dev.get_gui_name())
     print ("Storage info:")

src/calibre/devices/mtp/windows/content_enumeration.cpp

@@ -28,6 +28,7 @@ static IPortableDeviceKeyCollection* create_filesystem_properties_collection() {
     ADDPROP(WPD_OBJECT_PARENT_ID);
     ADDPROP(WPD_OBJECT_PERSISTENT_UNIQUE_ID);
     ADDPROP(WPD_OBJECT_NAME);
+    ADDPROP(WPD_OBJECT_ORIGINAL_FILE_NAME);
     // ADDPROP(WPD_OBJECT_SYNC_ID);
     ADDPROP(WPD_OBJECT_ISSYSTEM);
     ADDPROP(WPD_OBJECT_ISHIDDEN);
@@ -92,8 +93,9 @@ static void set_properties(PyObject *obj, IPortableDeviceValues *values) {
     set_content_type_property(obj, values);
 
     set_string_property(obj, WPD_OBJECT_PARENT_ID, "parent_id", values);
-    set_string_property(obj, WPD_OBJECT_NAME, "name", values);
+    set_string_property(obj, WPD_OBJECT_NAME, "nominal_name", values);
     // set_string_property(obj, WPD_OBJECT_SYNC_ID, "sync_id", values);
+    set_string_property(obj, WPD_OBJECT_ORIGINAL_FILE_NAME, "name", values);
     set_string_property(obj, WPD_OBJECT_PERSISTENT_UNIQUE_ID, "persistent_id", values);
 
     set_bool_property(obj, WPD_OBJECT_ISHIDDEN, "is_hidden", values);
@@ -370,6 +372,42 @@ end:
 } 
 // }}}
 
+static IPortableDeviceValues* create_object_properties(const wchar_t *parent_id, const wchar_t *name, const GUID content_type, unsigned PY_LONG_LONG size) { // {{{
+    IPortableDeviceValues *values = NULL;
+    HRESULT hr;
+    BOOL ok = FALSE;
+
+    hr = CoCreateInstance(CLSID_PortableDeviceValues, NULL,
+            CLSCTX_INPROC_SERVER, IID_PPV_ARGS(&values));
+    if (FAILED(hr)) { hresult_set_exc("Failed to create values interface", hr); goto end; }
+
+    hr = values->SetStringValue(WPD_OBJECT_PARENT_ID, parent_id);
+    if (FAILED(hr)) { hresult_set_exc("Failed to set parent_id value", hr); goto end; }
+
+    hr = values->SetStringValue(WPD_OBJECT_NAME, name);
+    if (FAILED(hr)) { hresult_set_exc("Failed to set name value", hr); goto end; }
+
+    hr = values->SetStringValue(WPD_OBJECT_ORIGINAL_FILE_NAME, name);
+    if (FAILED(hr)) { hresult_set_exc("Failed to set original_file_name value", hr); goto end; }
+
+    hr = values->SetGuidValue(WPD_OBJECT_FORMAT, WPD_OBJECT_FORMAT_UNSPECIFIED);
+    if (FAILED(hr)) { hresult_set_exc("Failed to set object_format value", hr); goto end; }
+
+    hr = values->SetGuidValue(WPD_OBJECT_CONTENT_TYPE, content_type);
+    if (FAILED(hr)) { hresult_set_exc("Failed to set content_type value", hr); goto end; }
+
+    if (!IsEqualGUID(WPD_CONTENT_TYPE_FOLDER, content_type)) {
+        hr = values->SetUnsignedLargeIntegerValue(WPD_OBJECT_SIZE, size);
+        if (FAILED(hr)) { hresult_set_exc("Failed to set size value", hr); goto end; }
+    }
+
+    ok = TRUE;
+
+end:
+    if (!ok && values != NULL) { values->Release(); values = NULL; }
+    return values;
+} // }}}
+
 PyObject* wpd::get_filesystem(IPortableDevice *device, const wchar_t *storage_id, IPortableDevicePropertiesBulk *bulk_properties) { // {{{
     PyObject *folders = NULL;
     IPortableDevicePropVariantCollection *object_ids = NULL;
@@ -467,7 +505,7 @@ PyObject* wpd::get_file(IPortableDevice *device, const wchar_t *object_id, PyObj
         total_read = total_read + bytes_read;
         if (hr == STG_E_ACCESSDENIED) { 
             PyErr_SetString(PyExc_IOError, "Read access is denied to this object"); break; 
-        } else if (hr == S_OK || hr == S_FALSE) {
+        } else if (SUCCEEDED(hr)) {
             if (bytes_read > 0) {
                 res = PyObject_CallMethod(dest, "write", "s#", buf, bytes_read);
                 if (res == NULL) break;
@@ -476,7 +514,7 @@ PyObject* wpd::get_file(IPortableDevice *device, const wchar_t *object_id, PyObj
             }
         } else { hresult_set_exc("Failed to read file from device", hr); break; }
 
-        if (hr == S_FALSE || bytes_read < bufsize) { 
+        if (bytes_read == 0) { 
             ok = TRUE; 
             Py_XDECREF(PyObject_CallMethod(dest, "flush", NULL));
             break;
@@ -500,4 +538,170 @@ end:
     Py_RETURN_NONE;
 } // }}}
 
+PyObject* wpd::create_folder(IPortableDevice *device, const wchar_t *parent_id, const wchar_t *name) { // {{{
+    IPortableDeviceContent *content = NULL;
+    IPortableDeviceValues *values = NULL;
+    IPortableDeviceProperties *devprops = NULL;
+    IPortableDeviceKeyCollection *properties = NULL;
+    wchar_t *newid = NULL;
+    PyObject *ans = NULL;
+    HRESULT hr;
+
+    values = create_object_properties(parent_id, name, WPD_CONTENT_TYPE_FOLDER, 0);
+    if (values == NULL) goto end;
+
+    Py_BEGIN_ALLOW_THREADS;
+    hr = device->Content(&content);
+    Py_END_ALLOW_THREADS;
+    if (FAILED(hr)) { hresult_set_exc("Failed to create content interface", hr); goto end; }
+
+    hr = content->Properties(&devprops);
+    if (FAILED(hr)) { hresult_set_exc("Failed to get IPortableDeviceProperties interface", hr); goto end; }
+
+    properties = create_filesystem_properties_collection();
+    if (properties == NULL) goto end;
+
+    Py_BEGIN_ALLOW_THREADS;
+    hr = content->CreateObjectWithPropertiesOnly(values, &newid);
+    Py_END_ALLOW_THREADS;
+    if (FAILED(hr) || newid == NULL) { hresult_set_exc("Failed to create folder", hr); goto end; }
+
+    ans = get_object_properties(devprops, properties, newid);
+end:
+    if (content != NULL) content->Release();
+    if (values != NULL) values->Release();
+    if (devprops != NULL) devprops->Release();
+    if (properties != NULL) properties->Release();
+    if (newid != NULL) CoTaskMemFree(newid);
+    return ans;
+
+} // }}}
+
+PyObject* wpd::delete_object(IPortableDevice *device, const wchar_t *object_id) { // {{{
+    IPortableDeviceContent *content = NULL;
+    HRESULT hr;
+    BOOL ok = FALSE;
+    PROPVARIANT pv;
+    IPortableDevicePropVariantCollection *object_ids = NULL;
+
+    PropVariantInit(&pv);
+    pv.vt      = VT_LPWSTR;
+
+    Py_BEGIN_ALLOW_THREADS;
+    hr = CoCreateInstance(CLSID_PortableDevicePropVariantCollection, NULL,
+            CLSCTX_INPROC_SERVER, IID_PPV_ARGS(&object_ids));
+    Py_END_ALLOW_THREADS;
+    if (FAILED(hr)) { hresult_set_exc("Failed to create propvariantcollection", hr); goto end; }
+    pv.pwszVal = (wchar_t*)object_id;
+    hr = object_ids->Add(&pv); 
+    pv.pwszVal = NULL;
+    if (FAILED(hr)) { hresult_set_exc("Failed to add device id to propvariantcollection", hr); goto end; }
+
+    Py_BEGIN_ALLOW_THREADS;
+    hr = device->Content(&content);
+    Py_END_ALLOW_THREADS;
+    if (FAILED(hr)) { hresult_set_exc("Failed to create content interface", hr); goto end; }
+
+    hr = content->Delete(PORTABLE_DEVICE_DELETE_NO_RECURSION, object_ids, NULL);
+    if (hr == E_ACCESSDENIED) PyErr_SetString(WPDError, "Do not have permission to delete this object");
+    else if (hr == HRESULT_FROM_WIN32(ERROR_DIR_NOT_EMPTY) || hr == HRESULT_FROM_WIN32(ERROR_INVALID_OPERATION)) PyErr_SetString(WPDError, "Cannot delete object as it has children");
+    else if (hr == HRESULT_FROM_WIN32(ERROR_NOT_FOUND) || SUCCEEDED(hr)) ok = TRUE;
+    else hresult_set_exc("Cannot delete object", hr);
+
+end:
+    PropVariantClear(&pv);
+    if (content != NULL) content->Release();
+    if (object_ids != NULL) object_ids->Release();
+    if (!ok) return NULL;
+    Py_RETURN_NONE;
+
+} // }}}
+
+PyObject* wpd::put_file(IPortableDevice *device, const wchar_t *parent_id, const wchar_t *name, PyObject *src, unsigned PY_LONG_LONG size, PyObject *callback) { // {{{
+    IPortableDeviceContent *content = NULL;
+    IPortableDeviceValues *values = NULL;
+    IPortableDeviceProperties *devprops = NULL;
+    IPortableDeviceKeyCollection *properties = NULL;
+    IStream *temp = NULL;
+    IPortableDeviceDataStream *dest = NULL;
+    char *buf = NULL;
+    wchar_t *newid = NULL;
+    PyObject *ans = NULL, *raw;
+    HRESULT hr;
+    DWORD bufsize = 0;
+    BOOL ok = FALSE;
+    Py_ssize_t bytes_read = 0;
+    ULONG bytes_written = 0, total_written = 0;
+
+    values = create_object_properties(parent_id, name, WPD_CONTENT_TYPE_GENERIC_FILE, size);
+    if (values == NULL) goto end;
+
+    Py_BEGIN_ALLOW_THREADS;
+    hr = device->Content(&content);
+    Py_END_ALLOW_THREADS;
+    if (FAILED(hr)) { hresult_set_exc("Failed to create content interface", hr); goto end; }
+
+    hr = content->Properties(&devprops);
+    if (FAILED(hr)) { hresult_set_exc("Failed to get IPortableDeviceProperties interface", hr); goto end; }
+
+    properties = create_filesystem_properties_collection();
+    if (properties == NULL) goto end;
+
+    Py_BEGIN_ALLOW_THREADS;
+    hr = content->CreateObjectWithPropertiesAndData(values, &temp, &bufsize, NULL);
+    Py_END_ALLOW_THREADS;
+    if (FAILED(hr)) { 
+        if (HRESULT_FROM_WIN32(ERROR_BUSY) == hr) {
+            PyErr_SetString(WPDFileBusy, "Object is in use");
+        } else hresult_set_exc("Failed to create stream interface to write to object", hr); 
+        goto end; 
+    }
+
+    hr = temp->QueryInterface(IID_PPV_ARGS(&dest));
+    if (FAILED(hr)) { hresult_set_exc("Failed to create IPortableDeviceStream", hr); goto end; }
+
+    while(TRUE) {
+        raw = PyObject_CallMethod(src, "read", "k", bufsize);
+        if (raw == NULL) break;
+        PyBytes_AsStringAndSize(raw, &buf, &bytes_read);
+        if (bytes_read > 0) {
+            Py_BEGIN_ALLOW_THREADS;
+            hr = dest->Write(buf, bytes_read, &bytes_written);
+            Py_END_ALLOW_THREADS;
+            Py_DECREF(raw);
+            if (hr == STG_E_MEDIUMFULL) { PyErr_SetString(WPDError, "Cannot write to device as it is full"); break; }
+            if (hr == STG_E_ACCESSDENIED) { PyErr_SetString(WPDError, "Cannot write to file as access is denied"); break; }
+            if (hr == STG_E_WRITEFAULT) { PyErr_SetString(WPDError, "Cannot write to file as there was a disk I/O error"); break; }
+            if (FAILED(hr)) { hresult_set_exc("Cannot write to file", hr); break; }
+            if (bytes_written != bytes_read) { PyErr_SetString(WPDError, "Writing to file failed, not all bytes were written"); break; }
+            total_written += bytes_written;
+            if (callback != NULL) Py_XDECREF(PyObject_CallFunction(callback, "kK", total_written, size));
+        } else Py_DECREF(raw);
+        if (bytes_read == 0) { ok = TRUE; break; }
+    }
+    if (!ok) {dest->Revert(); goto end;}
+    Py_BEGIN_ALLOW_THREADS;
+    hr = dest->Commit(STGC_DEFAULT);
+    Py_END_ALLOW_THREADS;
+    if (FAILED(hr)) { hresult_set_exc("Failed to write data to file, commit failed", hr); goto end; }
+    if (callback != NULL) Py_XDECREF(PyObject_CallFunction(callback, "kK", total_written, size));
+
+    Py_BEGIN_ALLOW_THREADS;
+    hr = dest->GetObjectID(&newid);
+    Py_END_ALLOW_THREADS;
+    if (FAILED(hr)) { hresult_set_exc("Failed to get id of newly created file", hr); goto end; }
+
+    ans = get_object_properties(devprops, properties, newid);
+end:
+    if (content != NULL) content->Release();
+    if (values != NULL) values->Release();
+    if (devprops != NULL) devprops->Release();
+    if (properties != NULL) properties->Release();
+    if (temp != NULL) temp->Release();
+    if (dest != NULL) dest->Release();
+    if (newid != NULL) CoTaskMemFree(newid);
+    return ans;
+
+} // }}}
+
 } // namespace wpd

src/calibre/devices/mtp/windows/device.cpp

@@ -78,20 +78,22 @@ update_data(Device *self, PyObject *args, PyObject *kwargs) {
 // get_filesystem() {{{
 static PyObject*
 py_get_filesystem(Device *self, PyObject *args, PyObject *kwargs) {
-    PyObject *storage_id;
+    PyObject *storage_id, *ret;
     wchar_t *storage;
 
     if (!PyArg_ParseTuple(args, "O", &storage_id)) return NULL;
     storage = unicode_to_wchar(storage_id);
     if (storage == NULL) return NULL;
 
-    return wpd::get_filesystem(self->device, storage, self->bulk_properties);
+    ret = wpd::get_filesystem(self->device, storage, self->bulk_properties);
+    free(storage);
+    return ret;
 } // }}}
 
 // get_file() {{{
 static PyObject*
 py_get_file(Device *self, PyObject *args, PyObject *kwargs) {
-    PyObject *object_id, *stream, *callback = NULL;
+    PyObject *object_id, *stream, *callback = NULL, *ret;
     wchar_t *object;
 
     if (!PyArg_ParseTuple(args, "OO|O", &object_id, &stream, &callback)) return NULL;
@@ -100,7 +102,59 @@ py_get_file(Device *self, PyObject *args, PyObject *kwargs) {
 
     if (callback == NULL || !PyCallable_Check(callback)) callback = NULL;
 
-    return wpd::get_file(self->device, object, stream, callback);
+    ret = wpd::get_file(self->device, object, stream, callback);
+    free(object);
+    return ret;
+} // }}}
+
+// create_folder() {{{
+static PyObject*
+py_create_folder(Device *self, PyObject *args, PyObject *kwargs) {
+    PyObject *pparent_id, *pname, *ret;
+    wchar_t *parent_id, *name;
+
+    if (!PyArg_ParseTuple(args, "OO", &pparent_id, &pname)) return NULL;
+    parent_id = unicode_to_wchar(pparent_id);
+    name = unicode_to_wchar(pname);
+    if (parent_id == NULL || name == NULL) return NULL;
+
+    ret = wpd::create_folder(self->device, parent_id, name);
+    free(parent_id); free(name);
+    return ret;
+} // }}}
+
+// delete_object() {{{
+static PyObject*
+py_delete_object(Device *self, PyObject *args, PyObject *kwargs) {
+    PyObject *pobject_id, *ret;
+    wchar_t *object_id;
+
+    if (!PyArg_ParseTuple(args, "O", &pobject_id)) return NULL;
+    object_id = unicode_to_wchar(pobject_id);
+    if (object_id == NULL) return NULL;
+
+    ret =  wpd::delete_object(self->device, object_id);
+    free(object_id);
+    return ret;
+} // }}}
+
+// get_file() {{{
+static PyObject*
+py_put_file(Device *self, PyObject *args, PyObject *kwargs) {
+    PyObject *pparent_id, *pname, *stream, *callback = NULL, *ret;
+    wchar_t *parent_id, *name;
+    unsigned PY_LONG_LONG size;
+
+    if (!PyArg_ParseTuple(args, "OOOK|O", &pparent_id, &pname, &stream, &size, &callback)) return NULL;
+    parent_id = unicode_to_wchar(pparent_id);
+    name = unicode_to_wchar(pname);
+    if (parent_id == NULL || name == NULL) return NULL;
+
+    if (callback == NULL || !PyCallable_Check(callback)) callback = NULL;
+
+    ret = wpd::put_file(self->device, parent_id, name, stream, size, callback);
+    free(parent_id); free(name);
+    return ret;
 } // }}}
 
 static PyMethodDef Device_methods[] = {
@@ -116,6 +170,18 @@ static PyMethodDef Device_methods[] = {
      "get_file(object_id, stream, callback=None) -> Get the file identified by object_id from the device. The file is written to the stream object, which must be a file like object. If callback is not None, it must be a callable that accepts two arguments: (bytes_read, total_size). It will be called after each chunk is read from the device. Note that it can be called multiple times with the same values."
     },
 
+    {"create_folder", (PyCFunction)py_create_folder, METH_VARARGS,
+     "create_folder(parent_id, name) -> Create a folder. Returns the folder metadata."
+    },
+
+    {"delete_object", (PyCFunction)py_delete_object, METH_VARARGS,
+     "delete_object(object_id) -> Delete the object identified by object_id. Note that trying to delete a non-empty folder will raise an error."
+    },
+
+    {"put_file", (PyCFunction)py_put_file, METH_VARARGS,
+     "put_file(parent_id, name, stream, size_in_bytes, callback=None) -> Copy a file from the stream object, creating a new file on the device with parent identified by parent_id. Returns the file metadata of the newly created file. callback should be a callable that accepts two argument: (bytes_written, total_size). It will be called after each chunk is written to the device. Note that it can be called multiple times with the same arguments."
+    },
+
     {NULL}
 };
 

src/calibre/devices/mtp/windows/driver.py

@@ -15,7 +15,7 @@ from itertools import chain
 from calibre import as_unicode, prints
 from calibre.constants import plugins, __appname__, numeric_version
 from calibre.ptempfile import SpooledTemporaryFile
-from calibre.devices.errors import OpenFailed
+from calibre.devices.errors import OpenFailed, DeviceError
 from calibre.devices.mtp.base import MTPDeviceBase
 from calibre.devices.mtp.filesystem_cache import FilesystemCache
 
@@ -156,6 +156,7 @@ class MTP_DEVICE(MTPDeviceBase):
                 storage = {'id':storage_id, 'size':capacity, 'name':name,
                         'is_folder':True}
                 id_map = self.dev.get_filesystem(storage_id)
+                for x in id_map.itervalues(): x['storage_id'] = storage_id
                 all_storage.append(storage)
                 items.append(id_map.itervalues())
             self._filesystem_cache = FilesystemCache(all_storage, chain(*items))
@@ -204,6 +205,10 @@ class MTP_DEVICE(MTPDeviceBase):
     def get_device_information(self, end_session=True):
         d = self.dev.data
         dv = d.get('device_version', '')
+        for sid, location_code in ( (self._main_id, 'main'), (self._carda_id,
+            'A'), (self._cardb_id, 'B')):
+            if sid is None: continue
+            # TODO: Implement the drive info dict
         return (self.current_friendly_name, dv, dv, '')
 
     @same_thread
@@ -240,12 +245,31 @@ class MTP_DEVICE(MTPDeviceBase):
 
     @same_thread
     def get_file(self, object_id, stream=None, callback=None):
+        f = self.filesystem_cache.id_map[object_id]
+        if f.is_folder:
+            raise ValueError('%s is a folder on the device'%f.full_path)
         if stream is None:
             stream = SpooledTemporaryFile(5*1024*1024, '_wpd_receive_file.dat')
         try:
-            self.dev.get_file(object_id, stream, callback)
+            try:
-        except self.wpd.WPDFileBusy:
+                self.dev.get_file(object_id, stream, callback)
-            time.sleep(2)
+            except self.wpd.WPDFileBusy:
-            self.dev.get_file(object_id, stream, callback)
+                time.sleep(2)
+                self.dev.get_file(object_id, stream, callback)
+        except Exception as e:
+            raise DeviceError('Failed to fetch the file %s with error: %s'%
+                    f.full_path, as_unicode(e))
         return stream
 
+    @same_thread
+    def create_folder(self, parent_id, name):
+        parent = self.filesystem_cache.id_map[parent_id]
+        if not parent.is_folder:
+            raise ValueError('%s is not a folder'%parent.full_path)
+        e = parent.folder_named(name)
+        if e is not None:
+            return e
+        ans = self.dev.create_folder(parent_id, name)
+        ans['storage_id'] = parent.storage_id
+        return parent.add_child(ans)
+

src/calibre/devices/mtp/windows/global.h

@@ -58,6 +58,9 @@ extern IPortableDevice* open_device(const wchar_t *pnp_id, IPortableDeviceValues
 extern PyObject* get_device_information(IPortableDevice *device, IPortableDevicePropertiesBulk **bulk_properties);
 extern PyObject* get_filesystem(IPortableDevice *device, const wchar_t *storage_id, IPortableDevicePropertiesBulk *bulk_properties);
 extern PyObject* get_file(IPortableDevice *device, const wchar_t *object_id, PyObject *dest, PyObject *callback);
+extern PyObject* create_folder(IPortableDevice *device, const wchar_t *parent_id, const wchar_t *name);
+extern PyObject* delete_object(IPortableDevice *device, const wchar_t *object_id);
+extern PyObject* put_file(IPortableDevice *device, const wchar_t *parent_id, const wchar_t *name, PyObject *src, unsigned PY_LONG_LONG size, PyObject *callback);
 
 }
 

src/calibre/devices/mtp/windows/remote.py

@@ -71,12 +71,16 @@ def main():
         print ('Total space', dev.total_space())
         print ('Free space', dev.free_space())
         dev.filesystem_cache.dump()
+        # pprint.pprint(dev.dev.create_folder(dev.filesystem_cache.entries[0].object_id,
+        #     'zzz'))
         # print ('Fetching file: oFF (198214 bytes)')
         # stream = dev.get_file('oFF')
         # print ("Fetched size: ", stream.tell())
     finally:
         dev.shutdown()
 
+    print ('Device connection shutdown')
+
 if __name__ == '__main__':
     main()
 

src/calibre/gui2/update.py

@@ -27,6 +27,10 @@ def get_newest_version():
             'win' if iswindows else 'osx' if isosx else 'oth')
     req.add_header('CALIBRE_INSTALL_UUID', prefs['installation_uuid'])
     version = br.open(req).read().strip()
+    try:
+        version = version.decode('utf-8')
+    except UnicodeDecodeError:
+        version = u''
     return version
 
 class CheckForUpdates(QThread):