Merge from trunk

setup/extensions.py

@@ -174,6 +174,20 @@ if isosx:
                 ldflags=['-framework', 'IOKit'])
             )
 
+if islinux:
+    extensions.append(Extension('libmtp',
+        [
+        'calibre/devices/mtp/unix/devices.c',
+        'calibre/devices/mtp/unix/libmtp.c'
+        ],
+        headers=[
+        'calibre/devices/mtp/unix/devices.h',
+        'calibre/devices/mtp/unix/upstream/music-players.h',
+        'calibre/devices/mtp/unix/upstream/device-flags.h',
+        ],
+        libraries=['mtp']
+    ))
+
 if isunix:
     cc = os.environ.get('CC', 'gcc')
     cxx = os.environ.get('CXX', 'g++')

src/calibre/constants.py

@@ -93,6 +93,8 @@ class Plugins(collections.Mapping):
             plugins.append('winutil')
         if isosx:
             plugins.append('usbobserver')
+        if islinux:
+            plugins.append('libmtp')
         self.plugins = frozenset(plugins)
 
     def load_plugin(self, name):

src/calibre/devices/interface.py

@@ -199,7 +199,7 @@ class DevicePlugin(Plugin):
     # }}}
 
     def reset(self, key='-1', log_packets=False, report_progress=None,
-            detected_device=None) :
+            detected_device=None):
         """
         :param key: The key to unlock the device
         :param log_packets: If true the packet stream to/from the device is logged

src/calibre/devices/mtp/__init__.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+
+

src/calibre/devices/mtp/base.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from calibre.devices.interface import DevicePlugin
+from calibre.devices.usbms.deviceconfig import DeviceConfig
+
+class MTPDeviceBase(DeviceConfig, DevicePlugin):
+    name = 'SmartDevice App Interface'
+    gui_name = _('MTP Device')
+    icon = I('devices/galaxy_s3.png')
+    description = _('Communicate with MTP devices')
+    author = 'Kovid Goyal'
+    version = (1, 0, 0)
+
+    # Invalid USB vendor information so the scanner will never match
+    VENDOR_ID                   = [0xffff]
+    PRODUCT_ID                  = [0xffff]
+    BCD                         = [0xffff]
+
+    THUMBNAIL_HEIGHT = 128
+    CAN_SET_METADATA = []
+
+    BACKLOADING_ERROR_MESSAGE = None
+
+    def reset(self, key='-1', log_packets=False, report_progress=None,
+            detected_device=None):
+        pass
+

src/calibre/devices/mtp/unix/__init__.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+
+'''
+libmtp based drivers for MTP devices on Unix like platforms.
+'''
+

src/calibre/devices/mtp/unix/detect.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from calibre.constants import plugins
+
+class MTPDetect(object):
+
+    def __init__(self):
+        p = plugins['libmtp']
+        self.libmtp = p[0]
+        if self.libmtp is None:
+            print ('Failed to load libmtp, MTP device detection disabled')
+            print (p[1])
+        self.cache = {}
+
+    def __call__(self, devices):
+        '''
+        Given a list of devices as returned by LinuxScanner, return the set of
+        devices that are likely to be MTP devices. This class maintains a cache
+        to minimize USB polling. Note that detection is partially based on a
+        list of known vendor and product ids. This is because polling some
+        older devices causes problems. Therefore, if this method identifies a
+        device as MTP, it is not actually guaranteed that it will be a working
+        MTP device.
+        '''
+        # First drop devices that have been disconnected from the cache
+        connected_devices = {(d.busnum, d.devnum, d.vendor_id, d.product_id,
+            d.bcd, d.serial) for d in devices}
+        for d in tuple(self.cache.iterkeys()):
+            if d not in connected_devices:
+                del self.cache[d]
+
+        # Since is_mtp_device() can cause USB traffic by probing the device, we
+        # cache its result
+        mtp_devices = set()
+        if self.libmtp is None:
+            return mtp_devices
+
+        for d in devices:
+            ans = self.cache.get((d.busnum, d.devnum, d.vendor_id, d.product_id,
+                    d.bcd, d.serial), None)
+            if ans is None:
+                ans = self.libmtp.is_mtp_device(d.busnum, d.devnum,
+                        d.vendor_id, d.product_id)
+                self.cache[(d.busnum, d.devnum, d.vendor_id, d.product_id,
+                    d.bcd, d.serial)] = ans
+            if ans:
+                mtp_devices.add(d)
+        return mtp_devices
+
+    def create_device(self, connected_device):
+        d = connected_device
+        return self.libmtp.Device(d.busnum, d.devnum, d.vendor_id,
+                d.product_id, d.manufacturer, d.product, d.serial)
+
+if __name__ == '__main__':
+    from calibre.devices.scanner import linux_scanner
+    mtp_detect = MTPDetect()
+    devs = mtp_detect(linux_scanner())
+    print ('Found %d MTP devices:'%len(devs))
+    for dev in devs:
+        print (dev, 'at busnum=%d and devnum=%d'%(dev.busnum, dev.devnum))
+        print()
+
+

src/calibre/devices/mtp/unix/devices.c

@@ -0,0 +1,16 @@
+/*
+ * devices.c
+ * Copyright (C) 2012 Kovid Goyal <kovid at kovidgoyal.net>
+ *
+ * Distributed under terms of the MIT license.
+ */
+
+#include "upstream/device-flags.h"
+#include "devices.h"
+
+const calibre_device_entry_t calibre_mtp_device_table[] = {
+#include "upstream/music-players.h"
+
+  , { NULL, 0xffff, NULL, 0xffff, DEVICE_FLAG_NONE }
+};
+

src/calibre/devices/mtp/unix/devices.h

@@ -0,0 +1,22 @@
+#pragma once
+/*
+ * devices.h
+ * Copyright (C) 2012 Kovid Goyal <kovid at kovidgoyal.net>
+ *
+ * Distributed under terms of the MIT license.
+ */
+#include <stdint.h>
+#include <stddef.h>
+
+struct calibre_device_entry_struct {
+  char *vendor; /**< The vendor of this device */
+  uint16_t vendor_id; /**< Vendor ID for this device */
+  char *product; /**< The product name of this device */
+  uint16_t product_id; /**< Product ID for this device */
+  uint32_t device_flags; /**< Bugs, device specifics etc */
+};
+
+typedef struct calibre_device_entry_struct calibre_device_entry_t;
+
+extern const calibre_device_entry_t calibre_mtp_device_table[];
+

src/calibre/devices/mtp/unix/driver.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import time
+from threading import RLock
+from functools import wraps
+
+from calibre.devices.errors import OpenFailed
+from calibre.devices.mtp.base import MTPDeviceBase
+from calibre.devices.mtp.unix.detect import MTPDetect
+
+def synchronous(func):
+    @wraps(func)
+    def synchronizer(self, *args, **kwargs):
+        with self.lock:
+            return func(self, *args, **kwargs)
+    return synchronizer
+
+class MTP_DEVICE(MTPDeviceBase):
+
+    supported_platforms = ['linux']
+
+    def __init__(self, *args, **kwargs):
+        MTPDeviceBase.__init__(self, *args, **kwargs)
+        self.detect = MTPDetect()
+        self.dev = None
+        self.lock = RLock()
+        self.blacklisted_devices = set()
+
+    @synchronous
+    def is_usb_connected(self, devices_on_system, debug=False,
+            only_presence=False):
+
+        # First remove blacklisted devices.
+        devs = []
+        for d in devices_on_system:
+            if (d.busnum, d.devnum, d.vendor_id,
+                d.product_id, d.bcd, d.serial) not in self.blacklisted_devices:
+                devs.append(d)
+
+        devs = self.detect(devs)
+        if self.dev is not None:
+            # Check if the currently opened device is still connected
+            ids = self.dev.ids
+            found = False
+            for d in devs:
+                if ( (d.busnum, d.devnum, d.vendor_id, d.product_id, d.serial)
+                        == ids ):
+                    found = True
+                    break
+            return found
+        # Check if any MTP capable device is present
+        return len(devs) > 0
+
+    @synchronous
+    def post_yank_cleanup(self):
+        self.dev = None
+
+    @synchronous
+    def open(self, connected_device, library_uuid):
+        def blacklist_device():
+            d = connected_device
+            self.blacklisted_devices.add((d.busnum, d.devnum, d.vendor_id,
+                    d.product_id, d.bcd, d.serial))
+        try:
+            self.detect.create_device(connected_device)
+        except ValueError:
+            # Give the device some time to settle
+            time.sleep(2)
+            try:
+                self.detect.create_device(connected_device)
+            except ValueError:
+                # Black list this device so that it is ignored for the
+                # remainder of this session.
+                blacklist_device()
+                raise OpenFailed('%s is not a MTP device'%connected_device)
+        except TypeError:
+            blacklist_device()
+            raise OpenFailed('')
+

src/calibre/devices/mtp/unix/libmtp.c

@@ -0,0 +1,318 @@
+#define UNICODE
+#include <Python.h>
+
+#include <stdlib.h>
+#include <libmtp.h>
+
+#include "devices.h"
+
+// Device object definition {{{
+typedef struct {
+    PyObject_HEAD
+    // Type-specific fields go here.
+    LIBMTP_mtpdevice_t *device;
+    PyObject *ids;
+    PyObject *friendly_name;
+    PyObject *manufacturer_name;
+    PyObject *model_name;
+    PyObject *serial_number;
+    PyObject *device_version;
+
+} libmtp_Device;
+
+static void
+libmtp_Device_dealloc(libmtp_Device* self)
+{
+    if (self->device != NULL) LIBMTP_Release_Device(self->device);
+    self->device = NULL;
+
+    Py_XDECREF(self->ids); self->ids = NULL;
+    Py_XDECREF(self->friendly_name); self->friendly_name = NULL;
+    Py_XDECREF(self->manufacturer_name); self->manufacturer_name = NULL;
+    Py_XDECREF(self->model_name); self->model_name = NULL;
+    Py_XDECREF(self->serial_number); self->serial_number = NULL;
+    Py_XDECREF(self->device_version); self->device_version = NULL;
+
+    self->ob_type->tp_free((PyObject*)self);
+}
+
+static int
+libmtp_Device_init(libmtp_Device *self, PyObject *args, PyObject *kwds)
+{
+    int busnum, devnum, vendor_id, product_id;
+    PyObject *usb_serialnum;
+    char *vendor, *product, *friendly_name, *manufacturer_name, *model_name, *serial_number, *device_version;
+    LIBMTP_raw_device_t rawdev;
+    LIBMTP_mtpdevice_t *dev;
+    size_t i;
+
+    if (!PyArg_ParseTuple(args, "iiiissO", &busnum, &devnum, &vendor_id, &product_id, &vendor, &product, &usb_serialnum)) return -1;
+
+    if (devnum < 0 || devnum > 255 || busnum < 0) { PyErr_SetString(PyExc_TypeError, "Invalid busnum/devnum"); return -1; }
+
+    self->ids = Py_BuildValue("iiiiO", busnum, devnum, vendor_id, product_id, usb_serialnum);
+    if (self->ids == NULL) return -1;
+
+    rawdev.bus_location = (uint32_t)busnum;
+    rawdev.devnum = (uint8_t)devnum;
+    rawdev.device_entry.vendor = vendor;
+    rawdev.device_entry.product = product;
+    rawdev.device_entry.vendor_id = vendor_id;
+    rawdev.device_entry.product_id = product_id;
+    rawdev.device_entry.device_flags = 0x00000000U;
+
+    Py_BEGIN_ALLOW_THREADS;
+    for (i = 0; ; i++) {
+        if (calibre_mtp_device_table[i].vendor == NULL && calibre_mtp_device_table[i].product == NULL && calibre_mtp_device_table[i].vendor_id == 0xffff) break;
+        if (calibre_mtp_device_table[i].vendor_id == vendor_id && calibre_mtp_device_table[i].product_id == product_id) {
+            rawdev.device_entry.device_flags = calibre_mtp_device_table[i].device_flags;
+        }
+    }
+
+    dev = LIBMTP_Open_Raw_Device_Uncached(&rawdev);
+    Py_END_ALLOW_THREADS;
+
+    if (dev == NULL) { 
+        PyErr_SetString(PyExc_ValueError, "Unable to open raw device."); 
+        return -1;
+    }
+
+    self->device = dev;
+
+    Py_BEGIN_ALLOW_THREADS;
+    friendly_name = LIBMTP_Get_Friendlyname(self->device);
+    manufacturer_name = LIBMTP_Get_Manufacturername(self->device);
+    model_name = LIBMTP_Get_Modelname(self->device);
+    serial_number = LIBMTP_Get_Serialnumber(self->device);
+    device_version = LIBMTP_Get_Deviceversion(self->device);
+    Py_END_ALLOW_THREADS;
+
+    if (friendly_name != NULL) {
+        self->friendly_name = PyUnicode_FromString(friendly_name);
+        free(friendly_name);
+    }
+    if (self->friendly_name == NULL) { self->friendly_name = Py_None; Py_INCREF(Py_None); }
+
+    if (manufacturer_name != NULL) {
+        self->manufacturer_name = PyUnicode_FromString(manufacturer_name);
+        free(manufacturer_name);
+    }
+    if (self->manufacturer_name == NULL) { self->manufacturer_name = Py_None; Py_INCREF(Py_None); }
+
+    if (model_name != NULL) {
+        self->model_name = PyUnicode_FromString(model_name);
+        free(model_name);
+    }
+    if (self->model_name == NULL) { self->model_name = Py_None; Py_INCREF(Py_None); }
+
+    if (serial_number != NULL) {
+        self->serial_number = PyUnicode_FromString(serial_number);
+        free(serial_number);
+    }
+    if (self->serial_number == NULL) { self->serial_number = Py_None; Py_INCREF(Py_None); }
+
+    if (device_version != NULL) {
+        self->device_version = PyUnicode_FromString(device_version);
+        free(device_version);
+    }
+    if (self->device_version == NULL) { self->device_version = Py_None; Py_INCREF(Py_None); }
+
+    return 0;
+}
+
+// Collator.friendly_name {{{
+static PyObject *
+libmtp_Device_friendly_name(libmtp_Device *self, void *closure) {
+    return Py_BuildValue("O", self->friendly_name);
+} // }}}
+
+// Collator.manufacturer_name {{{
+static PyObject *
+libmtp_Device_manufacturer_name(libmtp_Device *self, void *closure) {
+    return Py_BuildValue("O", self->manufacturer_name);
+} // }}}
+
+// Collator.model_name {{{
+static PyObject *
+libmtp_Device_model_name(libmtp_Device *self, void *closure) {
+    return Py_BuildValue("O", self->model_name);
+} // }}}
+
+// Collator.serial_number {{{
+static PyObject *
+libmtp_Device_serial_number(libmtp_Device *self, void *closure) {
+    return Py_BuildValue("O", self->serial_number);
+} // }}}
+
+// Collator.device_version {{{
+static PyObject *
+libmtp_Device_device_version(libmtp_Device *self, void *closure) {
+    return Py_BuildValue("O", self->device_version);
+} // }}}
+
+// Collator.ids {{{
+static PyObject *
+libmtp_Device_ids(libmtp_Device *self, void *closure) {
+    return Py_BuildValue("O", self->ids);
+} // }}}
+
+static PyMethodDef libmtp_Device_methods[] = {
+    {NULL}  /* Sentinel */
+};
+
+static PyGetSetDef libmtp_Device_getsetters[] = {
+    {(char *)"friendly_name", 
+     (getter)libmtp_Device_friendly_name, NULL,
+     (char *)"The friendly name of this device, can be None.",
+     NULL},
+
+    {(char *)"manufacturer_name", 
+     (getter)libmtp_Device_manufacturer_name, NULL,
+     (char *)"The manufacturer name of this device, can be None.",
+     NULL},
+
+    {(char *)"model_name", 
+     (getter)libmtp_Device_model_name, NULL,
+     (char *)"The model name of this device, can be None.",
+     NULL},
+
+    {(char *)"serial_number", 
+     (getter)libmtp_Device_serial_number, NULL,
+     (char *)"The serial number of this device, can be None.",
+     NULL},
+
+    {(char *)"device_version", 
+     (getter)libmtp_Device_device_version, NULL,
+     (char *)"The device version of this device, can be None.",
+     NULL},
+
+    {(char *)"ids", 
+     (getter)libmtp_Device_ids, NULL,
+     (char *)"The ids of the device (busnum, devnum, vendor_id, product_id, usb_serialnum)",
+     NULL},
+
+    {NULL}  /* Sentinel */
+};
+
+static PyTypeObject libmtp_DeviceType = { // {{{
+    PyObject_HEAD_INIT(NULL)
+    0,                         /*ob_size*/
+    "libmtp.Device",            /*tp_name*/
+    sizeof(libmtp_Device),      /*tp_basicsize*/
+    0,                         /*tp_itemsize*/
+    (destructor)libmtp_Device_dealloc, /*tp_dealloc*/
+    0,                         /*tp_print*/
+    0,                         /*tp_getattr*/
+    0,                         /*tp_setattr*/
+    0,                         /*tp_compare*/
+    0,                         /*tp_repr*/
+    0,                         /*tp_as_number*/
+    0,                         /*tp_as_sequence*/
+    0,                         /*tp_as_mapping*/
+    0,                         /*tp_hash */
+    0,                         /*tp_call*/
+    0,                         /*tp_str*/
+    0,                         /*tp_getattro*/
+    0,                         /*tp_setattro*/
+    0,                         /*tp_as_buffer*/
+    Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE,        /*tp_flags*/
+    "Device",                  /* tp_doc */
+    0,		               /* tp_traverse */
+    0,		               /* tp_clear */
+    0,		               /* tp_richcompare */
+    0,		               /* tp_weaklistoffset */
+    0,		               /* tp_iter */
+    0,		               /* tp_iternext */
+    libmtp_Device_methods,             /* tp_methods */
+    0,             /* tp_members */
+    libmtp_Device_getsetters,                         /* tp_getset */
+    0,                         /* tp_base */
+    0,                         /* tp_dict */
+    0,                         /* tp_descr_get */
+    0,                         /* tp_descr_set */
+    0,                         /* tp_dictoffset */
+    (initproc)libmtp_Device_init,      /* tp_init */
+    0,                         /* tp_alloc */
+    0,                 /* tp_new */
+}; // }}}
+
+// }}} End Device object definition
+
+static PyObject *
+libmtp_set_debug_level(PyObject *self, PyObject *args) {
+    int level;
+    if (!PyArg_ParseTuple(args, "i", &level)) return NULL;
+    LIBMTP_Set_Debug(level);
+    Py_RETURN_NONE;
+}
+
+
+static PyObject *
+libmtp_is_mtp_device(PyObject *self, PyObject *args) {
+    int busnum, devnum, vendor_id, prod_id, ans = 0;
+    size_t i;
+
+    if (!PyArg_ParseTuple(args, "iiii", &busnum, &devnum, &vendor_id, &prod_id)) return NULL;
+
+    for (i = 0; ; i++) {
+        if (calibre_mtp_device_table[i].vendor == NULL && calibre_mtp_device_table[i].product == NULL && calibre_mtp_device_table[i].vendor_id == 0xffff) break;
+        if (calibre_mtp_device_table[i].vendor_id == vendor_id && calibre_mtp_device_table[i].product_id == prod_id) {
+            Py_RETURN_TRUE;
+        }
+    }
+
+    /*
+     * LIBMTP_Check_Specific_Device does not seem to work at least on my linux
+     * system. Need to investigate why later. Most devices are in the device
+     * table so this is not terribly important.
+     */
+    /* LIBMTP_Set_Debug(LIBMTP_DEBUG_ALL); */
+    /* printf("Calling check: %d %d\n", busnum, devnum); */
+    Py_BEGIN_ALLOW_THREADS;
+    ans = LIBMTP_Check_Specific_Device(busnum, devnum);
+    Py_END_ALLOW_THREADS;
+
+    if (ans) Py_RETURN_TRUE;
+
+    Py_RETURN_FALSE;
+
+}
+
+static PyMethodDef libmtp_methods[] = {
+    {"set_debug_level", libmtp_set_debug_level, METH_VARARGS,
+        "set_debug_level(level)\n\nSet the debug level bit mask, see LIBMTP_DEBUG_* constants."
+    },
+
+    {"is_mtp_device", libmtp_is_mtp_device, METH_VARARGS,
+        "is_mtp_device(busnum, devnum, vendor_id, prod_id)\n\nReturn True if the device is recognized as an MTP device by its vendor/product ids. If it is not recognized a probe is done and True returned if the probe succeeds. Note that probing can cause some devices to malfunction, and it is not very reliable, which is why we prefer to use the device database."
+    },
+
+    {NULL, NULL, 0, NULL}
+};
+
+
+PyMODINIT_FUNC
+initlibmtp(void) {
+    PyObject *m;
+
+    libmtp_DeviceType.tp_new = PyType_GenericNew;
+    if (PyType_Ready(&libmtp_DeviceType) < 0)
+        return;
+    
+    m = Py_InitModule3("libmtp", libmtp_methods, "Interface to libmtp.");
+    if (m == NULL) return;
+
+    LIBMTP_Init();
+    LIBMTP_Set_Debug(LIBMTP_DEBUG_NONE);
+
+    Py_INCREF(&libmtp_DeviceType);
+    PyModule_AddObject(m, "Device", (PyObject *)&libmtp_DeviceType);
+
+    PyModule_AddStringMacro(m, LIBMTP_VERSION_STRING);
+    PyModule_AddIntMacro(m, LIBMTP_DEBUG_NONE);
+    PyModule_AddIntMacro(m, LIBMTP_DEBUG_PTP);
+    PyModule_AddIntMacro(m, LIBMTP_DEBUG_PLST);
+    PyModule_AddIntMacro(m, LIBMTP_DEBUG_USB);
+    PyModule_AddIntMacro(m, LIBMTP_DEBUG_DATA);
+    PyModule_AddIntMacro(m, LIBMTP_DEBUG_ALL);
+}

src/calibre/devices/mtp/unix/upstream/device-flags.h

@@ -0,0 +1,329 @@
+/**
+ * \file device-flags.h
+ * Special device flags to deal with bugs in specific devices.
+ *
+ * Copyright (C) 2005-2007 Richard A. Low <richard@wentnet.com>
+ * Copyright (C) 2005-2012 Linus Walleij <triad@df.lth.se>
+ * Copyright (C) 2006-2007 Marcus Meissner
+ * Copyright (C) 2007 Ted Bullock
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ *
+ * This file is supposed to be included by both libmtp and libgphoto2.
+ */
+
+/**
+ * These flags are used to indicate if some or other
+ * device need special treatment. These should be possible
+ * to concatenate using logical OR so please use one bit per
+ * feature and lets pray we don't need more than 32 bits...
+ */
+#define DEVICE_FLAG_NONE 0x00000000
+/**
+ * This means that the PTP_OC_MTP_GetObjPropList is broken
+ * in the sense that it won't return properly formatted metadata
+ * for ALL files on the device when you request an object
+ * property list for object 0xFFFFFFFF with parameter 3 likewise
+ * set to 0xFFFFFFFF. Compare to
+ * DEVICE_FLAG_BROKEN_MTPGETOBJECTPROPLIST which only signify
+ * that it's broken when getting metadata for a SINGLE object.
+ * A typical way the implementation may be broken is that it
+ * may not return a proper count of the objects, and sometimes
+ * (like on the ZENs) objects are simply missing from the list
+ * if you use this. Sometimes it has been used incorrectly to
+ * mask bugs in the code (like handling transactions of data
+ * with size given to -1 (0xFFFFFFFFU), in that case please
+ * help us remove it now the code is fixed. Sometimes this is
+ * used because getting all the objects is just too slow and
+ * the USB transaction will time out if you use this command.
+ */
+#define DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST_ALL 0x00000001
+/**
+ * This means that under Linux, another kernel module may
+ * be using this device's USB interface, so we need to detach
+ * it if it is. Typically this is on dual-mode devices that
+ * will present both an MTP compliant interface and device
+ * descriptor *and* a USB mass storage interface. If the USB
+ * mass storage interface is in use, other apps (like our
+ * userspace libmtp through libusb access path) cannot get in
+ * and get cosy with it. So we can remove the offending
+ * application. Typically this means you have to run the program
+ * as root as well.
+ */
+#define DEVICE_FLAG_UNLOAD_DRIVER 0x00000002
+/**
+ * This means that the PTP_OC_MTP_GetObjPropList (9805)
+ * is broken in some way, either it doesn't work at all
+ * (as for Android devices) or it won't properly return all
+ * object properties if parameter 3 is set to 0xFFFFFFFFU.
+ */
+#define DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST 0x00000004
+/**
+ * This means the device doesn't send zero packets to indicate
+ * end of transfer when the transfer boundary occurs at a
+ * multiple of 64 bytes (the USB 1.1 endpoint size). Instead,
+ * exactly one extra byte is sent at the end of the transfer
+ * if the size is an integer multiple of USB 1.1 endpoint size
+ * (64 bytes).
+ *
+ * This behaviour is most probably a workaround due to the fact
+ * that the hardware USB slave controller in the device cannot
+ * handle zero writes at all, and the usage of the USB 1.1
+ * endpoint size is due to the fact that the device will "gear
+ * down" on a USB 1.1 hub, and since 64 bytes is a multiple of
+ * 512 bytes, it will work with USB 1.1 and USB 2.0 alike.
+ */
+#define DEVICE_FLAG_NO_ZERO_READS 0x00000008
+/**
+ * This flag means that the device is prone to forgetting the
+ * OGG container file type, so that libmtp must look at the
+ * filename extensions in order to determine that a file is
+ * actually OGG. This is a clear and present firmware bug, and
+ * while firmware bugs should be fixed in firmware, we like
+ * OGG so much that we back it by introducing this flag.
+ * The error has only been seen on iriver devices. Turning this
+ * flag on won't hurt anything, just that the check against
+ * filename extension will be done for files of "unknown" type.
+ * If the player does not even know (reports) that it supports
+ * ogg even though it does, please use the stronger
+ * OGG_IS_UNKNOWN flag, which will forcedly support ogg on
+ * anything with the .ogg filename extension.
+ */
+#define DEVICE_FLAG_IRIVER_OGG_ALZHEIMER 0x00000010
+/**
+ * This flag indicates a limitation in the filenames a device
+ * can accept - they must be 7 bit (all chars <= 127/0x7F).
+ * It was found first on the Philips Shoqbox, and is a deviation
+ * from the PTP standard which mandates that any unicode chars
+ * may be used for filenames. I guess this is caused by a 7bit-only
+ * filesystem being used intrinsically on the device.
+ */
+#define DEVICE_FLAG_ONLY_7BIT_FILENAMES 0x00000020
+/**
+ * This flag indicates that the device will lock up if you
+ * try to get status of endpoints and/or release the interface
+ * when closing the device. This fixes problems with SanDisk
+ * Sansa devices especially. It may be a side-effect of a
+ * Windows behaviour of never releasing interfaces.
+ */
+#define DEVICE_FLAG_NO_RELEASE_INTERFACE 0x00000040
+/**
+ * This flag was introduced with the advent of Creative ZEN
+ * 8GB. The device sometimes return a broken PTP header
+ * like this: < 1502 0000 0200 01d1 02d1 01d2 >
+ * the latter 6 bytes (representing "code" and "transaction ID")
+ * contain junk. This is breaking the PTP/MTP spec but works
+ * on Windows anyway, probably because the Windows implementation
+ * does not check that these bytes are valid. To interoperate
+ * with devices like this, we need this flag to emulate the
+ * Windows bug. Broken headers has also been found in the
+ * Aricent MTP stack.
+ */
+#define DEVICE_FLAG_IGNORE_HEADER_ERRORS 0x00000080
+/**
+ * The Motorola RAZR2 V8 (others?) has broken set object
+ * proplist causing the metadata setting to fail. (The
+ * set object prop to set individual properties work on
+ * this device, but the metadata is plain ignored on
+ * tracks, though e.g. playlist names can be set.)
+ */
+#define DEVICE_FLAG_BROKEN_SET_OBJECT_PROPLIST 0x00000100
+/**
+ * The Samsung YP-T10 think Ogg files shall be sent with
+ * the "unknown" (PTP_OFC_Undefined) file type, this gives a
+ * side effect that is a combination of the iRiver Ogg Alzheimer
+ * problem (have to recognized Ogg files on file extension)
+ * and a need to report the Ogg support (the device itself does
+ * not properly claim to support it) and need to set filetype
+ * to unknown when storing Ogg files, even though they're not
+ * actually unknown. Later iRivers seem to need this flag since
+ * they do not report to support OGG even though they actually
+ * do. Often the device supports OGG in USB mass storage mode,
+ * then the firmware simply miss to declare metadata support
+ * for OGG properly.
+ */
+#define DEVICE_FLAG_OGG_IS_UNKNOWN 0x00000200
+/**
+ * The Creative Zen is quite unstable in libmtp but seems to
+ * be better with later firmware versions. However, it still
+ * frequently crashes when setting album art dimensions. This
+ * flag disables setting the dimensions (which seems to make
+ * no difference to how the graphic is displayed).
+ */
+#define DEVICE_FLAG_BROKEN_SET_SAMPLE_DIMENSIONS 0x00000400
+/**
+ * Some devices, particularly SanDisk Sansas, need to always
+ * have their "OS Descriptor" probed in order to work correctly.
+ * This flag provides that extra massage.
+ */
+#define DEVICE_FLAG_ALWAYS_PROBE_DESCRIPTOR 0x00000800
+/**
+ * Samsung has implimented its own playlist format as a .spl file
+ * stored in the normal file system, rather than a proper mtp
+ * playlist. There are multiple versions of the .spl format
+ * identified by a line in the file: VERSION X.XX
+ * Version 1.00 is just a simple playlist.
+ */
+#define DEVICE_FLAG_PLAYLIST_SPL_V1 0x00001000
+/**
+ * Samsung has implimented its own playlist format as a .spl file
+ * stored in the normal file system, rather than a proper mtp
+ * playlist. There are multiple versions of the .spl format
+ * identified by a line in the file: VERSION X.XX
+ * Version 2.00 is playlist but allows DNSe sound settings
+ * to be stored, per playlist.
+ */
+#define DEVICE_FLAG_PLAYLIST_SPL_V2 0x00002000
+/**
+ * The Sansa E250 is know to have this problem which is actually
+ * that the device claims that property PTP_OPC_DateModified
+ * is read/write but will still fail to update it. It can only
+ * be set properly the first time a file is sent.
+ */
+#define DEVICE_FLAG_CANNOT_HANDLE_DATEMODIFIED 0x00004000
+/**
+ * This avoids use of the send object proplist which
+ * is used when creating new objects (not just updating)
+ * The DEVICE_FLAG_BROKEN_SET_OBJECT_PROPLIST is related
+ * but only concerns the case where the object proplist
+ * is sent in to update an existing object. The Toshiba
+ * Gigabeat MEU202 for example has this problem.
+ */
+#define DEVICE_FLAG_BROKEN_SEND_OBJECT_PROPLIST 0x00008000
+/**
+ * Devices that cannot support reading out battery
+ * level.
+ */
+#define DEVICE_FLAG_BROKEN_BATTERY_LEVEL 0x00010000
+
+/**
+ * Devices that send "ObjectDeleted" events after deletion
+ * of images. (libgphoto2)
+ */
+#define DEVICE_FLAG_DELETE_SENDS_EVENT	0x00020000
+
+/**
+ * Cameras that can capture images. (libgphoto2)
+ */
+#define DEVICE_FLAG_CAPTURE		0x00040000
+
+/**
+ * Cameras that can capture images. (libgphoto2)
+ */
+#define DEVICE_FLAG_CAPTURE_PREVIEW	0x00080000
+
+/**
+ * Nikon broken capture support without proper ObjectAdded events.
+ * (libgphoto2)
+ */
+#define DEVICE_FLAG_NIKON_BROKEN_CAPTURE	0x00100000
+
+/**
+ * Broken capture support where cameras do not send CaptureComplete events.
+ * (libgphoto2)
+ */
+#define DEVICE_FLAG_NO_CAPTURE_COMPLETE		0x00400000
+
+/**
+ * Direct PTP match required.
+ * (libgphoto2)
+ */
+#define DEVICE_FLAG_MATCH_PTP_INTERFACE		0x00800000
+/**
+ * This flag is like DEVICE_FLAG_OGG_IS_UNKNOWN but for FLAC
+ * files instead. Using the unknown filetype for FLAC files.
+ */
+#define DEVICE_FLAG_FLAC_IS_UNKNOWN		0x01000000
+/**
+ * Device needs unique filenames, no two files can be
+ * named the same string.
+ */
+#define DEVICE_FLAG_UNIQUE_FILENAMES		0x02000000
+/**
+ * This flag performs some random magic on the BlackBerry
+ * device to switch from USB mass storage to MTP mode we think.
+ */
+#define DEVICE_FLAG_SWITCH_MODE_BLACKBERRY	0x04000000
+/**
+ * This flag indicates that the device need an extra long
+ * timeout on some operations.
+ */
+#define DEVICE_FLAG_LONG_TIMEOUT		0x08000000
+/**
+ * This flag indicates that the device need an explicit
+ * USB reset after each connection. Some devices don't
+ * like this, so it's not done by default.
+ */
+#define DEVICE_FLAG_FORCE_RESET_ON_CLOSE	0x10000000
+/**
+ * Early Creative Zen (etc) models actually only support
+ * command 9805 (Get object property list) and will hang
+ * if you try to get individual properties of an object.
+ */
+#define DEVICE_FLAG_BROKEN_GET_OBJECT_PROPVAL	0x20000000
+/**
+ * It seems that some devices return an bad data when
+ * using the GetObjectInfo operation. So in these cases
+ * we prefer to override the PTP-compatible object infos
+ * with the MTP property list.
+ *
+ * For example Some Samsung Galaxy S devices contain an MTP
+ * stack that present the ObjectInfo in 64 bit instead of
+ * 32 bit.
+ */
+#define DEVICE_FLAG_PROPLIST_OVERRIDES_OI	0x40000000
+
+/**
+ * All these bug flags need to be set on SONY NWZ Walkman
+ * players, and will be autodetected on unknown devices
+ * by detecting the vendor extension descriptor "sony.net"
+ */
+#define DEVICE_FLAGS_SONY_NWZ_BUGS \
+  (DEVICE_FLAG_UNLOAD_DRIVER | \
+   DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST | \
+   DEVICE_FLAG_UNIQUE_FILENAMES | \
+   DEVICE_FLAG_FORCE_RESET_ON_CLOSE )
+/**
+ * All these bug flags need to be set on Android devices,
+ * they claim to support MTP operations they actually
+ * cannot handle, especially 9805 (Get object property list).
+ * These are auto-assigned to devices reporting
+ * "android.com" in their device extension descriptor.
+ */
+#define DEVICE_FLAGS_ANDROID_BUGS \
+  (DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST | \
+   DEVICE_FLAG_BROKEN_SET_OBJECT_PROPLIST | \
+   DEVICE_FLAG_BROKEN_SEND_OBJECT_PROPLIST | \
+   DEVICE_FLAG_UNLOAD_DRIVER | \
+   DEVICE_FLAG_LONG_TIMEOUT )
+/**
+ * All these bug flags appear on a number of SonyEricsson
+ * devices including Android devices not using the stock
+ * Android 4.0+ (Ice Cream Sandwich) MTP stack. It is highly
+ * supected that these bugs comes from an MTP implementation
+ * from Aricent, so it is called the Aricent bug flags as a
+ * shorthand. Especially the header errors that need to be
+ * ignored is typical for this stack.
+ *
+ * After some guesswork we auto-assign these bug flags to
+ * devices that present the "microsoft.com/WPDNA", and
+ * "sonyericsson.com/SE" but NOT the "android.com"
+ * descriptor.
+ */
+#define DEVICE_FLAGS_ARICENT_BUGS \
+  (DEVICE_FLAG_IGNORE_HEADER_ERRORS | \
+   DEVICE_FLAG_BROKEN_SEND_OBJECT_PROPLIST | \
+   DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST )

src/calibre/devices/mtp/unix/upstream/update.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+MP = 'http://libmtp.git.sourceforge.net/git/gitweb.cgi?p=libmtp/libmtp;a=blob_plain;f=src/music-players.h;hb=HEAD'
+DF = 'http://libmtp.git.sourceforge.net/git/gitweb.cgi?p=libmtp/libmtp;a=blob_plain;f=src/device-flags.h;hb=HEAD'
+
+import urllib, os, shutil
+
+base = os.path.dirname(os.path.abspath(__file__))
+
+for url, fname in [(MP, 'music-players.h'), (DF, 'device-flags.h')]:
+    with open(os.path.join(base, fname), 'wb') as f:
+        shutil.copyfileobj(urllib.urlopen(url), f)
+

src/calibre/devices/scanner.py

@@ -7,6 +7,7 @@ manner.
 
 import sys, os, re
 from threading import RLock
+from collections import namedtuple
 
 from calibre import prints, as_unicode
 from calibre.constants import iswindows, isosx, plugins, islinux, isfreebsd
@@ -107,6 +108,15 @@ class WinPNPScanner(object):
 
 win_pnp_drives = WinPNPScanner()
 
+_USBDevice = namedtuple('USBDevice',
+    'vendor_id product_id bcd manufacturer product serial')
+
+class USBDevice(_USBDevice):
+
+    def __init__(self, *args, **kwargs):
+        _USBDevice.__init__(self, *args, **kwargs)
+        self.busnum = self.devnum = -1
+
 class LinuxScanner(object):
 
     SYSFS_PATH = os.environ.get('SYSFS_PATH', '/sys')
@@ -122,6 +132,10 @@ class LinuxScanner(object):
         if not self.ok:
             raise RuntimeError('DeviceScanner requires the /sys filesystem to work.')
 
+        def read(f):
+            with open(f, 'rb') as s:
+                return s.read().strip()
+
         for x in os.listdir(self.base):
             base = os.path.join(self.base, x)
             ven = os.path.join(base, 'idVendor')
@@ -132,31 +146,46 @@ class LinuxScanner(object):
             prod_string = os.path.join(base, 'product')
             dev = []
             try:
-                dev.append(int('0x'+open(ven).read().strip(), 16))
+                # Ignore USB HUBs
+                if read(os.path.join(base, 'bDeviceClass')) == b'09':
+                    continue
             except:
                 continue
             try:
-                dev.append(int('0x'+open(prod).read().strip(), 16))
+                dev.append(int(b'0x'+read(ven), 16))
             except:
                 continue
             try:
-                dev.append(int('0x'+open(bcd).read().strip(), 16))
+                dev.append(int(b'0x'+read(prod), 16))
             except:
                 continue
             try:
-                dev.append(open(man).read().strip())
+                dev.append(int(b'0x'+read(bcd), 16))
             except:
-                dev.append('')
+                continue
             try:
-                dev.append(open(prod_string).read().strip())
+                dev.append(read(man))
             except:
-                dev.append('')
+                dev.append(b'')
             try:
-                dev.append(open(serial).read().strip())
+                dev.append(read(prod_string))
             except:
-                dev.append('')
+                dev.append(b'')
+            try:
+                dev.append(read(serial))
+            except:
+                dev.append(b'')
 
-            ans.add(tuple(dev))
+            dev = USBDevice(*dev)
+            try:
+                dev.busnum = int(read(os.path.join(base, 'busnum')))
+            except:
+                pass
+            try:
+                dev.devnum = int(read(os.path.join(base, 'devnum')))
+            except:
+                pass
+            ans.add(dev)
         return ans
 
 class FreeBSDScanner(object):

src/calibre/library/database2.py

@@ -31,7 +31,8 @@ from calibre.ptempfile import (PersistentTemporaryFile,
 from calibre.customize.ui import run_plugins_on_import
 from calibre import isbytestring
 from calibre.utils.filenames import ascii_filename
-from calibre.utils.date import utcnow, now as nowf, utcfromtimestamp, parse_date
+from calibre.utils.date import (utcnow, now as nowf, utcfromtimestamp,
+        parse_only_date)
 from calibre.utils.config import prefs, tweaks, from_json, to_json
 from calibre.utils.icu import sort_key, strcmp, lower
 from calibre.utils.search_query_parser import saved_searches, set_saved_searches
@@ -2479,8 +2480,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
 
     def set_pubdate(self, id, dt, notify=True, commit=True):
         if dt:
-            if isinstance(dt, (str, unicode, bytes)):
-                dt = parse_date(dt)
+            if isinstance(dt, basestring):
+                dt = parse_only_date(dt)
             self.conn.execute('UPDATE books SET pubdate=? WHERE id=?', (dt, id))
             self.data.set(id, self.FIELD_MAP['pubdate'], dt, row_is_id=True)
             self.dirtied([id], commit=False)