Merge from trunk

resources/catalog/stylesheet.css

@@ -74,7 +74,7 @@ p.date_read {
 p.author {
 	font-size:large;
 	margin-top:0em;
-	margin-bottom:0em;
+	margin-bottom:0.1em;
 	text-align: center;
 	text-indent: 0em;
   	}
@@ -122,7 +122,7 @@ p.genres {
 
 p.series {
 	font-style:italic;
-	margin-top:0.25em;
+	margin-top:0.10em;
 	margin-bottom:0em;
 	margin-left:2em;
 	text-align:left;

setup/extensions.py

@@ -298,7 +298,8 @@ class Build(Command):
         self.obj_dir = os.path.join(os.path.dirname(SRC), 'build', 'objects')
         if not os.path.exists(self.obj_dir):
             os.makedirs(self.obj_dir)
-        self.build_style(self.j(self.SRC, 'calibre', 'plugins'))
+        if not opts.only:
+            self.build_style(self.j(self.SRC, 'calibre', 'plugins'))
         for ext in extensions:
             if opts.only != 'all' and opts.only != ext.name:
                 continue

src/calibre/constants.py

@@ -28,7 +28,8 @@ isosx     = 'darwin' in _plat
 isnewosx  = isosx and getattr(sys, 'new_app_bundle', False)
 isfreebsd = 'freebsd' in _plat
 isnetbsd = 'netbsd' in _plat
-isbsd = isfreebsd or isnetbsd
+isdragonflybsd = 'dragonfly' in _plat
+isbsd = isfreebsd or isnetbsd or isdragonflybsd
 islinux   = not(iswindows or isosx or isbsd)
 isfrozen  = hasattr(sys, 'frozen')
 isunix = isosx or islinux

src/calibre/devices/android/driver.py

@@ -87,7 +87,7 @@ class ANDROID(USBMS):
 
             # Google
             0x18d1 : {
-                0x0001 : [0x0223, 0x9999],
+                0x0001 : [0x0223, 0x230, 0x9999],
                 0x0003 : [0x0230],
                 0x4e11 : [0x0100, 0x226, 0x227],
                 0x4e12 : [0x0100, 0x226, 0x227],
@@ -196,7 +196,7 @@ class ANDROID(USBMS):
             'GENERIC-', 'ZTE', 'MID', 'QUALCOMM', 'PANDIGIT', 'HYSTON',
             'VIZIO', 'GOOGLE', 'FREESCAL', 'KOBO_INC', 'LENOVO', 'ROCKCHIP',
             'POCKET', 'ONDA_MID', 'ZENITHIN', 'INGENIC', 'PMID701C', 'PD',
-            'PMP5097C', 'MASS', 'NOVO7', 'ZEKI']
+            'PMP5097C', 'MASS', 'NOVO7', 'ZEKI', 'COBY']
     WINDOWS_MAIN_MEM = ['ANDROID_PHONE', 'A855', 'A853', 'INC.NEXUS_ONE',
             '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD', 'SGH-I897',
             'GT-I9000', 'FILE-STOR_GADGET', 'SGH-T959_CARD', 'SGH-T959', 'SAMSUNG_ANDROID',
@@ -214,7 +214,8 @@ class ANDROID(USBMS):
             'KTABLET_PC', 'INGENIC', 'GT-I9001_CARD', 'USB_2.0_DRIVER',
             'GT-S5830L_CARD', 'UNIVERSE', 'XT875', 'PRO', '.KOBO_VOX',
             'THINKPAD_TABLET', 'SGH-T989', 'YP-G70', 'STORAGE_DEVICE',
-            'ADVANCED', 'SGH-I727', 'USB_FLASH_DRIVER', 'ANDROID']
+            'ADVANCED', 'SGH-I727', 'USB_FLASH_DRIVER', 'ANDROID',
+            'S5830I_CARD', 'MID7042']
     WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897',
             'FILE-STOR_GADGET', 'SGH-T959_CARD', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD',
             'A70S', 'A101IT', '7', 'INCREDIBLE', 'A7EB', 'SGH-T849_CARD',
@@ -224,7 +225,7 @@ class ANDROID(USBMS):
             'USB_2.0_DRIVER', 'I9100T', 'P999DW_SD_CARD', 'KTABLET_PC',
             'FILE-CD_GADGET', 'GT-I9001_CARD', 'USB_2.0_DRIVER', 'XT875',
             'UMS_COMPOSITE', 'PRO', '.KOBO_VOX', 'SGH-T989_CARD', 'SGH-I727',
-            'USB_FLASH_DRIVER', 'ANDROID']
+            'USB_FLASH_DRIVER', 'ANDROID', 'MID7042']
 
     OSX_MAIN_MEM = 'Android Device Main Memory'
 

src/calibre/devices/mtp/unix/libmtp.c

@@ -1,3 +1,11 @@
+/*
+ * libmtp.c
+ * Copyright (C) 2012 Kovid Goyal <kovid at kovidgoyal.net>
+ *
+ * Distributed under terms of the GPL3 license.
+ */
+
+
 #define UNICODE
 #include <Python.h>
 

src/calibre/devices/mtp/windows/device_enumeration.cpp

@@ -2,7 +2,7 @@
  * device_enumeration.cpp
  * Copyright (C) 2012 Kovid Goyal <kovid at kovidgoyal.net>
  *
- * Distributed under terms of the MIT license.
+ * Distributed under terms of the GPL3 license.
  */
 
 #include "global.h"
@@ -72,6 +72,119 @@ IPortableDevice *open_device(const wchar_t *pnp_id, IPortableDeviceValues *clien
 
 } // }}}
 
+PyObject* get_storage_info(IPortableDevice *device) { // {{{
+    HRESULT hr, hr2;
+    IPortableDeviceContent *content = NULL;
+    IEnumPortableDeviceObjectIDs *objects = NULL;
+    IPortableDeviceProperties *properties = NULL;
+    IPortableDeviceKeyCollection *storage_properties = NULL;
+    IPortableDeviceValues *values = NULL;
+    PyObject *ans = NULL, *storage = NULL, *so = NULL, *desc = NULL, *soid = NULL;
+    DWORD fetched, i;
+    PWSTR object_ids[10];
+    GUID guid;
+    ULONGLONG capacity, free_space, capacity_objects, free_objects;
+    ULONG access;
+    LPWSTR storage_desc = NULL;
+
+    storage = PyList_New(0);
+    if (storage == NULL) { PyErr_NoMemory(); goto end; }
+
+    Py_BEGIN_ALLOW_THREADS;
+    hr = device->Content(&content);
+    Py_END_ALLOW_THREADS;
+    if (FAILED(hr)) {hresult_set_exc("Failed to get content interface from device", hr); goto end;}
+
+    Py_BEGIN_ALLOW_THREADS;
+    hr = content->Properties(&properties);
+    Py_END_ALLOW_THREADS;
+    if (FAILED(hr)) {hresult_set_exc("Failed to get properties interface", hr); goto end;}
+
+    Py_BEGIN_ALLOW_THREADS;
+    hr = CoCreateInstance(CLSID_PortableDeviceKeyCollection, NULL,
+            CLSCTX_INPROC_SERVER, IID_PPV_ARGS(&storage_properties));
+    Py_END_ALLOW_THREADS;
+    if (FAILED(hr)) {hresult_set_exc("Failed to create storage properties collection", hr); goto end;}
+
+    Py_BEGIN_ALLOW_THREADS;
+    hr = storage_properties->Add(WPD_OBJECT_CONTENT_TYPE);
+    hr = storage_properties->Add(WPD_FUNCTIONAL_OBJECT_CATEGORY);
+    hr = storage_properties->Add(WPD_STORAGE_DESCRIPTION);
+    hr = storage_properties->Add(WPD_STORAGE_CAPACITY);
+    hr = storage_properties->Add(WPD_STORAGE_CAPACITY_IN_OBJECTS);
+    hr = storage_properties->Add(WPD_STORAGE_FREE_SPACE_IN_BYTES);
+    hr = storage_properties->Add(WPD_STORAGE_FREE_SPACE_IN_OBJECTS);
+    hr = storage_properties->Add(WPD_STORAGE_ACCESS_CAPABILITY);
+    hr = storage_properties->Add(WPD_STORAGE_FILE_SYSTEM_TYPE);
+    hr = storage_properties->Add(WPD_OBJECT_NAME);
+    Py_END_ALLOW_THREADS;
+    if (FAILED(hr)) {hresult_set_exc("Failed to create collection of properties for storage query", hr); goto end; }
+
+    Py_BEGIN_ALLOW_THREADS;
+    hr = content->EnumObjects(0, WPD_DEVICE_OBJECT_ID, NULL, &objects);
+    Py_END_ALLOW_THREADS;
+    if (FAILED(hr)) {hresult_set_exc("Failed to get objects from device", hr); goto end;}
+
+    hr = S_OK;
+    while (hr == S_OK) {
+        Py_BEGIN_ALLOW_THREADS;
+        hr = objects->Next(10, object_ids, &fetched);
+        Py_END_ALLOW_THREADS;
+        if (SUCCEEDED(hr)) {
+            for(i = 0; i < fetched; i++) {
+                Py_BEGIN_ALLOW_THREADS;
+                hr2 = properties->GetValues(object_ids[i], storage_properties, &values);
+                Py_END_ALLOW_THREADS;
+                if SUCCEEDED(hr2) {
+                    if (
+                        SUCCEEDED(values->GetGuidValue(WPD_OBJECT_CONTENT_TYPE, &guid)) && IsEqualGUID(guid, WPD_CONTENT_TYPE_FUNCTIONAL_OBJECT) &&
+                        SUCCEEDED(values->GetGuidValue(WPD_FUNCTIONAL_OBJECT_CATEGORY, &guid)) && IsEqualGUID(guid, WPD_FUNCTIONAL_CATEGORY_STORAGE)
+                       ) {
+                        capacity = 0; capacity_objects = 0; free_space = 0; free_objects = 0;
+                        values->GetUnsignedLargeIntegerValue(WPD_STORAGE_CAPACITY, &capacity);
+                        values->GetUnsignedLargeIntegerValue(WPD_STORAGE_CAPACITY_IN_OBJECTS, &capacity_objects);
+                        values->GetUnsignedLargeIntegerValue(WPD_STORAGE_FREE_SPACE_IN_BYTES, &free_space);
+                        values->GetUnsignedLargeIntegerValue(WPD_STORAGE_FREE_SPACE_IN_OBJECTS, &free_objects);
+                        desc = Py_False;
+                        if (SUCCEEDED(values->GetUnsignedIntegerValue(WPD_STORAGE_ACCESS_CAPABILITY, &access)) && access == WPD_STORAGE_ACCESS_CAPABILITY_READWRITE) desc = Py_True;
+                        soid = PyUnicode_FromWideChar(object_ids[i], wcslen(object_ids[i]));
+                        if (soid == NULL) { PyErr_NoMemory(); goto end; }
+                        so = Py_BuildValue("{s:K,s:K,s:K,s:K,s:O,s:N}", 
+                                "capacity", capacity, "capacity_objects", capacity_objects, "free_space", free_space, "free_objects", free_objects, "rw", desc, "id", soid);
+                        if (so == NULL) { PyErr_NoMemory(); goto end; }
+                        if (SUCCEEDED(values->GetStringValue(WPD_STORAGE_DESCRIPTION, &storage_desc))) {
+                                desc = PyUnicode_FromWideChar(storage_desc, wcslen(storage_desc));
+                                if (desc != NULL) { PyDict_SetItemString(so, "description", desc); Py_DECREF(desc);}
+                                CoTaskMemFree(storage_desc); storage_desc = NULL;
+                        }
+                        if (SUCCEEDED(values->GetStringValue(WPD_OBJECT_NAME, &storage_desc))) {
+                                desc = PyUnicode_FromWideChar(storage_desc, wcslen(storage_desc));
+                                if (desc != NULL) { PyDict_SetItemString(so, "name", desc); Py_DECREF(desc);}
+                                CoTaskMemFree(storage_desc); storage_desc = NULL;
+                        }
+                        if (SUCCEEDED(values->GetStringValue(WPD_STORAGE_FILE_SYSTEM_TYPE, &storage_desc))) {
+                                desc = PyUnicode_FromWideChar(storage_desc, wcslen(storage_desc));
+                                if (desc != NULL) { PyDict_SetItemString(so, "filesystem", desc); Py_DECREF(desc);}
+                                CoTaskMemFree(storage_desc); storage_desc = NULL;
+                        }
+                        PyList_Append(storage, so);
+                        Py_DECREF(so);
+                    }
+                }
+            }
+        }
+    }
+    ans = storage;
+
+end:
+    if (content != NULL) content->Release();
+    if (objects != NULL) objects->Release();
+    if (properties != NULL) properties->Release();
+    if (storage_properties != NULL) storage_properties->Release();
+    if (values != NULL) values->Release();
+    return ans;
+} // }}}
+
 PyObject* get_device_information(IPortableDevice *device) { // {{{
     IPortableDeviceContent *content = NULL;
     IPortableDeviceProperties *properties = NULL;
@@ -83,7 +196,7 @@ PyObject* get_device_information(IPortableDevice *device) { // {{{
     DWORD num_of_categories, i;
     LPWSTR temp;
     ULONG ti;
-    PyObject *t, *ans = NULL;
+    PyObject *t, *ans = NULL, *storage = NULL;
     char *type;
 
     Py_BEGIN_ALLOW_THREADS;
@@ -216,6 +329,13 @@ PyObject* get_device_information(IPortableDevice *device) { // {{{
     }
     PyDict_SetItemString(ans, "has_storage", t);
 
+    if (t == Py_True) {
+        storage = get_storage_info(device);
+        if (storage == NULL) goto end;
+        PyDict_SetItemString(ans, "storage", storage);
+        
+    }
+
 end:
     if (keys != NULL) keys->Release();
     if (values != NULL) values->Release();

src/calibre/devices/mtp/windows/driver.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import time
+from threading import RLock
+
+from calibre import as_unicode, prints
+from calibre.constants import plugins, __appname__, numeric_version
+from calibre.devices.mtp.base import MTPDeviceBase, synchronous
+
+class MTP_DEVICE(MTPDeviceBase):
+
+    supported_platforms = ['windows']
+
+    def __init__(self, *args, **kwargs):
+        MTPDeviceBase.__init__(self, *args, **kwargs)
+        self.dev = None
+        self.lock = RLock()
+        self.blacklisted_devices = set()
+        self.ejected_devices = set()
+        self.currently_connected_pnp_id = None
+        self.detected_devices = {}
+        self.previous_devices_on_system = frozenset()
+        self.last_refresh_devices_time = time.time()
+        self.wpd = self.wpd_error = None
+
+    @synchronous
+    def startup(self):
+        self.wpd, self.wpd_error = plugins['wpd']
+        if self.wpd is not None:
+            try:
+                self.wpd.init(__appname__, *(numeric_version[:3]))
+            except self.wpd.NoWPD:
+                self.wpd_error = _(
+                    'The Windows Portable Devices service is not available'
+                    ' on your computer. You may need to install Windows'
+                    ' Media Player 11 or newer and/or restart your computer')
+            except Exception as e:
+                self.wpd_error = as_unicode(e)
+
+    @synchronous
+    def shutdown(self):
+        if self.wpd is not None:
+            self.wpd.uninit()
+
+    @synchronous
+    def detect_managed_devices(self, devices_on_system):
+        if self.wpd is None: return None
+
+        devices_on_system = frozenset(devices_on_system)
+        if (devices_on_system != self.previous_devices_on_system or time.time()
+                - self.last_refresh_devices_time > 10):
+            self.previous_devices_on_system = devices_on_system
+            self.last_refresh_devices_time = time.time()
+            try:
+                pnp_ids = frozenset(self.wpd.enumerate_devices())
+            except:
+                return None
+
+            self.detected_devices = {dev:self.detected_devices.get(dev, None)
+                    for dev in pnp_ids}
+
+        # Get device data for detected devices. If there is an error, we will
+        # try again for that device the next time this method is called.
+        for dev in tuple(self.detected_devices.iterkeys()):
+            data = self.detected_devices.get(dev, None)
+            if data is None or data is False:
+                try:
+                    data = self.wpd.device_info(dev)
+                except Exception as e:
+                    prints('Failed to get device info for device:', dev,
+                            as_unicode(e))
+                    data = {} if data is False else False
+                self.detected_devices[dev] = data
+                # Remove devices that have been disconnected from ejected
+                # devices
+                self.ejected_devices = set(self.detected_devices).intersection(self.ejected_devices)
+
+        if self.currently_connected_pnp_id is not None:
+            return (self.currently_connected_pnp_id if
+                    self.currently_connected_pnp_id in self.detected_devices
+                    else None)
+
+        for dev, data in self.detected_devices.iteritems():
+            if dev in self.blacklisted_devices or dev in self.ejected_devices:
+                # Ignore blacklisted and ejected devices
+                continue
+            if data and self.is_suitable_wpd_device(data):
+                return dev
+
+        return None
+
+    def is_suitable_wpd_device(self, devdata):
+        # Check that protocol is MTP
+        protocol = devdata.get('protocol', '').lower()
+        if not protocol.startswith('mtp:'): return False
+
+        # Check that the device has some read-write storage
+        if not devdata.get('has_storage', False): return False
+        has_rw_storage = False
+        for s in devdata.get('storage', []):
+            if s.get('rw', False):
+                has_rw_storage = True
+                break
+        if not has_rw_storage: return False
+
+        return True
+
+    @synchronous
+    def post_yank_cleanup(self):
+        self.currently_connected_pnp_id = None
+
+    @synchronous
+    def eject(self):
+        if self.currently_connected_pnp_id is None: return
+        self.ejected_devices.add(self.currently_connected_pnp_id)
+        self.currently_connected_pnp_id = None
+
+

src/calibre/devices/mtp/windows/global.h

@@ -2,7 +2,7 @@
  * global.h
  * Copyright (C) 2012 Kovid Goyal <kovid at kovidgoyal.net>
  *
- * Distributed under terms of the MIT license.
+ * Distributed under terms of the GPL3 license.
  */
 
 #pragma once

src/calibre/devices/mtp/windows/remote.py

@@ -7,39 +7,66 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import subprocess, sys, os
+import subprocess, sys, os, pprint, signal, time, glob
-
+pprint
-def build():
-    builder = subprocess.Popen('ssh xp_build ~/build-wpd'.split())
-    syncer = subprocess.Popen('ssh getafix ~/test-wpd'.split())
-    if builder.wait() != 0:
-        raise Exception('Failed to build plugin')
-    if syncer.wait() != 0:
-        raise Exception('Failed to rsync to getafix')
-    subprocess.check_call(
-        'scp xp_build:build/calibre/src/calibre/plugins/wpd.pyd /tmp'.split())
-    subprocess.check_call(
-        'scp /tmp/wpd.pyd getafix:calibre/src/calibre/devices/mtp/windows'.split())
-    p = subprocess.Popen(
-        'ssh getafix calibre-debug -e calibre/src/calibre/devices/mtp/windows/remote.py'.split())
-    p.wait()
-    print()
 
+def build(mod='wpd'):
+    master = subprocess.Popen('ssh -MN getafix'.split())
+    master2 = subprocess.Popen('ssh -MN xp_build'.split())
+    try:
+        while not glob.glob(os.path.expanduser('~/.ssh/*kovid@xp_build*')):
+            time.sleep(0.05)
+        builder = subprocess.Popen('ssh xp_build ~/build-wpd'.split())
+        if builder.wait() != 0:
+            raise Exception('Failed to build plugin')
+        while not glob.glob(os.path.expanduser('~/.ssh/*kovid@getafix*')):
+            time.sleep(0.05)
+        syncer = subprocess.Popen('ssh getafix ~/test-wpd'.split())
+        if syncer.wait() != 0:
+            raise Exception('Failed to rsync to getafix')
+        subprocess.check_call(
+            ('scp xp_build:build/calibre/src/calibre/plugins/%s.pyd /tmp'%mod).split())
+        subprocess.check_call(
+            ('scp /tmp/%s.pyd getafix:calibre/src/calibre/devices/mtp/windows'%mod).split())
+        p = subprocess.Popen(
+            'ssh getafix calibre-debug -e calibre/src/calibre/devices/mtp/windows/remote.py'.split())
+        p.wait()
+        print()
+    finally:
+        for m in (master2, master):
+            m.send_signal(signal.SIGHUP)
+        for m in (master2, master):
+            m.wait()
 
 def main():
-    import pprint
+    fp, d = os.path.abspath(__file__), os.path.dirname
-    sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+    if b'CALIBRE_DEVELOP_FROM' not in os.environ:
+        env = os.environ.copy()
+        env[b'CALIBRE_DEVELOP_FROM'] = bytes(d(d(d(d(d(fp))))))
+        subprocess.call(['calibre-debug', '-e', fp], env=env)
+        return
+
+    sys.path.insert(0, os.path.dirname(fp))
+    if 'wpd' in sys.modules:
+        del sys.modules['wpd']
     import wpd
     from calibre.constants import plugins
     plugins._plugins['wpd'] = (wpd, '')
     sys.path.pop(0)
-    wpd.init('calibre', 1, 0, 0)
+
+    from calibre.devices.scanner import win_scanner
+    from calibre.devices.mtp.windows.driver import MTP_DEVICE
+    dev = MTP_DEVICE(None)
+    dev.startup()
+    print (dev.wpd, dev.wpd_error)
+
     try:
-        for pnp_id in wpd.enumerate_devices():
+        devices = win_scanner()
-            print (pnp_id)
+        pnp_id = dev.detect_managed_devices(devices)
-            pprint.pprint(wpd.device_info(pnp_id))
+        # pprint.pprint(dev.detected_devices)
+        print ('Trying to connect to:', pnp_id)
     finally:
-        wpd.uninit()
+        dev.shutdown()
 
 if __name__ == '__main__':
     main()

src/calibre/devices/mtp/windows/utils.cpp

@@ -2,7 +2,7 @@
  * utils.cpp
  * Copyright (C) 2012 Kovid Goyal <kovid at kovidgoyal.net>
  *
- * Distributed under terms of the MIT license.
+ * Distributed under terms of the GPL3 license.
  */
 
 #include "global.h"
@@ -33,13 +33,12 @@ PyObject *wpd::hresult_set_exc(const char *msg, HRESULT hr) {
 wchar_t *wpd::unicode_to_wchar(PyObject *o) {
     wchar_t *buf;
     Py_ssize_t len;
-    if (!PyUnicode_Check(o)) {PyErr_Format(PyExc_TypeError, "The pnp id must be a unicode object"); return NULL;}
+    if (!PyUnicode_Check(o)) {PyErr_Format(PyExc_TypeError, "The python object must be a unicode object"); return NULL;}
     len = PyUnicode_GET_SIZE(o);
-    if (len < 1) {PyErr_Format(PyExc_TypeError, "The pnp id must not be empty."); return NULL;}
     buf = (wchar_t *)calloc(len+2, sizeof(wchar_t));
     if (buf == NULL) { PyErr_NoMemory(); return NULL; }
     len = PyUnicode_AsWideChar((PyUnicodeObject*)o, buf, len);
-    if (len == -1) { free(buf); PyErr_Format(PyExc_TypeError, "Invalid pnp id."); return NULL; }
+    if (len == -1) { free(buf); PyErr_Format(PyExc_TypeError, "Invalid python unicode object."); return NULL; }
     return buf;
 }
 

src/calibre/devices/mtp/windows/wpd.cpp

@@ -2,7 +2,7 @@
  * mtp.c
  * Copyright (C) 2012 Kovid Goyal <kovid at kovidgoyal.net>
  *
- * Distributed under terms of the MIT license.
+ * Distributed under terms of the GPL3 license.
  */
 
 #include "global.h"
@@ -92,14 +92,10 @@ wpd_enumerate_devices(PyObject *self, PyObject *args) {
 
     ENSURE_WPD(NULL);
 
-    if (!PyArg_ParseTuple(args, "|O", &refresh)) return NULL;
+    Py_BEGIN_ALLOW_THREADS;
-
+    hr = portable_device_manager->RefreshDeviceList();
-    if (refresh != NULL && PyObject_IsTrue(refresh)) {
+    Py_END_ALLOW_THREADS;
-        Py_BEGIN_ALLOW_THREADS;
+    if (FAILED(hr)) return hresult_set_exc("Failed to refresh the list of portable devices", hr);
-        hr = portable_device_manager->RefreshDeviceList();
-        Py_END_ALLOW_THREADS;
-        if (FAILED(hr)) return hresult_set_exc("Failed to refresh the list of portable devices", hr);
-    }
 
     hr = portable_device_manager->GetDevices(NULL, &num_of_devices);
     num_of_devices += 15; // Incase new devices were connected between this call and the next
@@ -148,6 +144,7 @@ wpd_device_info(PyObject *self, PyObject *args) {
 
     if (!PyArg_ParseTuple(args, "O", &py_pnp_id)) return NULL;
     pnp_id = unicode_to_wchar(py_pnp_id);
+    if (wcslen(pnp_id) < 1) { PyErr_SetString(WPDError, "The PNP id must not be empty."); return NULL; }
     if (pnp_id == NULL) return NULL;
 
     client_information = get_client_information();
@@ -174,7 +171,7 @@ static PyMethodDef wpd_methods[] = {
     },
 
     {"enumerate_devices", wpd_enumerate_devices, METH_VARARGS,
-        "enumerate_devices(refresh=False)\n\n Get the list of device PnP ids for all connected devices recognized by the WPD service. The result is cached, unless refresh=True. Do not call with refresh=True too often as it is resource intensive."
+        "enumerate_devices()\n\n Get the list of device PnP ids for all connected devices recognized by the WPD service. Do not call too often as it is resource intensive."
     },
 
     {"device_info", wpd_device_info, METH_VARARGS,

src/calibre/devices/scanner.py

@@ -10,7 +10,8 @@ from threading import RLock
 from collections import namedtuple
 
 from calibre import prints, as_unicode
-from calibre.constants import iswindows, isosx, plugins, islinux, isfreebsd
+from calibre.constants import (iswindows, isosx, plugins, islinux, isfreebsd,
+        isnetbsd)
 
 osx_scanner = win_scanner = linux_scanner = None
 
@@ -253,13 +254,18 @@ freebsd_scanner = None
 if isfreebsd:
     freebsd_scanner = FreeBSDScanner()
 
+netbsd_scanner = None
+
+''' NetBSD support currently not written yet '''
+if isnetbsd:
+    netbsd_scanner = None
 
 class DeviceScanner(object):
 
     def __init__(self, *args):
         if isosx and osx_scanner is None:
             raise RuntimeError('The Python extension usbobserver must be available on OS X.')
-        self.scanner = win_scanner if iswindows else osx_scanner if isosx else freebsd_scanner if isfreebsd else linux_scanner
+        self.scanner = win_scanner if iswindows else osx_scanner if isosx else freebsd_scanner if isfreebsd else netbsd_scanner if isnetbsd else linux_scanner
         self.devices = []
 
     def scan(self):

src/calibre/gui2/catalog/catalog_epub_mobi.py

@@ -15,8 +15,9 @@ from calibre.utils.icu import sort_key
 
 from catalog_epub_mobi_ui import Ui_Form
 from PyQt4.Qt import (Qt, QAbstractItemView, QCheckBox, QComboBox,
-        QDoubleSpinBox, QIcon, QLineEdit, QRadioButton, QSize, QSizePolicy,
+        QDoubleSpinBox, QIcon, QLineEdit, QObject, QRadioButton, QSize, QSizePolicy,
-        QTableWidget, QTableWidgetItem, QToolButton, QVBoxLayout, QWidget)
+        QTableWidget, QTableWidgetItem, QToolButton, QVBoxLayout, QWidget,
+        SIGNAL)
 
 class PluginWidget(QWidget,Ui_Form):
 
@@ -454,6 +455,8 @@ class GenericRulesTable(QTableWidget):
     Add QTableWidget, controls to parent QGroupBox
     placeholders for basic methods to be overriden
     '''
+    FOCUS_SWITCHING = True
+    DEBUG = False
 
     def __init__(self, parent_gb, object_name, rules, eligible_custom_fields, db):
         self.rules = rules
@@ -476,11 +479,15 @@ class GenericRulesTable(QTableWidget):
         self.setRowCount(0)
         self.layout.addWidget(self)
 
-        self.last_row_selected = self.currentRow()
+        if self.FOCUS_SWITCHING:
-        self.last_rows_selected = self.selectionModel().selectedRows()
+            self.last_row_selected = self.currentRow()
+            self.last_rows_selected = self.selectionModel().selectedRows()
 
         self._init_controls()
 
+        # Hook check_box changes. Everything else is already hooked
+        QObject.connect(self, SIGNAL('cellChanged(int,int)'), self.enabled_state_changed)
+
     def _init_controls(self):
         # Add the control set
         vbl = QVBoxLayout()
@@ -516,7 +523,12 @@ class GenericRulesTable(QTableWidget):
 
     def add_row(self):
         self.setFocus()
-        row = self.last_row_selected + 1
+        if self.FOCUS_SWITCHING:
+            row = self.last_row_selected + 1
+        else:
+            row = self.currentRow() + 1
+        if self.DEBUG and self.FOCUS_SWITCHING:
+            print("%s:add_row(): last_row_selected: %d, row: %d" % (self.objectName(), self.last_row_selected, row))
         self.insertRow(row)
         self.populate_table_row(row, self.create_blank_row_data())
         self.select_and_scroll_to_row(row)
@@ -538,7 +550,10 @@ class GenericRulesTable(QTableWidget):
 
     def delete_row(self):
         self.setFocus()
-        rows = self.last_rows_selected
+        if self.FOCUS_SWITCHING:
+            rows = self.last_rows_selected
+        else:
+            rows = self.selectionModel().selectedRows()
         if len(rows) == 0:
             return
 
@@ -558,11 +573,18 @@ class GenericRulesTable(QTableWidget):
         elif self.rowCount() > 0:
             self.select_and_scroll_to_row(first_sel_row - 1)
 
+    def focusInEvent(self,e):
+        if self.DEBUG:
+            print("%s:focusInEvent()" % self.objectName())
+
     def focusOutEvent(self,e):
         # Override of QTableWidget method - clear selection when table loses focus
-        self.last_row_selected = self.currentRow()
+        if self.FOCUS_SWITCHING:
-        self.last_rows_selected = self.selectionModel().selectedRows()
+            self.last_row_selected = self.currentRow()
-        self.clearSelection()
+            self.last_rows_selected = self.selectionModel().selectedRows()
+            self.clearSelection()
+            if self.DEBUG:
+                print("%s:focusOutEvent(): self.last_row_selected: %d" % (self.objectName(),self.last_row_selected))
 
     def get_data(self):
         '''
@@ -572,7 +594,10 @@ class GenericRulesTable(QTableWidget):
 
     def move_row_down(self):
         self.setFocus()
-        rows = self.last_rows_selected
+        if self.FOCUS_SWITCHING:
+            rows = self.last_rows_selected
+        else:
+            rows = self.selectionModel().selectedRows()
         if len(rows) == 0:
             return
         last_sel_row = rows[-1].row()
@@ -598,13 +623,16 @@ class GenericRulesTable(QTableWidget):
 
         self.blockSignals(False)
         scroll_to_row = last_sel_row + 1
-        if scroll_to_row < self.rowCount() - 1:
+        #if scroll_to_row < self.rowCount() - 1:
-            scroll_to_row = scroll_to_row + 1
+        #    scroll_to_row = scroll_to_row + 1
         self.select_and_scroll_to_row(scroll_to_row)
 
     def move_row_up(self):
         self.setFocus()
-        rows = self.last_rows_selected
+        if self.FOCUS_SWITCHING:
+            rows = self.last_rows_selected
+        else:
+            rows = self.selectionModel().selectedRows()
         if len(rows) == 0:
             return
         first_sel_row = rows[0].row()
@@ -623,10 +651,14 @@ class GenericRulesTable(QTableWidget):
             self.removeRow(selrow.row() - 1)
         self.blockSignals(False)
 
-        scroll_to_row = first_sel_row - 1
+        scroll_to_row = first_sel_row
         if scroll_to_row > 0:
             scroll_to_row = scroll_to_row - 1
         self.select_and_scroll_to_row(scroll_to_row)
+        if self.DEBUG:
+            print("%s:move_row_up(): first_sel_row: %d" % (self.objectName(), first_sel_row))
+            print("%s:move_row_up(): scroll_to_row: %d" % (self.objectName(), scroll_to_row))
+            print("%s move_row_down(): current_row: %d" % (self.objectName(), self.currentRow()))
 
     def populate_table_row(self):
         '''
@@ -642,13 +674,69 @@ class GenericRulesTable(QTableWidget):
     def rule_name_edited(self):
         current_row = self.currentRow()
         self.cellWidget(current_row,1).home(False)
-        self.setFocus()
         self.select_and_scroll_to_row(current_row)
 
     def select_and_scroll_to_row(self, row):
+        self.setFocus()
         self.selectRow(row)
         self.scrollToItem(self.currentItem())
 
+    def _source_index_changed(self, combo):
+        # Figure out which row we're in
+        for row in range(self.rowCount()):
+            if self.cellWidget(row, self.COLUMNS['FIELD']['ordinal']) is combo:
+                break
+
+        if self.DEBUG:
+            print("%s:_source_index_changed(): calling source_index_changed with row: %d " %
+                  (self.objectName(), row))
+
+        self.source_index_changed(combo, row)
+
+    def source_index_changed(self, combo, row, pattern=''):
+        # Populate the Pattern field based upon the Source field
+
+        source_field = str(combo.currentText())
+        if source_field == '':
+            values = []
+        elif source_field == 'Tags':
+            values = sorted(self.db.all_tags(), key=sort_key)
+        else:
+            if self.eligible_custom_fields[unicode(source_field)]['datatype'] in ['enumeration', 'text']:
+                values = self.db.all_custom(self.db.field_metadata.key_to_label(
+                                            self.eligible_custom_fields[unicode(source_field)]['field']))
+                values = sorted(values, key=sort_key)
+            elif self.eligible_custom_fields[unicode(source_field)]['datatype'] in ['bool']:
+                values = [_('True'),_('False'),_('unspecified')]
+            elif self.eligible_custom_fields[unicode(source_field)]['datatype'] in ['composite']:
+                values = [_('any value'),_('unspecified')]
+            elif self.eligible_custom_fields[unicode(source_field)]['datatype'] in ['datetime']:
+                values = [_('any date'),_('unspecified')]
+
+        values_combo = ComboBox(self, values, pattern)
+        values_combo.currentIndexChanged.connect(partial(self.values_index_changed, values_combo))
+        self.setCellWidget(row, self.COLUMNS['PATTERN']['ordinal'], values_combo)
+        self.select_and_scroll_to_row(row)
+
+    def values_index_changed(self, combo):
+        # After edit, select row
+        for row in range(self.rowCount()):
+            if self.cellWidget(row, self.COLUMNS['PATTERN']['ordinal']) is combo:
+                self.select_and_scroll_to_row(row)
+                break
+
+        if self.DEBUG:
+            print("%s:values_index_changed(): row %d " %
+                  (self.objectName(), row))
+
+    def enabled_state_changed(self, row, col):
+        # After state change, select row
+        if col in [self.COLUMNS['ENABLED']['ordinal']]:
+            self.select_and_scroll_to_row(row)
+            if self.DEBUG:
+                print("%s:enabled_state_changed(): row %d col %d" %
+                      (self.objectName(), row, col))
+
 class ExclusionRules(GenericRulesTable):
 
     COLUMNS = { 'ENABLED':{'ordinal': 0, 'name': ''},
@@ -658,6 +746,7 @@ class ExclusionRules(GenericRulesTable):
 
     def __init__(self, parent_gb_hl, object_name, rules, eligible_custom_fields, db):
         super(ExclusionRules, self).__init__(parent_gb_hl, object_name, rules, eligible_custom_fields, db)
+        self.setObjectName("exclusion_rules_table")
         self._init_table_widget()
         self._initialize()
 
@@ -730,7 +819,7 @@ class ExclusionRules(GenericRulesTable):
 
         def set_source_field_in_row(row, col, field=''):
             source_combo = ComboBox(self, sorted(self.eligible_custom_fields.keys(), key=sort_key), field)
-            source_combo.currentIndexChanged.connect(partial(self.source_index_changed, source_combo, row))
+            source_combo.currentIndexChanged.connect(partial(self._source_index_changed, source_combo))
             self.setCellWidget(row, col, source_combo)
             return source_combo
 
@@ -738,7 +827,8 @@ class ExclusionRules(GenericRulesTable):
         self.blockSignals(True)
 
         # Enabled
-        self.setItem(row, self.COLUMNS['ENABLED']['ordinal'], CheckableTableWidgetItem(data['enabled']))
+        check_box = CheckableTableWidgetItem(data['enabled'])
+        self.setItem(row, self.COLUMNS['ENABLED']['ordinal'], check_box)
 
         # Rule name
         set_rule_name_in_row(row, self.COLUMNS['NAME']['ordinal'], name=data['name'])
@@ -748,32 +838,10 @@ class ExclusionRules(GenericRulesTable):
 
         # Pattern
         # The contents of the Pattern field is driven by the Source field
-        self.source_index_changed(source_combo, row, self.COLUMNS['PATTERN']['ordinal'], pattern=data['pattern'])
+        self.source_index_changed(source_combo, row, pattern=data['pattern'])
 
         self.blockSignals(False)
 
-    def source_index_changed(self, combo, row, col, pattern=''):
-        # Populate the Pattern field based upon the Source field
-        source_field = str(combo.currentText())
-        if source_field == '':
-            values = []
-        elif source_field == 'Tags':
-            values = sorted(self.db.all_tags(), key=sort_key)
-        else:
-            if self.eligible_custom_fields[unicode(source_field)]['datatype'] in ['enumeration', 'text']:
-                values = self.db.all_custom(self.db.field_metadata.key_to_label(
-                                            self.eligible_custom_fields[unicode(source_field)]['field']))
-                values = sorted(values, key=sort_key)
-            elif self.eligible_custom_fields[unicode(source_field)]['datatype'] in ['bool']:
-                values = ['True','False','unspecified']
-            elif self.eligible_custom_fields[unicode(source_field)]['datatype'] in ['composite']:
-                values = ['any value','unspecified']
-            elif self.eligible_custom_fields[unicode(source_field)]['datatype'] in ['datetime']:
-                values = ['any date','unspecified']
-
-        values_combo = ComboBox(self, values, pattern)
-        self.setCellWidget(row, self.COLUMNS['PATTERN']['ordinal'], values_combo)
-
 class PrefixRules(GenericRulesTable):
 
     COLUMNS = { 'ENABLED':{'ordinal': 0, 'name': ''},
@@ -784,6 +852,7 @@ class PrefixRules(GenericRulesTable):
 
     def __init__(self, parent_gb_hl, object_name, rules, eligible_custom_fields, db):
         super(PrefixRules, self).__init__(parent_gb_hl, object_name, rules, eligible_custom_fields, db)
+        self.setObjectName("prefix_rules_table")
         self._init_table_widget()
         self._initialize()
 
@@ -998,14 +1067,12 @@ class PrefixRules(GenericRulesTable):
 
         def set_source_field_in_row(row, col, field=''):
             source_combo = ComboBox(self, sorted(self.eligible_custom_fields.keys(), key=sort_key), field)
-            source_combo.currentIndexChanged.connect(partial(self.source_index_changed, source_combo, row))
+            source_combo.currentIndexChanged.connect(partial(self._source_index_changed, source_combo))
             self.setCellWidget(row, col, source_combo)
             return source_combo
 
-
         # Entry point
         self.blockSignals(True)
-        #print("prefix_rules_populate_table_row processing rule:\n%s\n" % data)
 
         # Enabled
         self.setItem(row, self.COLUMNS['ENABLED']['ordinal'], CheckableTableWidgetItem(data['enabled']))
@@ -1021,31 +1088,7 @@ class PrefixRules(GenericRulesTable):
 
         # Pattern
         # The contents of the Pattern field is driven by the Source field
-        self.source_index_changed(source_combo, row, self.COLUMNS['PATTERN']['ordinal'], pattern=data['pattern'])
+        self.source_index_changed(source_combo, row, pattern=data['pattern'])
 
         self.blockSignals(False)
 
-    def source_index_changed(self, combo, row, col, pattern=''):
-        # Populate the Pattern field based upon the Source field
-        # row, col are the control that changed
-
-        source_field = str(combo.currentText())
-        if source_field == '':
-            values = []
-        elif source_field == 'Tags':
-            values = sorted(self.db.all_tags(), key=sort_key)
-        else:
-            if self.eligible_custom_fields[unicode(source_field)]['datatype'] in ['enumeration', 'text']:
-                values = self.db.all_custom(self.db.field_metadata.key_to_label(
-                                            self.eligible_custom_fields[unicode(source_field)]['field']))
-                values = sorted(values, key=sort_key)
-            elif self.eligible_custom_fields[unicode(source_field)]['datatype'] in ['bool']:
-                values = ['True','False','unspecified']
-            elif self.eligible_custom_fields[unicode(source_field)]['datatype'] in ['composite']:
-                values = ['any value','unspecified']
-            elif self.eligible_custom_fields[unicode(source_field)]['datatype'] in ['datetime']:
-                values = ['any date','unspecified']
-
-        values_combo = ComboBox(self, values, pattern)
-        self.setCellWidget(row, self.COLUMNS['PATTERN']['ordinal'], values_combo)
-

src/calibre/library/catalogs/epub_mobi_builder.py

@@ -1191,8 +1191,7 @@ Author '{0}':
 
                 if self.opts.generate_series:
                     aTag = Tag(soup,'a')
-                    aTag['href'] = "%s.html#%s_series" % ('BySeries',
+                    aTag['href'] = "%s.html#%s" % ('BySeries',self.generateSeriesAnchor(book['series']))
-                                                    re.sub('\s','',book['series']).lower())
                     aTag.insert(0, book['series'])
                     pSeriesTag.insert(0, aTag)
                 else:
@@ -1333,8 +1332,9 @@ Author '{0}':
                         pSeriesTag['class'] = "series"
                         if self.opts.generate_series:
                             aTag = Tag(soup,'a')
-                            aTag['href'] = "%s.html#%s_series" % ('BySeries',
+
-                                                            re.sub('\W','',new_entry['series']).lower())
+                            if self.letter_or_symbol(new_entry['series']) == self.SYMBOLS:
+                                aTag['href'] = "%s.html#%s" % ('BySeries',self.generateSeriesAnchor(new_entry['series']))
                             aTag.insert(0, new_entry['series'])
                             pSeriesTag.insert(0, aTag)
                         else:
@@ -1741,17 +1741,6 @@ Author '{0}':
         body = soup.find('body')
 
         btc = 0
-
-        pTag = Tag(soup, "p")
-        pTag['style'] = 'display:none'
-        ptc = 0
-        aTag = Tag(soup,'a')
-        aTag['id'] = 'section_start'
-        pTag.insert(ptc, aTag)
-        ptc += 1
-        body.insert(btc, pTag)
-        btc += 1
-
         divTag = Tag(soup, "div")
         dtc = 0
         current_letter = ""
@@ -1788,10 +1777,7 @@ Author '{0}':
                 pSeriesTag = Tag(soup,'p')
                 pSeriesTag['class'] = "series"
                 aTag = Tag(soup, 'a')
-                if self.letter_or_symbol(book['series']):
+                aTag['id'] = self.generateSeriesAnchor(book['series'])
-                    aTag['id'] = "symbol_%s_series" % re.sub('\W','',book['series']).lower()
-                else:
-                    aTag['id'] = "%s_series" % re.sub('\W','',book['series']).lower()
                 pSeriesTag.insert(0,aTag)
                 pSeriesTag.insert(1,NavigableString('%s' % book['series']))
                 divTag.insert(dtc,pSeriesTag)
@@ -1847,19 +1833,23 @@ Author '{0}':
             divTag.insert(dtc, pBookTag)
             dtc += 1
 
+        pTag = Tag(soup, "p")
+        pTag['class'] = 'title'
+        ptc = 0
+        aTag = Tag(soup,'a')
+        aTag['id'] = 'section_start'
+        pTag.insert(ptc, aTag)
+        ptc += 1
+
         if not self.__generateForKindle:
             # Insert the <h2> tag with book_count at the head
-            #<h2><a name="byseries" id="byseries"></a>By Series</h2>
-            pTag = Tag(soup, "p")
-            pTag['class'] = 'title'
             aTag = Tag(soup, "a")
             anchor_name = friendly_name.lower()
             aTag['id'] = anchor_name.replace(" ","")
             pTag.insert(0,aTag)
-            #h2Tag.insert(1,NavigableString('%s (%d)' % (friendly_name, series_count)))
             pTag.insert(1,NavigableString('%s' % friendly_name))
-            body.insert(btc,pTag)
+        body.insert(btc,pTag)
-            btc += 1
+        btc += 1
 
         # Add the divTag to the body
         body.insert(btc, divTag)
@@ -3353,15 +3343,17 @@ Author '{0}':
             return codeTag
         else:
             spanTag = Tag(soup, "span")
+            #spanTag['class'] = "prefix"
             if prefix_char is None:
                 spanTag['style'] = "color:white"
                 prefix_char = self.defaultPrefix
+                #prefix_char = "&nbsp;"
             spanTag.insert(0,NavigableString(prefix_char))
             return spanTag
 
     def generateAuthorAnchor(self, author):
-        # Strip white space to ''
+        # Generate a legal XHTML id/href string
-        return re.sub("\W","", author)
+        return re.sub("\W","", ascii_text(author))
 
     def generateFormatArgs(self, book):
         series_index = str(book['series_index'])
@@ -3438,8 +3430,7 @@ Author '{0}':
                 pSeriesTag['class'] = "series"
                 if self.opts.generate_series:
                     aTag = Tag(soup,'a')
-                    aTag['href'] = "%s.html#%s_series" % ('BySeries',
+                    aTag['href'] = "%s.html#%s" % ('BySeries', self.generateSeriesAnchor(book['series']))
-                                                    re.sub('\W','',book['series']).lower())
                     aTag.insert(0, book['series'])
                     pSeriesTag.insert(0, aTag)
                 else:
@@ -3641,12 +3632,7 @@ Author '{0}':
         if aTag:
             if book['series']:
                 if self.opts.generate_series:
-                    if self.letter_or_symbol(book['series']):
+                    aTag['href'] = "%s.html#%s" % ('BySeries',self.generateSeriesAnchor(book['series']))
-                        aTag['href'] = "%s.html#symbol_%s_series" % ('BySeries',
-                                    re.sub('\W','',book['series']).lower())
-                    else:
-                        aTag['href'] = "%s.html#%s_series" % ('BySeries',
-                                    re.sub('\s','',book['series']).lower())
             else:
                 aTag.extract()
 
@@ -3780,6 +3766,14 @@ Author '{0}':
             pass
         return rating
 
+    def generateSeriesAnchor(self, series):
+        # Generate a legal XHTML id/href string
+        if self.letter_or_symbol(series) == self.SYMBOLS:
+            return "symbol_%s_series" % re.sub('\W','',series).lower()
+        else:
+            return "%s_series" % re.sub('\W','',ascii_text(series)).lower()
+
+
     def generateShortDescription(self, description, dest=None):
         # Truncate the description, on word boundaries if necessary
         # Possible destinations:

src/calibre/library/database2.py

@@ -640,12 +640,12 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             if name and name != fname:
                 changed = True
                 break
-        if path == current_path and not changed:
-            return
-
         tpath = os.path.join(self.library_path, *path.split('/'))
         if not os.path.exists(tpath):
             os.makedirs(tpath)
+        if path == current_path and not changed:
+            return
+
         spath = os.path.join(self.library_path, *current_path.split('/'))
 
         if current_path and os.path.exists(spath): # Migrate existing files
@@ -1150,7 +1150,16 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
 
         `data`: Can be either a QImage, QPixmap, file object or bytestring
         '''
-        path = os.path.join(self.library_path, self.path(id, index_is_id=True), 'cover.jpg')
+        base_path = os.path.join(self.library_path, self.path(id,
+            index_is_id=True))
+        if not os.path.exists(base_path):
+            self.set_path(id, index_is_id=True)
+            base_path = os.path.join(self.library_path, self.path(id,
+                index_is_id=True))
+            self.dirtied([id])
+
+        path = os.path.join(base_path, 'cover.jpg')
+
         if callable(getattr(data, 'save', None)):
             data.save(path)
         else: